RL

// Basic — prompts only
{"messages": [{"role": "user", "content": "Search for AI news"}]}

// With expected answer — for Task Completion / JSON Validation reward
{"messages": [...], "metadata": {"expected_answer": "4"}}

// With rubric — for LLM-as-a-Judge reward
{"messages": [...], "metadata": {"rubric": "Rate quality"}}

// With expected tools — for Tool Usage reward
{"messages": [...], "metadata": {"expected_tools": ["search", "calc"]}}

from typing import Any

def grade(sample: Any, item: Any) -> float:
    """
    Evaluate agent trajectory and return a reward score.

    Args:
        sample: Agent output generated during training.
            - trajectories: list of steps, each with:
                - response: str — agent's text output
                - tool_calls: list[dict] — functions the agent called
                - tool_results: list — results from tool execution
        item: One row from your uploaded JSONL dataset.
            - prompt: the original user message
            - metadata: your custom fields (e.g. expected_answer, rubric)

    Returns:
        float: reward score for this trajectory
    """
    ...