FinQA

AgentFlow.run(task, config)
  │
  └── Multi-turn loop (up to 20 turns, native OpenAI tool calls)
        │
        ├── client.chat.completions.create(messages, tools=TOOL_SPECS)
        │
        ├── If msg.tool_calls is empty → that's the final answer, break.
        │
        └── Else: dispatch each tool call → append a `tool` message → repeat.
              (track table_name in `accessed_tables` whenever
               `get_table_info` is invoked, for the table-access bonus)
  │
  └── episode.artifacts = {"answer": full_response, "accessed_tables": [...], "turns": N}

Evaluator.evaluate(task, episode)
  │
  └── extract FINAL ANSWER from artifacts["answer"], grade via judge LLM,
      add table-access bonus, return EvalOutput.

uv pip install -e ".[tinker]"                      # rllm + tinker backend
uv pip install --no-deps -e cookbooks/finqa        # this cookbook
rllm agent list                                    # should show "finqa"

python cookbooks/finqa/prepare_data.py

export OPENAI_API_KEY=sk-…

rllm eval finqa \
    --agent finqa \
    --evaluator finqa \
    --model rLLM/rLLM-FinQA-4B \
    --base-url http://localhost:30000/v1 \
    --split test \
    --max-examples 20

export OPENAI_API_KEY=sk-…

# Tinker (LoRA on 30B)
bash cookbooks/finqa/train_tinker.sh

# Verl (distributed multi-GPU)
bash cookbooks/finqa/train_verl.sh

@rllm.rollout(name="finqa")
async def finqa_flow(task: Task, config: AgentConfig) -> Episode:
    client = AsyncOpenAI(base_url=config.base_url, api_key="EMPTY")
    messages = [
        {"role": "system", "content": SYSTEM_PROMPT},
        {"role": "user", "content": task.metadata.get("question")},
    ]

    accessed_tables, steps, final = [], [], ""
    for turn in range(MAX_TURNS):
        resp = await client.chat.completions.create(
            model=config.model, messages=messages, tools=TOOL_SPECS, ...,
        )
        msg = resp.choices[0].message
        messages.append(_msg_to_dict(msg))
        steps.append(Step(chat_completions=list(messages), model_response=msg.content or "", ...))

        if not msg.tool_calls:
            final = msg.content or ""
            break

        for tc in msg.tool_calls:
            output = _exec_tool_call(tc, accessed_tables)
            messages.append({"role": "tool", "tool_call_id": tc.id, "content": output})

    return Episode(
        trajectories=[Trajectory(name="finqa", steps=steps)],
        artifacts={"answer": final, "accessed_tables": accessed_tables, "turns": len(steps)},
    )

TOOL_FNS = {
    "get_table_names": get_table_names,    # list company → tables
    "get_table_info": get_table_info,      # column metadata + sample values
    "sql_query": sql_query,                 # filtered SELECT against in-memory DB
    "calculator": calculator,               # asteval safe arithmetic
}

TOOL_SPECS = [
    {"type": "function", "function": {"name": "get_table_names", ...}},
    {"type": "function", "function": {"name": "get_table_info", ...}},
    ...
]