FrozenLake

uv pip install -e ".[tinker]"                          # rllm + tinker backend
uv pip install --no-deps -e cookbooks/frozenlake       # this cookbook (gymnasium pulled in transitively)
rllm agent list                                        # should show "frozenlake"

python cookbooks/frozenlake/prepare_data.py
# Or with custom sizes:
python cookbooks/frozenlake/prepare_data.py --train-size 5000 --test-size 200 --slippery

rllm eval frozenlake \
    --agent frozenlake \
    --evaluator frozenlake \
    --model Qwen/Qwen3-4B-Instruct-2507 \
    --base-url http://localhost:8000/v1 \
    --split test \
    --max-examples 20

# Single-machine LoRA (tinker)
bash cookbooks/frozenlake/train_tinker.sh

# Distributed multi-GPU (verl)
bash cookbooks/frozenlake/train_verl.sh

rllm train frozenlake \
    --agent frozenlake \
    --evaluator frozenlake \
    --model Qwen/Qwen3-4B-Instruct-2507 \
    --group-size 8 \
    --batch-size 32 \
    --lora-rank 32

@rllm.rollout(name="frozenlake")
async def frozenlake_flow(task: Task, config: AgentConfig) -> Episode:
    meta = task.metadata or {}
    desc = generate_random_map(size=meta["size"], p=meta["p"], seed=meta["seed"])
    env = gym.make("FrozenLake-v1", desc=desc, is_slippery=meta["is_slippery"])
    env.reset(seed=meta["seed"])

    client = AsyncOpenAI(base_url=config.base_url, api_key="EMPTY")
    messages = [
        {"role": "system", "content": SYSTEM_PROMPT},
        {"role": "user", "content": render_first_turn(env, max_turns)},
    ]

    steps, won = [], False
    for turn in range(max_turns):
        resp = await client.chat.completions.create(model=config.model, messages=messages, ...)
        content = resp.choices[0].message.content or ""
        action = parse_action(content)         # e.g. "```Up```" → 3
        messages.append({"role": "assistant", "content": content})
        steps.append(Step(chat_completions=list(messages), action=_ACTION_LABELS.get(action), …))

        if action is None:
            messages.append({"role": "user", "content": "Please reply with a valid action…"})
            continue

        _, reward, terminated, truncated, _ = env.step(action)
        if terminated:
            won = float(reward) > 0
            break
        if truncated:
            break
        messages.append({"role": "user", "content": render_next_turn(env, turn + 1)})

    return Episode(
        trajectories=[Trajectory(name="frozenlake", steps=steps)],
        artifacts={"won": won, "turns": len(steps)},
        is_correct=won,
    )