simulation glue code

examples/drive-thru/agent.py

@@ -485,7 +485,7 @@ async def drive_thru_agent(ctx: JobContext) -> None:
                 ],
             },
         ),
-        llm=inference.LLM("openai/gpt-5-mini"),
+        llm=inference.LLM("openai/gpt-4.1-mini"),
         tts=inference.TTS("cartesia/sonic-3", voice="f786b574-daa5-4673-aa0c-cbe3e8534c02"),
         turn_detection=MultilingualModel(),
         vad=silero.VAD.load(),

examples/drive-thru/database.py

@@ -6,7 +6,7 @@
 from pydantic import BaseModel
 
 COMMON_INSTRUCTIONS = (
-    "You are Mac, a quick and friendly McDonald’s drive-thru attendant. \n"
+    "You are a quick and friendly McDonald’s drive-thru attendant. \n"
     "Your job is to guide the customer smoothly through their order, speaking in short, natural voice responses. \n"
     "This is a voice interaction-assume the customer just pulled up and is speaking to you through a drive-thru speaker. \n"
     "Respond like you're hearing them, not reading text. \n"

examples/hotel_receptionist/agent.py

@@ -9,6 +9,7 @@
 
 sys.path.append(os.path.dirname(os.path.abspath(__file__)))
 
+from benchmark import build_expected, diff_databases
 from book_restaurant import BookRestaurantTask
 from book_room import BookRoomTask
 from dotenv import load_dotenv
@@ -40,6 +41,7 @@
     AgentSession,
     JobContext,
     RunContext,
+    SimulationContext,
     ToolError,
     cli,
     function_tool,
@@ -585,6 +587,30 @@ def _speak_code(code: str) -> str:
 _SEED_DB_BYTES = build_seed_bytes(TODAY)
 
 
+async def on_simulation_end(ctx: SimulationContext) -> None:
+    # Grade the run on final DB state: build the scenario's `expected_state` on a
+    # fresh seed, then diff it against the agent's DB. The diff compares
+    # agent-decided facts only (room type, dates, extras, status), so minted
+    # codes / order / which-king don't matter and the agent need not reproduce the
+    # statements — while collateral damage still surfaces.
+    expected_state = ctx.userdata().get("expected_state") or []
+    if not expected_state:
+        return
+
+    session = ctx.job_context.primary_session
+    expected = await build_expected(_SEED_DB_BYTES, expected_state)
+    try:
+        diffs = diff_databases(expected.connection, session.userdata.db.connection)
+    finally:
+        await expected.aclose()
+
+    # Veto the run if the final DB state diverged. The effective result is the AND of
+    # this check and the simulator's conversation judgment, so a mismatch fails a run
+    # the simulator passed; a match simply leaves the simulator's verdict to stand.
+    if diffs:
+        ctx.fail(reason="final DB diverges from expected: " + " | ".join(diffs[:8]))
+
+
 async def on_session_end(ctx: JobContext) -> None:
     try:
         report = ctx.make_session_report()
@@ -624,7 +650,7 @@ async def on_session_end(ctx: JobContext) -> None:
         logger.exception("error closing hotel DB")
 
 
-@server.rtc_session(on_session_end=on_session_end)
+@server.rtc_session(on_session_end=on_session_end, on_simulation_end=on_simulation_end)
 async def hotel_receptionist_agent(ctx: JobContext) -> None:
     await ctx.connect()
 

examples/hotel_receptionist/benchmark.py

@@ -0,0 +1,114 @@
+"""Grade a simulation on final DB state (tau-bench style).
+
+A scenario's `userdata.expected_state` is SQL run against a fresh copy of the
+seed to build the expected end state; we then diff it against the agent's DB.
+The agent is graded on the resulting *state*, not on reproducing the SQL.
+
+The diff is a *denylist*: every column of every transactional table is compared
+except an explicit, justified set that genuinely can't match across two correct
+runs. Foreign-key surrogates are resolved to their stable attribute (room_id ->
+type, table_id -> location) so "which king" doesn't matter but the type does.
+Comparison is an order-invariant multiset, so collateral damage (an extra,
+missing, or altered row anywhere) still surfaces.
+"""
+
+from __future__ import annotations
+
+import collections
+from typing import Any
+
+import apsw
+from hotel_db import HotelDB
+
+# Transactional tables the agent's tools mutate. Static reference data
+# (hotel_rooms, restaurant_tables), the UI table (lk_descriptions), and
+# hotel_invoices (fully derived from the booking) are not compared directly.
+TRANSACTIONAL_TABLES: tuple[str, ...] = (
+    "hotel_bookings",
+    "restaurant_reservations",
+    "hotel_followups",
+    "hotel_disputes",
+)
+
+# The only columns excluded from comparison, by reason:
+DENY_COLUMNS = frozenset(
+    {
+        # surrogate / randomly-minted ids — vary per run and with action order
+        "id",
+        "code",
+        "case_number",
+        "booking_code",
+        # booking math: tool-computed by compute_invoice from type + dates +
+        # extras, which are all already compared — so asserting it would grade the
+        # mock's arithmetic, not the agent. (It also varies with which room of a
+        # type book_room picks, since seed rates differ within a type.)
+        "total",
+        "subtotal",
+        "taxes",
+        "line_items",
+        # free text written by the agent / simulated user
+        "summary",
+        "caller_note",
+        "notes",
+        "late_arrival_note",
+    }
+)
+
+# Resolve FK surrogate -> stable attribute (correlated subquery, single table).
+FK_RESOLVE: dict[tuple[str, str], str] = {
+    ("hotel_bookings", "room_id"): "(SELECT type FROM hotel_rooms WHERE id = room_id) AS room_type",
+    (
+        "restaurant_reservations",
+        "table_id",
+    ): "(SELECT location FROM restaurant_tables WHERE id = table_id) AS table_location",
+}
+
+
+def _select_sql(conn: apsw.Connection, table: str) -> str:
+    cols = [row[1] for row in conn.execute(f"PRAGMA table_info('{table}')")]
+    parts = [FK_RESOLVE.get((table, c), f'"{c}"') for c in cols if c not in DENY_COLUMNS]
+    return f'SELECT {", ".join(parts)} FROM "{table}"'  # noqa: S608
+
+
+def _rows(
+    conn: apsw.Connection, sql: str
+) -> tuple[list[str], collections.Counter[tuple[Any, ...]]]:
+    cur = conn.execute(sql)
+    cols: list[str] = []
+    counter: collections.Counter[tuple[Any, ...]] = collections.Counter()
+    for row in cur:
+        if not cols:  # getdescription() is only valid while a row is in flight
+            cols = [d[0] for d in cur.getdescription()]
+        counter[tuple(row)] += 1
+    return cols, counter
+
+
+def diff_databases(
+    expected: apsw.Connection,
+    actual: apsw.Connection,
+    *,
+    tables: tuple[str, ...] = TRANSACTIONAL_TABLES,
+) -> list[str]:
+    """Order-invariant denylist diff of two hotel DBs. Empty list == states match."""
+    diffs: list[str] = []
+    for table in tables:
+        sql = _select_sql(expected, table)
+        ecols, exp = _rows(expected, sql)
+        acols, act = _rows(actual, sql)
+        cols = ecols or acols
+        for row, n in (exp - act).items():
+            diffs.append(f"{table}: missing {n}x {dict(zip(cols, row, strict=True))}")
+        for row, n in (act - exp).items():
+            diffs.append(f"{table}: unexpected {n}x {dict(zip(cols, row, strict=True))}")
+    return diffs
+
+
+async def build_expected(seed_bytes: bytes, expected_state: list[str]) -> HotelDB:
+    """Construct the expected end state by applying `expected_state` SQL to a fresh
+    seed. The agent's DB is compared against this by *state* (see diff_databases) —
+    the agent does NOT have to reproduce these statements. The seed is pinned to a
+    fixed date for simulations (HOTEL_TODAY), so dates are plain literals."""
+    db = HotelDB.from_bytes(seed_bytes)
+    for stmt in expected_state:
+        db.connection.execute(stmt)
+    return db

examples/hotel_receptionist/fake_data/seed.py

@@ -26,21 +26,21 @@
 _LATE = PRICING.late_checkout
 
 # fmt: off
-# (room_number, type, nightly_rate_cents, max_occupancy, smoking, pets, view)
+# (id (room number, floor+number), type, nightly_rate_cents, max_occupancy, smoking, pets, view)
 ROOMS = [
-    ("201", "king", 24000, 2, 0, 0, "city"),
-    ("202", "king", 26000, 2, 0, 1, "ocean"),
-    ("203", "king", 24000, 2, 1, 0, "city"),
-    ("204", "queen_2beds", 22000, 4, 0, 0, "city"),
-    ("205", "queen_2beds", 22000, 4, 0, 1, "garden"),
-    ("206", "double_queen", 26000, 4, 0, 0, "ocean"),
-    ("301", "king", 28000, 2, 0, 0, "ocean"),
-    ("302", "king", 28000, 2, 0, 0, "ocean"),
-    ("303", "queen_2beds", 24000, 4, 0, 0, "city"),
-    ("304", "double_queen", 28000, 4, 0, 1, "ocean"),
-    ("401", "suite", 48000, 4, 0, 1, "ocean"),
-    ("402", "suite", 52000, 4, 0, 0, "ocean"),
-    ("PH", "penthouse", 120000, 6, 0, 1, "ocean"),
+    ("RM_201", "king", 24000, 2, 0, 0, "city"),
+    ("RM_202", "king", 26000, 2, 0, 1, "ocean"),
+    ("RM_203", "king", 24000, 2, 1, 0, "city"),
+    ("RM_204", "queen_2beds", 22000, 4, 0, 0, "city"),
+    ("RM_205", "queen_2beds", 22000, 4, 0, 1, "garden"),
+    ("RM_206", "double_queen", 26000, 4, 0, 0, "ocean"),
+    ("RM_301", "king", 28000, 2, 0, 0, "ocean"),
+    ("RM_302", "king", 28000, 2, 0, 0, "ocean"),
+    ("RM_303", "queen_2beds", 24000, 4, 0, 0, "city"),
+    ("RM_304", "double_queen", 28000, 4, 0, 1, "ocean"),
+    ("RM_401", "suite", 48000, 4, 0, 1, "ocean"),
+    ("RM_402", "suite", 52000, 4, 0, 0, "ocean"),
+    ("RM_PH", "penthouse", 120000, 6, 0, 1, "ocean"),
 ]
 
 # (label, capacity, location, description)
@@ -123,7 +123,7 @@ def populate(db: HotelDB, today: date) -> None:
     reservation dates are stored as offsets from `today`."""
     conn = db.connection
     conn.executemany(
-        "INSERT INTO hotel_rooms (room_number, type, nightly_rate, max_occupancy, smoking, pets_allowed, room_view) VALUES (?,?,?,?,?,?,?)",
+        "INSERT INTO hotel_rooms (id, type, nightly_rate, max_occupancy, smoking, pets_allowed, room_view) VALUES (?,?,?,?,?,?,?)",
         ROOMS,
     )
     conn.executemany(
@@ -146,7 +146,7 @@ def populate(db: HotelDB, today: date) -> None:
         status,
     ) in BOOKINGS:
         room_row = conn.execute(
-            "SELECT id, nightly_rate FROM hotel_rooms WHERE room_number = ?", (room_no,)
+            "SELECT id, nightly_rate FROM hotel_rooms WHERE id = ?", (f"RM_{room_no}",)
         ).fetchone()
         assert room_row is not None, f"seed fixture references unknown room {room_no}"
         room_id, nightly = room_row

examples/hotel_receptionist/hotel_db.py

@@ -175,7 +175,7 @@ class RoomTypeAvailability:
 class RoomBooking:
     id: int
     code: str
-    room_id: int
+    room_id: str
     room_type: RoomType
     smoking: bool
     nightly_rate: int
@@ -750,8 +750,7 @@ def _install_schema(conn: apsw.Connection) -> None:
 PRAGMA foreign_keys = ON;
 
 CREATE TABLE IF NOT EXISTS hotel_rooms (
-    id            INTEGER PRIMARY KEY,
-    room_number   TEXT    NOT NULL UNIQUE,
+    id            TEXT    PRIMARY KEY,  -- human room number, e.g. 'RM_201' (floor 2, room 01)
     type          TEXT    NOT NULL CHECK (type IN ('king','queen_2beds','double_queen','suite','penthouse')),
     nightly_rate  INTEGER NOT NULL,
     max_occupancy INTEGER NOT NULL,
@@ -763,7 +762,7 @@ def _install_schema(conn: apsw.Connection) -> None:
 CREATE TABLE IF NOT EXISTS hotel_bookings (
     id                 INTEGER PRIMARY KEY,
     code               TEXT    NOT NULL UNIQUE,
-    room_id            INTEGER NOT NULL REFERENCES hotel_rooms(id),
+    room_id            TEXT    NOT NULL REFERENCES hotel_rooms(id),
     first_name         TEXT    NOT NULL,
     last_name          TEXT    NOT NULL,
     email              TEXT    NOT NULL,
@@ -845,14 +844,14 @@ def _install_schema(conn: apsw.Connection) -> None:
 VIEWS = f"""
 DROP VIEW IF EXISTS hotel_room_status;
 CREATE VIEW hotel_room_status AS
-SELECT r.room_number, r.type, r.room_view, r.max_occupancy, r.nightly_rate,
+SELECT r.id AS room_number, r.type, r.room_view, r.max_occupancy, r.nightly_rate,
        CASE WHEN b.code IS NULL THEN 'available' ELSE 'occupied' END AS status,
        b.code AS current_booking, b.first_name, b.last_name, b.check_out AS free_after
 FROM hotel_rooms r
 LEFT JOIN hotel_bookings b
     ON b.room_id = r.id AND b.status = 'confirmed'
    AND b.check_in <= '{TODAY.isoformat()}' AND b.check_out > '{TODAY.isoformat()}'
-ORDER BY r.room_number;
+ORDER BY r.id;
 
 DROP VIEW IF EXISTS restaurant_table_status;
 CREATE VIEW restaurant_table_status AS

examples/hotel_receptionist/scenarios.yaml

@@ -0,0 +1,25 @@
+name: Room booking
+scenarios:
+  - label: Book a king room for one night
+    instructions: >
+      You are Jordan Reyes (email jordan.reyes@example.com, phone 5550142).
+      Book a king room for the night of 2026-06-09, checking out the 10th, just
+      you. No breakfast, no late checkout. Pay with the card ending 4242.
+    agent_expectations: 'Room booked successfully'
+    tags:
+      feature: room_booking
+    # The expected end state, built by applying this SQL to a fresh seed and then
+    # diffed against the agent's DB. The agent is graded on the resulting *state*,
+    # not on running these statements. NOT-NULL columns the diff ignores get the
+    # 'IGNORED' sentinel (SQLite's flexible typing lets it sit in INTEGER cols
+    # too); extras/status are omitted since the schema defaults them. room_id is a
+    # king (RM_201/202/203/301/302 in the fixed seed) so the diff's room_id->type
+    # sees "king". Dates are literals — sims pin the seed date via HOTEL_TODAY (2026-06-08).
+    userdata:
+      expected_state:
+        - >
+          INSERT INTO hotel_bookings
+            (code, room_id, first_name, last_name, email, phone,
+             check_in, check_out, guests, total, card_last4)
+          VALUES ('IGNORED', 'RM_201', 'Jordan', 'Reyes', 'jordan.reyes@example.com', '5550142',
+                  '2026-06-09', '2026-06-10', 1, 'IGNORED', '4242')

livekit-agents/livekit/agents/__init__.py

@@ -58,6 +58,15 @@
     function_tool,
 )
 from .plugin import Plugin
+from .simulation import (
+    Scenario,
+    ScenarioGroup,
+    ScenarioUserdata,
+    SimulationContext,
+    SimulationDispatch,
+    SimulationRun,
+    SimulationVerdict,
+)
 from .types import (
     DEFAULT_API_CONNECT_OPTIONS,
     NOT_GIVEN,
@@ -184,6 +193,13 @@ def __getattr__(name: str) -> typing.Any:
     "ToolError",
     "RunContext",
     "Plugin",
+    "Scenario",
+    "ScenarioGroup",
+    "ScenarioUserdata",
+    "SimulationContext",
+    "SimulationDispatch",
+    "SimulationRun",
+    "SimulationVerdict",
     "AgentSession",
     "RecordingOptions",
     "text_transforms",