diff --git a/livekit-protocol/livekit/protocol/agent_simulation.py b/livekit-protocol/livekit/protocol/agent_simulation.py index 4faabda4..76d8112b 100644 --- a/livekit-protocol/livekit/protocol/agent_simulation.py +++ b/livekit-protocol/livekit/protocol/agent_simulation.py @@ -29,7 +29,7 @@ from . import models as _models_ -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1elivekit_agent_simulation.proto\x12\x07livekit\x1a!agent/livekit_agent_session.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x19livekit_cloud_agent.proto\x1a\x14livekit_models.proto\"\xea\x02\n\x14SimulationRunSummary\x12\x0e\n\x06passed\x18\x01 \x01(\x05\x12\x0e\n\x06\x66\x61iled\x18\x02 \x01(\x05\x12\x12\n\ngoing_well\x18\x03 \x01(\t\x12\x12\n\nto_improve\x18\x04 \x01(\t\x12\x33\n\x06issues\x18\x05 \x03(\x0b\x32#.livekit.SimulationRunSummary.Issue\x12\x44\n\x0c\x63hat_history\x18\x06 \x03(\x0b\x32..livekit.SimulationRunSummary.ChatHistoryEntry\x1aN\n\x10\x43hatHistoryEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12)\n\x05value\x18\x02 \x01(\x0b\x32\x1a.livekit.agent.ChatContext:\x02\x38\x01\x1a?\n\x05Issue\x12\x13\n\x0b\x64\x65scription\x18\x01 \x01(\t\x12\x12\n\nsuggestion\x18\x02 \x01(\t\x12\r\n\x05label\x18\x03 \x01(\t\"\x99\x38\n\rSimulationRun\x12\n\n\x02id\x18\x01 \x01(\t\x12\x12\n\nproject_id\x18\x02 \x01(\t\x12-\n\x06status\x18\x03 \x01(\x0e\x32\x1d.livekit.SimulationRun.Status\x12\x19\n\x11\x61gent_description\x18\x04 \x01(\t\x12\r\n\x05\x65rror\x18\x05 \x01(\t\x12.\n\ncreated_at\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12(\n\x04jobs\x18\x07 \x03(\x0b\x32\x1a.livekit.SimulationRun.Job\x12\x12\n\nagent_name\x18\t \x01(\t\x12.\n\x0escenario_group\x18\n \x01(\x0b\x32\x16.livekit.ScenarioGroup\x12,\n\x08\x65nded_at\x18\x0b \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x11\n\tjob_count\x18\x0c \x01(\x05\x12\x14\n\x0cpassed_count\x18\r \x01(\x05\x12\x14\n\x0c\x66\x61iled_count\x18\x0e \x01(\x05\x12\x17\n\x0fnum_simulations\x18\x0f \x01(\x05\x12+\n\x05usage\x18\x10 \x01(\x0b\x32\x1c.livekit.SimulationRun.Usage\x12\x13\n\x0b\x63oncurrency\x18\x11 \x01(\x05\x12%\n\x04mode\x18\x12 \x01(\x0e\x32\x17.livekit.SimulationMode\x12\x32\n\x07metrics\x18\x13 \x01(\x0b\x32!.livekit.SimulationRun.RunMetrics\x12\x14\n\x0csummary_zstd\x18\x14 \x01(\x0c\x1a\xbe\x04\n\x03Job\x12\n\n\x02id\x18\x01 \x01(\t\x12\x31\n\x06status\x18\x02 \x01(\x0e\x32!.livekit.SimulationRun.Job.Status\x12\x14\n\x0cinstructions\x18\x03 \x01(\t\x12\r\n\x05\x65rror\x18\x04 \x01(\t\x12\x1a\n\x12\x61gent_expectations\x18\x05 \x01(\t\x12\r\n\x05label\x18\x06 \x01(\t\x12\x0c\n\x04tags\x18\x07 \x03(\t\x12\x11\n\troom_name\x18\x08 \x01(\t\x12.\n\nstarted_at\x18\n \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12,\n\x08\x65nded_at\x18\x0b \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0f\n\x07room_id\x18\x0c \x01(\t\x12/\n\x05usage\x18\r \x01(\x0b\x32 .livekit.SimulationRun.Job.Usage\x12\x32\n\x07metrics\x18\x0e \x01(\x0b\x32!.livekit.SimulationRun.JobMetrics\x1a<\n\x05Usage\x12\x18\n\x10text_turns_count\x18\x01 \x01(\x05\x12\x19\n\x11\x61udio_turns_count\x18\x02 \x01(\x05\"o\n\x06Status\x12\x12\n\x0eSTATUS_PENDING\x10\x00\x12\x12\n\x0eSTATUS_RUNNING\x10\x01\x12\x14\n\x10STATUS_COMPLETED\x10\x02\x12\x11\n\rSTATUS_FAILED\x10\x03\x12\x14\n\x10STATUS_CANCELLED\x10\x04J\x04\x08\t\x10\n\x1a\x87\x1d\n\nJobMetrics\x12\x1b\n\x0e\x61\x63\x63uracy_score\x18\x01 \x01(\x02H\x00\x88\x01\x01\x12\x1d\n\x10\x65xperience_score\x18\x02 \x01(\x02H\x01\x88\x01\x01\x12\x1c\n\x0ftask_completion\x18\x03 \x01(\x02H\x02\x88\x01\x01\x12\x32\n\x03stt\x18\x04 \x01(\x0b\x32%.livekit.SimulationRun.JobMetrics.STT\x12\x32\n\x03llm\x18\x05 \x01(\x0b\x32%.livekit.SimulationRun.JobMetrics.LLM\x12\x32\n\x03tts\x18\x06 \x01(\x0b\x32%.livekit.SimulationRun.JobMetrics.TTS\x12\x44\n\x0c\x63onversation\x18\x07 \x01(\x0b\x32..livekit.SimulationRun.JobMetrics.Conversation\x12\x35\n\x05turns\x18\t \x03(\x0b\x32&.livekit.SimulationRun.JobMetrics.Turn\x12\x13\n\x0bjudge_model\x18\n \x01(\t\x12\x1a\n\x12has_remote_session\x18\x0c \x01(\x08\x12&\n\x02t0\x18\r \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x18\n\x0b\x63onciseness\x18\x0e \x01(\x02H\x03\x88\x01\x01\x12#\n\x16unnecessary_tool_calls\x18\x0f \x01(\x08H\x04\x88\x01\x01\x12\x1d\n\x10information_loss\x18\x10 \x01(\x08H\x05\x88\x01\x01\x12!\n\x14redundant_statements\x18\x11 \x01(\x08H\x06\x88\x01\x01\x12\"\n\x15poor_question_quality\x18\x12 \x01(\x08H\x07\x88\x01\x01\x12%\n\x18\x63onversation_progression\x18\x13 \x01(\x02H\x08\x88\x01\x01\x1a\xdb\x04\n\x03STT\x12\x10\n\x03wer\x18\x01 \x01(\x02H\x00\x88\x01\x01\x12\x12\n\x05words\x18\x02 \x01(\rH\x01\x88\x01\x01\x12\x18\n\x0bword_errors\x18\x03 \x01(\rH\x02\x88\x01\x01\x12\x10\n\x03\x63\x65r\x18\x04 \x01(\x02H\x03\x88\x01\x01\x12\x12\n\x05\x63hars\x18\t \x01(\rH\x04\x88\x01\x01\x12\x18\n\x0b\x63har_errors\x18\n \x01(\rH\x05\x88\x01\x01\x12\x19\n\x0cstt_delay_ms\x18\x08 \x01(\rH\x06\x88\x01\x01\x12\x1f\n\x12\x65ntity_recognition\x18\x0b \x01(\x02H\x07\x88\x01\x01\x12\x1d\n\x10\x65ntities_uttered\x18\x0c \x01(\rH\x08\x88\x01\x01\x12 \n\x13\x65ntities_recognized\x18\r \x01(\rH\t\x88\x01\x01\x12\x1a\n\rentity_recall\x18\x0e \x01(\x02H\n\x88\x01\x01\x12&\n\x19\x65ntities_acquired_uttered\x18\x0f \x01(\rH\x0b\x88\x01\x01\x12)\n\x1c\x65ntities_acquired_recognized\x18\x10 \x01(\rH\x0c\x88\x01\x01\x42\x06\n\x04_werB\x08\n\x06_wordsB\x0e\n\x0c_word_errorsB\x06\n\x04_cerB\x08\n\x06_charsB\x0e\n\x0c_char_errorsB\x0f\n\r_stt_delay_msB\x15\n\x13_entity_recognitionB\x13\n\x11_entities_utteredB\x16\n\x14_entities_recognizedB\x10\n\x0e_entity_recallB\x1c\n\x1a_entities_acquired_utteredB\x1f\n\x1d_entities_acquired_recognized\x1a\x7f\n\x03LLM\x12\x14\n\x07ttft_ms\x18\x01 \x01(\rH\x00\x88\x01\x01\x12\x14\n\x07ttfs_ms\x18\x02 \x01(\rH\x01\x88\x01\x01\x12\x1e\n\x11tokens_per_second\x18\x03 \x01(\x02H\x02\x88\x01\x01\x42\n\n\x08_ttft_msB\n\n\x08_ttfs_msB\x14\n\x12_tokens_per_second\x1a\xd1\x04\n\x03TTS\x12\x14\n\x07ttfb_ms\x18\x02 \x01(\rH\x00\x88\x01\x01\x12\x10\n\x03wer\x18\x03 \x01(\x02H\x01\x88\x01\x01\x12\x12\n\x05words\x18\x07 \x01(\rH\x02\x88\x01\x01\x12\x18\n\x0bword_errors\x18\x08 \x01(\rH\x03\x88\x01\x01\x12\x10\n\x03\x63\x65r\x18\t \x01(\x02H\x04\x88\x01\x01\x12\x12\n\x05\x63hars\x18\n \x01(\rH\x05\x88\x01\x01\x12\x18\n\x0b\x63har_errors\x18\x0b \x01(\rH\x06\x88\x01\x01\x12\x1f\n\x12\x65ntity_recognition\x18\x0e \x01(\x02H\x07\x88\x01\x01\x12\x1d\n\x10\x65ntities_uttered\x18\x0f \x01(\rH\x08\x88\x01\x01\x12 \n\x13\x65ntities_recognized\x18\x10 \x01(\rH\t\x88\x01\x01\x12\x1a\n\rentity_recall\x18\x11 \x01(\x02H\n\x88\x01\x01\x12&\n\x19\x65ntities_acquired_uttered\x18\x12 \x01(\rH\x0b\x88\x01\x01\x12)\n\x1c\x65ntities_acquired_recognized\x18\x13 \x01(\rH\x0c\x88\x01\x01\x42\n\n\x08_ttfb_msB\x06\n\x04_werB\x08\n\x06_wordsB\x0e\n\x0c_word_errorsB\x06\n\x04_cerB\x08\n\x06_charsB\x0e\n\x0c_char_errorsB\x15\n\x13_entity_recognitionB\x13\n\x11_entities_utteredB\x16\n\x14_entities_recognizedB\x10\n\x0e_entity_recallB\x1c\n\x1a_entities_acquired_utteredB\x1f\n\x1d_entities_acquired_recognized\x1a\x82\x06\n\x0c\x43onversation\x12\x1e\n\x11turn_taking_score\x18\x01 \x01(\x02H\x00\x88\x01\x01\x12%\n\x18heard_e2e_latency_p50_ms\x18\x02 \x01(\x05H\x01\x88\x01\x01\x12%\n\x18heard_e2e_latency_p95_ms\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12%\n\x18heard_e2e_latency_p99_ms\x18\x04 \x01(\x05H\x03\x88\x01\x01\x12\x1d\n\x10time_to_yield_ms\x18\x05 \x01(\rH\x04\x88\x01\x01\x12$\n\x17\x65ot_misprediction_count\x18\x06 \x01(\rH\x05\x88\x01\x01\x12\x1a\n\roverlap_ratio\x18\x07 \x01(\x02H\x06\x88\x01\x01\x12\x1e\n\x11overlap_speech_ms\x18\x08 \x01(\rH\x07\x88\x01\x01\x12\x1c\n\x0ftotal_speech_ms\x18\t \x01(\rH\x08\x88\x01\x01\x12\"\n\x15\x61wkward_silence_count\x18\x0b \x01(\rH\t\x88\x01\x01\x12\x1d\n\x10unanswered_turns\x18\x0c \x01(\rH\n\x88\x01\x01\x12%\n\x18\x66\x61lse_interruption_count\x18\r \x01(\rH\x0b\x88\x01\x01\x12\x1b\n\x0e\x65\x32\x65_latency_ms\x18\x0f \x01(\rH\x0c\x88\x01\x01\x42\x14\n\x12_turn_taking_scoreB\x1b\n\x19_heard_e2e_latency_p50_msB\x1b\n\x19_heard_e2e_latency_p95_msB\x1b\n\x19_heard_e2e_latency_p99_msB\x13\n\x11_time_to_yield_msB\x1a\n\x18_eot_misprediction_countB\x10\n\x0e_overlap_ratioB\x14\n\x12_overlap_speech_msB\x12\n\x10_total_speech_msB\x18\n\x16_awkward_silence_countB\x13\n\x11_unanswered_turnsB\x1b\n\x19_false_interruption_countB\x11\n\x0f_e2e_latency_ms\x1a\xdc\x05\n\x04Turn\x12\r\n\x05index\x18\x01 \x01(\r\x12%\n\x04role\x18\x02 \x01(\x0e\x32\x17.livekit.agent.ChatRole\x12\x15\n\x08start_ms\x18\x03 \x01(\rH\x00\x88\x01\x01\x12\x13\n\x06\x65nd_ms\x18\x04 \x01(\rH\x01\x88\x01\x01\x12\x19\n\x0cstt_delay_ms\x18\x05 \x01(\rH\x02\x88\x01\x01\x12\x18\n\x0bllm_ttft_ms\x18\x06 \x01(\rH\x03\x88\x01\x01\x12\x18\n\x0bllm_ttfs_ms\x18\x07 \x01(\rH\x04\x88\x01\x01\x12\x14\n\x07llm_tps\x18\x08 \x01(\x02H\x05\x88\x01\x01\x12\x18\n\x0btts_ttfb_ms\x18\t \x01(\rH\x06\x88\x01\x01\x12\x1b\n\x0e\x65\x32\x65_latency_ms\x18\n \x01(\rH\x07\x88\x01\x01\x12!\n\x14heard_e2e_latency_ms\x18\x0b \x01(\x05H\x08\x88\x01\x01\x12\x1d\n\x10time_to_yield_ms\x18\x0c \x01(\rH\t\x88\x01\x01\x12\x1e\n\x11turn_taking_score\x18\r \x01(\x02H\n\x88\x01\x01\x12\x18\n\x0b\x63onciseness\x18\x0e \x01(\x02H\x0b\x88\x01\x01\x12\x1e\n\x11\x65ot_misprediction\x18\x0f \x01(\x08H\x0c\x88\x01\x01\x12\x1c\n\x0f\x61wkward_silence\x18\x10 \x01(\x08H\r\x88\x01\x01\x12\x17\n\nunanswered\x18\x11 \x01(\x08H\x0e\x88\x01\x01\x42\x0b\n\t_start_msB\t\n\x07_end_msB\x0f\n\r_stt_delay_msB\x0e\n\x0c_llm_ttft_msB\x0e\n\x0c_llm_ttfs_msB\n\n\x08_llm_tpsB\x0e\n\x0c_tts_ttfb_msB\x11\n\x0f_e2e_latency_msB\x17\n\x15_heard_e2e_latency_msB\x13\n\x11_time_to_yield_msB\x14\n\x12_turn_taking_scoreB\x0e\n\x0c_concisenessB\x14\n\x12_eot_mispredictionB\x12\n\x10_awkward_silenceB\r\n\x0b_unansweredB\x11\n\x0f_accuracy_scoreB\x13\n\x11_experience_scoreB\x12\n\x10_task_completionB\x0e\n\x0c_concisenessB\x19\n\x17_unnecessary_tool_callsB\x13\n\x11_information_lossB\x17\n\x15_redundant_statementsB\x18\n\x16_poor_question_qualityB\x1b\n\x19_conversation_progression\x1a\x8a\x03\n\nRunMetrics\x12\x1b\n\x0e\x61\x63\x63uracy_score\x18\x01 \x01(\x02H\x00\x88\x01\x01\x12\x1d\n\x10\x65xperience_score\x18\x02 \x01(\x02H\x01\x88\x01\x01\x12\x1f\n\x12scenario_pass_rate\x18\x03 \x01(\x02H\x02\x88\x01\x01\x12\x32\n\x03stt\x18\x04 \x01(\x0b\x32%.livekit.SimulationRun.JobMetrics.STT\x12\x32\n\x03llm\x18\x05 \x01(\x0b\x32%.livekit.SimulationRun.JobMetrics.LLM\x12\x32\n\x03tts\x18\x06 \x01(\x0b\x32%.livekit.SimulationRun.JobMetrics.TTS\x12\x44\n\x0c\x63onversation\x18\x07 \x01(\x0b\x32..livekit.SimulationRun.JobMetrics.ConversationB\x11\n\x0f_accuracy_scoreB\x13\n\x11_experience_scoreB\x15\n\x13_scenario_pass_rate\x1a\xcc\x03\n\x06\x43reate\x1a\xdb\x02\n\x07Request\x12\x12\n\nproject_id\x18\x01 \x01(\t\x12\x12\n\nagent_name\x18\x02 \x01(\t\x12\x17\n\x0fnum_simulations\x18\x04 \x01(\x05\x12\x0e\n\x06region\x18\x06 \x01(\t\x12\x33\n\x0escenario_group\x18\x07 \x01(\x0b\x32\x16.livekit.ScenarioGroupH\x00\x88\x01\x01\x12\x18\n\x0b\x63oncurrency\x18\x08 \x01(\x05H\x01\x88\x01\x01\x12%\n\x04mode\x18\t \x01(\x0e\x32\x17.livekit.SimulationMode\x12\x18\n\x10\x62\x61\x63kground_noise\x18\n \x01(\x08\x12\x1e\n\x16low_quality_microphone\x18\x0b \x01(\x08\x12\x13\n\x0bpacket_loss\x18\x0c \x01(\x08\x42\x11\n\x0f_scenario_groupB\x0e\n\x0c_concurrencyJ\x04\x08\x03\x10\x04R\x11\x61gent_description\x1a\x64\n\x08Response\x12\x19\n\x11simulation_run_id\x18\x01 \x01(\t\x12=\n\x16presigned_post_request\x18\x02 \x01(\x0b\x32\x1d.livekit.PresignedPostRequest\x1at\n\x13\x43onfirmSourceUpload\x1aQ\n\x07Request\x12\x12\n\nproject_id\x18\x01 \x01(\t\x12\x19\n\x11simulation_run_id\x18\x02 \x01(\t\x12\x17\n\x0f\x63ode_entrypoint\x18\x03 \x01(\t\x1a\n\n\x08Response\x1ap\n\x03Get\x1a\x38\n\x07Request\x12\x12\n\nproject_id\x18\x01 \x01(\t\x12\x19\n\x11simulation_run_id\x18\x02 \x01(\t\x1a/\n\x08Response\x12#\n\x03run\x18\x01 \x01(\x0b\x32\x16.livekit.SimulationRun\x1a\xb8\x03\n\x04List\x1a\xca\x02\n\x07Request\x12\x12\n\nproject_id\x18\x01 \x01(\t\x12\x32\n\x06status\x18\x02 \x01(\x0e\x32\x1d.livekit.SimulationRun.StatusH\x00\x88\x01\x01\x12\x31\n\npage_token\x18\x03 \x01(\x0b\x32\x18.livekit.TokenPaginationH\x01\x88\x01\x01\x12\x33\n\nstart_time\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x02\x88\x01\x01\x12\x31\n\x08\x65nd_time\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x03\x88\x01\x01\x12\x17\n\nagent_name\x18\x06 \x01(\tH\x04\x88\x01\x01\x42\t\n\x07_statusB\r\n\x0b_page_tokenB\r\n\x0b_start_timeB\x0b\n\t_end_timeB\r\n\x0b_agent_name\x1a\x63\n\x08Response\x12$\n\x04runs\x18\x01 \x03(\x0b\x32\x16.livekit.SimulationRun\x12\x31\n\x0fnext_page_token\x18\x02 \x01(\x0b\x32\x18.livekit.TokenPagination\x1a\xa8\x03\n\x06\x43ounts\x1a\xe2\x01\n\x07Request\x12\x12\n\nproject_id\x18\x01 \x01(\t\x12.\n\nstart_time\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12,\n\x08\x65nd_time\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x32\n\x06status\x18\x04 \x01(\x0e\x32\x1d.livekit.SimulationRun.StatusH\x00\x88\x01\x01\x12\x17\n\nagent_name\x18\x05 \x01(\tH\x01\x88\x01\x01\x42\t\n\x07_statusB\r\n\x0b_agent_name\x1an\n\x08Response\x12\x35\n\x07\x62uckets\x18\x01 \x03(\x0b\x32$.livekit.SimulationRun.Counts.Bucket\x12+\n\x08interval\x18\x02 \x01(\x0b\x32\x19.google.protobuf.Duration\x1aI\n\x06\x42ucket\x12\x30\n\x0c\x62ucket_start\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\r\n\x05\x63ount\x18\x02 \x01(\x04\x1aN\n\x06\x43\x61ncel\x1a\x38\n\x07Request\x12\x12\n\nproject_id\x18\x01 \x01(\t\x12\x19\n\x11simulation_run_id\x18\x02 \x01(\t\x1a\n\n\x08Response\x1a<\n\x05Usage\x12\x18\n\x10text_turns_count\x18\x01 \x01(\x05\x12\x19\n\x11\x61udio_turns_count\x18\x02 \x01(\x05\"\xa5\x01\n\x06Status\x12\x19\n\x15STATUS_PENDING_UPLOAD\x10\x00\x12\x15\n\x11STATUS_GENERATING\x10\x01\x12\x12\n\x0eSTATUS_RUNNING\x10\x02\x12\x16\n\x12STATUS_SUMMARIZING\x10\x03\x12\x14\n\x10STATUS_COMPLETED\x10\x04\x12\x11\n\rSTATUS_FAILED\x10\x05\x12\x14\n\x10STATUS_CANCELLED\x10\x06J\x04\x08\x08\x10\tR\x07summary\"\xbc\x02\n\x08Scenario\x12\r\n\x05label\x18\x01 \x01(\t\x12\x14\n\x0cinstructions\x18\x02 \x01(\t\x12\x1a\n\x12\x61gent_expectations\x18\x03 \x01(\t\x12)\n\x04tags\x18\x04 \x03(\x0b\x32\x1b.livekit.Scenario.TagsEntry\x12\x10\n\x08userdata\x18\x05 \x01(\t\x1a\x84\x01\n\x11\x43reateFromSession\x1a>\n\x07Request\x12\x12\n\nproject_id\x18\x01 \x01(\t\x12\x0f\n\x07room_id\x18\x02 \x01(\t\x12\x0e\n\x06region\x18\x03 \x01(\t\x1a/\n\x08Response\x12#\n\x08scenario\x18\x01 \x01(\x0b\x32\x11.livekit.Scenario\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"C\n\rScenarioGroup\x12\x0c\n\x04name\x18\x01 \x01(\t\x12$\n\tscenarios\x18\x02 \x03(\x0b\x32\x11.livekit.Scenario\"\x8b\x01\n\x12SimulationDispatch\x12\x19\n\x11simulation_run_id\x18\x01 \x01(\t\x12\x0e\n\x06job_id\x18\x02 \x01(\t\x12#\n\x08scenario\x18\x03 \x01(\x0b\x32\x11.livekit.Scenario\x12%\n\x04mode\x18\x04 \x01(\x0e\x32\x17.livekit.SimulationMode*f\n\x0eSimulationMode\x12\x1f\n\x1bSIMULATION_MODE_UNSPECIFIED\x10\x00\x12\x18\n\x14SIMULATION_MODE_TEXT\x10\x01\x12\x19\n\x15SIMULATION_MODE_AUDIO\x10\x02\x32\x84\x06\n\x0f\x41gentSimulation\x12\x64\n\x13\x43reateSimulationRun\x12%.livekit.SimulationRun.Create.Request\x1a&.livekit.SimulationRun.Create.Response\x12\x88\x01\n\x1d\x43onfirmSimulationSourceUpload\x12\x32.livekit.SimulationRun.ConfirmSourceUpload.Request\x1a\x33.livekit.SimulationRun.ConfirmSourceUpload.Response\x12[\n\x10GetSimulationRun\x12\".livekit.SimulationRun.Get.Request\x1a#.livekit.SimulationRun.Get.Response\x12_\n\x12ListSimulationRuns\x12#.livekit.SimulationRun.List.Request\x1a$.livekit.SimulationRun.List.Response\x12\x64\n\x13\x43ountSimulationRuns\x12%.livekit.SimulationRun.Counts.Request\x1a&.livekit.SimulationRun.Counts.Response\x12\x64\n\x13\x43\x61ncelSimulationRun\x12%.livekit.SimulationRun.Cancel.Request\x1a&.livekit.SimulationRun.Cancel.Response\x12v\n\x19\x43reateScenarioFromSession\x12+.livekit.Scenario.CreateFromSession.Request\x1a,.livekit.Scenario.CreateFromSession.ResponseBFZ#github.com/livekit/protocol/livekit\xaa\x02\rLiveKit.Proto\xea\x02\x0eLiveKit::Protob\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1elivekit_agent_simulation.proto\x12\x07livekit\x1a!agent/livekit_agent_session.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x19livekit_cloud_agent.proto\x1a\x14livekit_models.proto\"\xea\x02\n\x14SimulationRunSummary\x12\x0e\n\x06passed\x18\x01 \x01(\x05\x12\x0e\n\x06\x66\x61iled\x18\x02 \x01(\x05\x12\x12\n\ngoing_well\x18\x03 \x01(\t\x12\x12\n\nto_improve\x18\x04 \x01(\t\x12\x33\n\x06issues\x18\x05 \x03(\x0b\x32#.livekit.SimulationRunSummary.Issue\x12\x44\n\x0c\x63hat_history\x18\x06 \x03(\x0b\x32..livekit.SimulationRunSummary.ChatHistoryEntry\x1aN\n\x10\x43hatHistoryEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12)\n\x05value\x18\x02 \x01(\x0b\x32\x1a.livekit.agent.ChatContext:\x02\x38\x01\x1a?\n\x05Issue\x12\x13\n\x0b\x64\x65scription\x18\x01 \x01(\t\x12\x12\n\nsuggestion\x18\x02 \x01(\t\x12\r\n\x05label\x18\x03 \x01(\t\"\x87\x39\n\rSimulationRun\x12\n\n\x02id\x18\x01 \x01(\t\x12\x12\n\nproject_id\x18\x02 \x01(\t\x12-\n\x06status\x18\x03 \x01(\x0e\x32\x1d.livekit.SimulationRun.Status\x12\x19\n\x11\x61gent_description\x18\x04 \x01(\t\x12\r\n\x05\x65rror\x18\x05 \x01(\t\x12.\n\ncreated_at\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12(\n\x04jobs\x18\x07 \x03(\x0b\x32\x1a.livekit.SimulationRun.Job\x12\x12\n\nagent_name\x18\t \x01(\t\x12.\n\x0escenario_group\x18\n \x01(\x0b\x32\x16.livekit.ScenarioGroup\x12,\n\x08\x65nded_at\x18\x0b \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x11\n\tjob_count\x18\x0c \x01(\x05\x12\x14\n\x0cpassed_count\x18\r \x01(\x05\x12\x14\n\x0c\x66\x61iled_count\x18\x0e \x01(\x05\x12\x17\n\x0fnum_simulations\x18\x0f \x01(\x05\x12+\n\x05usage\x18\x10 \x01(\x0b\x32\x1c.livekit.SimulationRun.Usage\x12\x13\n\x0b\x63oncurrency\x18\x11 \x01(\x05\x12%\n\x04mode\x18\x12 \x01(\x0e\x32\x17.livekit.SimulationMode\x12\x32\n\x07metrics\x18\x13 \x01(\x0b\x32!.livekit.SimulationRun.RunMetrics\x12\x14\n\x0csummary_zstd\x18\x14 \x01(\x0c\x1a\xbe\x04\n\x03Job\x12\n\n\x02id\x18\x01 \x01(\t\x12\x31\n\x06status\x18\x02 \x01(\x0e\x32!.livekit.SimulationRun.Job.Status\x12\x14\n\x0cinstructions\x18\x03 \x01(\t\x12\r\n\x05\x65rror\x18\x04 \x01(\t\x12\x1a\n\x12\x61gent_expectations\x18\x05 \x01(\t\x12\r\n\x05label\x18\x06 \x01(\t\x12\x0c\n\x04tags\x18\x07 \x03(\t\x12\x11\n\troom_name\x18\x08 \x01(\t\x12.\n\nstarted_at\x18\n \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12,\n\x08\x65nded_at\x18\x0b \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0f\n\x07room_id\x18\x0c \x01(\t\x12/\n\x05usage\x18\r \x01(\x0b\x32 .livekit.SimulationRun.Job.Usage\x12\x32\n\x07metrics\x18\x0e \x01(\x0b\x32!.livekit.SimulationRun.JobMetrics\x1a<\n\x05Usage\x12\x18\n\x10text_turns_count\x18\x01 \x01(\x05\x12\x19\n\x11\x61udio_turns_count\x18\x02 \x01(\x05\"o\n\x06Status\x12\x12\n\x0eSTATUS_PENDING\x10\x00\x12\x12\n\x0eSTATUS_RUNNING\x10\x01\x12\x14\n\x10STATUS_COMPLETED\x10\x02\x12\x11\n\rSTATUS_FAILED\x10\x03\x12\x14\n\x10STATUS_CANCELLED\x10\x04J\x04\x08\t\x10\n\x1a\x87\x1d\n\nJobMetrics\x12\x1b\n\x0e\x61\x63\x63uracy_score\x18\x01 \x01(\x02H\x00\x88\x01\x01\x12\x1d\n\x10\x65xperience_score\x18\x02 \x01(\x02H\x01\x88\x01\x01\x12\x1c\n\x0ftask_completion\x18\x03 \x01(\x02H\x02\x88\x01\x01\x12\x32\n\x03stt\x18\x04 \x01(\x0b\x32%.livekit.SimulationRun.JobMetrics.STT\x12\x32\n\x03llm\x18\x05 \x01(\x0b\x32%.livekit.SimulationRun.JobMetrics.LLM\x12\x32\n\x03tts\x18\x06 \x01(\x0b\x32%.livekit.SimulationRun.JobMetrics.TTS\x12\x44\n\x0c\x63onversation\x18\x07 \x01(\x0b\x32..livekit.SimulationRun.JobMetrics.Conversation\x12\x35\n\x05turns\x18\t \x03(\x0b\x32&.livekit.SimulationRun.JobMetrics.Turn\x12\x13\n\x0bjudge_model\x18\n \x01(\t\x12\x1a\n\x12has_remote_session\x18\x0c \x01(\x08\x12&\n\x02t0\x18\r \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x18\n\x0b\x63onciseness\x18\x0e \x01(\x02H\x03\x88\x01\x01\x12#\n\x16unnecessary_tool_calls\x18\x0f \x01(\x08H\x04\x88\x01\x01\x12\x1d\n\x10information_loss\x18\x10 \x01(\x08H\x05\x88\x01\x01\x12!\n\x14redundant_statements\x18\x11 \x01(\x08H\x06\x88\x01\x01\x12\"\n\x15poor_question_quality\x18\x12 \x01(\x08H\x07\x88\x01\x01\x12%\n\x18\x63onversation_progression\x18\x13 \x01(\x02H\x08\x88\x01\x01\x1a\xdb\x04\n\x03STT\x12\x10\n\x03wer\x18\x01 \x01(\x02H\x00\x88\x01\x01\x12\x12\n\x05words\x18\x02 \x01(\rH\x01\x88\x01\x01\x12\x18\n\x0bword_errors\x18\x03 \x01(\rH\x02\x88\x01\x01\x12\x10\n\x03\x63\x65r\x18\x04 \x01(\x02H\x03\x88\x01\x01\x12\x12\n\x05\x63hars\x18\t \x01(\rH\x04\x88\x01\x01\x12\x18\n\x0b\x63har_errors\x18\n \x01(\rH\x05\x88\x01\x01\x12\x19\n\x0cstt_delay_ms\x18\x08 \x01(\rH\x06\x88\x01\x01\x12\x1f\n\x12\x65ntity_recognition\x18\x0b \x01(\x02H\x07\x88\x01\x01\x12\x1d\n\x10\x65ntities_uttered\x18\x0c \x01(\rH\x08\x88\x01\x01\x12 \n\x13\x65ntities_recognized\x18\r \x01(\rH\t\x88\x01\x01\x12\x1a\n\rentity_recall\x18\x0e \x01(\x02H\n\x88\x01\x01\x12&\n\x19\x65ntities_acquired_uttered\x18\x0f \x01(\rH\x0b\x88\x01\x01\x12)\n\x1c\x65ntities_acquired_recognized\x18\x10 \x01(\rH\x0c\x88\x01\x01\x42\x06\n\x04_werB\x08\n\x06_wordsB\x0e\n\x0c_word_errorsB\x06\n\x04_cerB\x08\n\x06_charsB\x0e\n\x0c_char_errorsB\x0f\n\r_stt_delay_msB\x15\n\x13_entity_recognitionB\x13\n\x11_entities_utteredB\x16\n\x14_entities_recognizedB\x10\n\x0e_entity_recallB\x1c\n\x1a_entities_acquired_utteredB\x1f\n\x1d_entities_acquired_recognized\x1a\x7f\n\x03LLM\x12\x14\n\x07ttft_ms\x18\x01 \x01(\rH\x00\x88\x01\x01\x12\x14\n\x07ttfs_ms\x18\x02 \x01(\rH\x01\x88\x01\x01\x12\x1e\n\x11tokens_per_second\x18\x03 \x01(\x02H\x02\x88\x01\x01\x42\n\n\x08_ttft_msB\n\n\x08_ttfs_msB\x14\n\x12_tokens_per_second\x1a\xd1\x04\n\x03TTS\x12\x14\n\x07ttfb_ms\x18\x02 \x01(\rH\x00\x88\x01\x01\x12\x10\n\x03wer\x18\x03 \x01(\x02H\x01\x88\x01\x01\x12\x12\n\x05words\x18\x07 \x01(\rH\x02\x88\x01\x01\x12\x18\n\x0bword_errors\x18\x08 \x01(\rH\x03\x88\x01\x01\x12\x10\n\x03\x63\x65r\x18\t \x01(\x02H\x04\x88\x01\x01\x12\x12\n\x05\x63hars\x18\n \x01(\rH\x05\x88\x01\x01\x12\x18\n\x0b\x63har_errors\x18\x0b \x01(\rH\x06\x88\x01\x01\x12\x1f\n\x12\x65ntity_recognition\x18\x0e \x01(\x02H\x07\x88\x01\x01\x12\x1d\n\x10\x65ntities_uttered\x18\x0f \x01(\rH\x08\x88\x01\x01\x12 \n\x13\x65ntities_recognized\x18\x10 \x01(\rH\t\x88\x01\x01\x12\x1a\n\rentity_recall\x18\x11 \x01(\x02H\n\x88\x01\x01\x12&\n\x19\x65ntities_acquired_uttered\x18\x12 \x01(\rH\x0b\x88\x01\x01\x12)\n\x1c\x65ntities_acquired_recognized\x18\x13 \x01(\rH\x0c\x88\x01\x01\x42\n\n\x08_ttfb_msB\x06\n\x04_werB\x08\n\x06_wordsB\x0e\n\x0c_word_errorsB\x06\n\x04_cerB\x08\n\x06_charsB\x0e\n\x0c_char_errorsB\x15\n\x13_entity_recognitionB\x13\n\x11_entities_utteredB\x16\n\x14_entities_recognizedB\x10\n\x0e_entity_recallB\x1c\n\x1a_entities_acquired_utteredB\x1f\n\x1d_entities_acquired_recognized\x1a\x82\x06\n\x0c\x43onversation\x12\x1e\n\x11turn_taking_score\x18\x01 \x01(\x02H\x00\x88\x01\x01\x12%\n\x18heard_e2e_latency_p50_ms\x18\x02 \x01(\x05H\x01\x88\x01\x01\x12%\n\x18heard_e2e_latency_p95_ms\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12%\n\x18heard_e2e_latency_p99_ms\x18\x04 \x01(\x05H\x03\x88\x01\x01\x12\x1d\n\x10time_to_yield_ms\x18\x05 \x01(\rH\x04\x88\x01\x01\x12$\n\x17\x65ot_misprediction_count\x18\x06 \x01(\rH\x05\x88\x01\x01\x12\x1a\n\roverlap_ratio\x18\x07 \x01(\x02H\x06\x88\x01\x01\x12\x1e\n\x11overlap_speech_ms\x18\x08 \x01(\rH\x07\x88\x01\x01\x12\x1c\n\x0ftotal_speech_ms\x18\t \x01(\rH\x08\x88\x01\x01\x12\"\n\x15\x61wkward_silence_count\x18\x0b \x01(\rH\t\x88\x01\x01\x12\x1d\n\x10unanswered_turns\x18\x0c \x01(\rH\n\x88\x01\x01\x12%\n\x18\x66\x61lse_interruption_count\x18\r \x01(\rH\x0b\x88\x01\x01\x12\x1b\n\x0e\x65\x32\x65_latency_ms\x18\x0f \x01(\rH\x0c\x88\x01\x01\x42\x14\n\x12_turn_taking_scoreB\x1b\n\x19_heard_e2e_latency_p50_msB\x1b\n\x19_heard_e2e_latency_p95_msB\x1b\n\x19_heard_e2e_latency_p99_msB\x13\n\x11_time_to_yield_msB\x1a\n\x18_eot_misprediction_countB\x10\n\x0e_overlap_ratioB\x14\n\x12_overlap_speech_msB\x12\n\x10_total_speech_msB\x18\n\x16_awkward_silence_countB\x13\n\x11_unanswered_turnsB\x1b\n\x19_false_interruption_countB\x11\n\x0f_e2e_latency_ms\x1a\xdc\x05\n\x04Turn\x12\r\n\x05index\x18\x01 \x01(\r\x12%\n\x04role\x18\x02 \x01(\x0e\x32\x17.livekit.agent.ChatRole\x12\x15\n\x08start_ms\x18\x03 \x01(\rH\x00\x88\x01\x01\x12\x13\n\x06\x65nd_ms\x18\x04 \x01(\rH\x01\x88\x01\x01\x12\x19\n\x0cstt_delay_ms\x18\x05 \x01(\rH\x02\x88\x01\x01\x12\x18\n\x0bllm_ttft_ms\x18\x06 \x01(\rH\x03\x88\x01\x01\x12\x18\n\x0bllm_ttfs_ms\x18\x07 \x01(\rH\x04\x88\x01\x01\x12\x14\n\x07llm_tps\x18\x08 \x01(\x02H\x05\x88\x01\x01\x12\x18\n\x0btts_ttfb_ms\x18\t \x01(\rH\x06\x88\x01\x01\x12\x1b\n\x0e\x65\x32\x65_latency_ms\x18\n \x01(\rH\x07\x88\x01\x01\x12!\n\x14heard_e2e_latency_ms\x18\x0b \x01(\x05H\x08\x88\x01\x01\x12\x1d\n\x10time_to_yield_ms\x18\x0c \x01(\rH\t\x88\x01\x01\x12\x1e\n\x11turn_taking_score\x18\r \x01(\x02H\n\x88\x01\x01\x12\x18\n\x0b\x63onciseness\x18\x0e \x01(\x02H\x0b\x88\x01\x01\x12\x1e\n\x11\x65ot_misprediction\x18\x0f \x01(\x08H\x0c\x88\x01\x01\x12\x1c\n\x0f\x61wkward_silence\x18\x10 \x01(\x08H\r\x88\x01\x01\x12\x17\n\nunanswered\x18\x11 \x01(\x08H\x0e\x88\x01\x01\x42\x0b\n\t_start_msB\t\n\x07_end_msB\x0f\n\r_stt_delay_msB\x0e\n\x0c_llm_ttft_msB\x0e\n\x0c_llm_ttfs_msB\n\n\x08_llm_tpsB\x0e\n\x0c_tts_ttfb_msB\x11\n\x0f_e2e_latency_msB\x17\n\x15_heard_e2e_latency_msB\x13\n\x11_time_to_yield_msB\x14\n\x12_turn_taking_scoreB\x0e\n\x0c_concisenessB\x14\n\x12_eot_mispredictionB\x12\n\x10_awkward_silenceB\r\n\x0b_unansweredB\x11\n\x0f_accuracy_scoreB\x13\n\x11_experience_scoreB\x12\n\x10_task_completionB\x0e\n\x0c_concisenessB\x19\n\x17_unnecessary_tool_callsB\x13\n\x11_information_lossB\x17\n\x15_redundant_statementsB\x18\n\x16_poor_question_qualityB\x1b\n\x19_conversation_progression\x1a\xf8\x03\n\nRunMetrics\x12\x1b\n\x0e\x61\x63\x63uracy_score\x18\x01 \x01(\x02H\x00\x88\x01\x01\x12\x1d\n\x10\x65xperience_score\x18\x02 \x01(\x02H\x01\x88\x01\x01\x12\x1f\n\x12scenario_pass_rate\x18\x03 \x01(\x02H\x02\x88\x01\x01\x12\x32\n\x03stt\x18\x04 \x01(\x0b\x32%.livekit.SimulationRun.JobMetrics.STT\x12\x32\n\x03llm\x18\x05 \x01(\x0b\x32%.livekit.SimulationRun.JobMetrics.LLM\x12\x32\n\x03tts\x18\x06 \x01(\x0b\x32%.livekit.SimulationRun.JobMetrics.TTS\x12\x44\n\x0c\x63onversation\x18\x07 \x01(\x0b\x32..livekit.SimulationRun.JobMetrics.Conversation\x12\x18\n\x0b\x63onciseness\x18\x08 \x01(\x02H\x03\x88\x01\x01\x12%\n\x18\x63onversation_progression\x18\t \x01(\x02H\x04\x88\x01\x01\x42\x11\n\x0f_accuracy_scoreB\x13\n\x11_experience_scoreB\x15\n\x13_scenario_pass_rateB\x0e\n\x0c_concisenessB\x1b\n\x19_conversation_progression\x1a\xcc\x03\n\x06\x43reate\x1a\xdb\x02\n\x07Request\x12\x12\n\nproject_id\x18\x01 \x01(\t\x12\x12\n\nagent_name\x18\x02 \x01(\t\x12\x17\n\x0fnum_simulations\x18\x04 \x01(\x05\x12\x0e\n\x06region\x18\x06 \x01(\t\x12\x33\n\x0escenario_group\x18\x07 \x01(\x0b\x32\x16.livekit.ScenarioGroupH\x00\x88\x01\x01\x12\x18\n\x0b\x63oncurrency\x18\x08 \x01(\x05H\x01\x88\x01\x01\x12%\n\x04mode\x18\t \x01(\x0e\x32\x17.livekit.SimulationMode\x12\x18\n\x10\x62\x61\x63kground_noise\x18\n \x01(\x08\x12\x1e\n\x16low_quality_microphone\x18\x0b \x01(\x08\x12\x13\n\x0bpacket_loss\x18\x0c \x01(\x08\x42\x11\n\x0f_scenario_groupB\x0e\n\x0c_concurrencyJ\x04\x08\x03\x10\x04R\x11\x61gent_description\x1a\x64\n\x08Response\x12\x19\n\x11simulation_run_id\x18\x01 \x01(\t\x12=\n\x16presigned_post_request\x18\x02 \x01(\x0b\x32\x1d.livekit.PresignedPostRequest\x1at\n\x13\x43onfirmSourceUpload\x1aQ\n\x07Request\x12\x12\n\nproject_id\x18\x01 \x01(\t\x12\x19\n\x11simulation_run_id\x18\x02 \x01(\t\x12\x17\n\x0f\x63ode_entrypoint\x18\x03 \x01(\t\x1a\n\n\x08Response\x1ap\n\x03Get\x1a\x38\n\x07Request\x12\x12\n\nproject_id\x18\x01 \x01(\t\x12\x19\n\x11simulation_run_id\x18\x02 \x01(\t\x1a/\n\x08Response\x12#\n\x03run\x18\x01 \x01(\x0b\x32\x16.livekit.SimulationRun\x1a\xb8\x03\n\x04List\x1a\xca\x02\n\x07Request\x12\x12\n\nproject_id\x18\x01 \x01(\t\x12\x32\n\x06status\x18\x02 \x01(\x0e\x32\x1d.livekit.SimulationRun.StatusH\x00\x88\x01\x01\x12\x31\n\npage_token\x18\x03 \x01(\x0b\x32\x18.livekit.TokenPaginationH\x01\x88\x01\x01\x12\x33\n\nstart_time\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x02\x88\x01\x01\x12\x31\n\x08\x65nd_time\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x03\x88\x01\x01\x12\x17\n\nagent_name\x18\x06 \x01(\tH\x04\x88\x01\x01\x42\t\n\x07_statusB\r\n\x0b_page_tokenB\r\n\x0b_start_timeB\x0b\n\t_end_timeB\r\n\x0b_agent_name\x1a\x63\n\x08Response\x12$\n\x04runs\x18\x01 \x03(\x0b\x32\x16.livekit.SimulationRun\x12\x31\n\x0fnext_page_token\x18\x02 \x01(\x0b\x32\x18.livekit.TokenPagination\x1a\xa8\x03\n\x06\x43ounts\x1a\xe2\x01\n\x07Request\x12\x12\n\nproject_id\x18\x01 \x01(\t\x12.\n\nstart_time\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12,\n\x08\x65nd_time\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x32\n\x06status\x18\x04 \x01(\x0e\x32\x1d.livekit.SimulationRun.StatusH\x00\x88\x01\x01\x12\x17\n\nagent_name\x18\x05 \x01(\tH\x01\x88\x01\x01\x42\t\n\x07_statusB\r\n\x0b_agent_name\x1an\n\x08Response\x12\x35\n\x07\x62uckets\x18\x01 \x03(\x0b\x32$.livekit.SimulationRun.Counts.Bucket\x12+\n\x08interval\x18\x02 \x01(\x0b\x32\x19.google.protobuf.Duration\x1aI\n\x06\x42ucket\x12\x30\n\x0c\x62ucket_start\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\r\n\x05\x63ount\x18\x02 \x01(\x04\x1aN\n\x06\x43\x61ncel\x1a\x38\n\x07Request\x12\x12\n\nproject_id\x18\x01 \x01(\t\x12\x19\n\x11simulation_run_id\x18\x02 \x01(\t\x1a\n\n\x08Response\x1a<\n\x05Usage\x12\x18\n\x10text_turns_count\x18\x01 \x01(\x05\x12\x19\n\x11\x61udio_turns_count\x18\x02 \x01(\x05\"\xa5\x01\n\x06Status\x12\x19\n\x15STATUS_PENDING_UPLOAD\x10\x00\x12\x15\n\x11STATUS_GENERATING\x10\x01\x12\x12\n\x0eSTATUS_RUNNING\x10\x02\x12\x16\n\x12STATUS_SUMMARIZING\x10\x03\x12\x14\n\x10STATUS_COMPLETED\x10\x04\x12\x11\n\rSTATUS_FAILED\x10\x05\x12\x14\n\x10STATUS_CANCELLED\x10\x06J\x04\x08\x08\x10\tR\x07summary\"\xbc\x02\n\x08Scenario\x12\r\n\x05label\x18\x01 \x01(\t\x12\x14\n\x0cinstructions\x18\x02 \x01(\t\x12\x1a\n\x12\x61gent_expectations\x18\x03 \x01(\t\x12)\n\x04tags\x18\x04 \x03(\x0b\x32\x1b.livekit.Scenario.TagsEntry\x12\x10\n\x08userdata\x18\x05 \x01(\t\x1a\x84\x01\n\x11\x43reateFromSession\x1a>\n\x07Request\x12\x12\n\nproject_id\x18\x01 \x01(\t\x12\x0f\n\x07room_id\x18\x02 \x01(\t\x12\x0e\n\x06region\x18\x03 \x01(\t\x1a/\n\x08Response\x12#\n\x08scenario\x18\x01 \x01(\x0b\x32\x11.livekit.Scenario\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"C\n\rScenarioGroup\x12\x0c\n\x04name\x18\x01 \x01(\t\x12$\n\tscenarios\x18\x02 \x03(\x0b\x32\x11.livekit.Scenario\"\x8b\x01\n\x12SimulationDispatch\x12\x19\n\x11simulation_run_id\x18\x01 \x01(\t\x12\x0e\n\x06job_id\x18\x02 \x01(\t\x12#\n\x08scenario\x18\x03 \x01(\x0b\x32\x11.livekit.Scenario\x12%\n\x04mode\x18\x04 \x01(\x0e\x32\x17.livekit.SimulationMode*f\n\x0eSimulationMode\x12\x1f\n\x1bSIMULATION_MODE_UNSPECIFIED\x10\x00\x12\x18\n\x14SIMULATION_MODE_TEXT\x10\x01\x12\x19\n\x15SIMULATION_MODE_AUDIO\x10\x02\x32\x84\x06\n\x0f\x41gentSimulation\x12\x64\n\x13\x43reateSimulationRun\x12%.livekit.SimulationRun.Create.Request\x1a&.livekit.SimulationRun.Create.Response\x12\x88\x01\n\x1d\x43onfirmSimulationSourceUpload\x12\x32.livekit.SimulationRun.ConfirmSourceUpload.Request\x1a\x33.livekit.SimulationRun.ConfirmSourceUpload.Response\x12[\n\x10GetSimulationRun\x12\".livekit.SimulationRun.Get.Request\x1a#.livekit.SimulationRun.Get.Response\x12_\n\x12ListSimulationRuns\x12#.livekit.SimulationRun.List.Request\x1a$.livekit.SimulationRun.List.Response\x12\x64\n\x13\x43ountSimulationRuns\x12%.livekit.SimulationRun.Counts.Request\x1a&.livekit.SimulationRun.Counts.Response\x12\x64\n\x13\x43\x61ncelSimulationRun\x12%.livekit.SimulationRun.Cancel.Request\x1a&.livekit.SimulationRun.Cancel.Response\x12v\n\x19\x43reateScenarioFromSession\x12+.livekit.Scenario.CreateFromSession.Request\x1a,.livekit.Scenario.CreateFromSession.ResponseBFZ#github.com/livekit/protocol/livekit\xaa\x02\rLiveKit.Proto\xea\x02\x0eLiveKit::Protob\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -41,8 +41,8 @@ _globals['_SIMULATIONRUNSUMMARY_CHATHISTORYENTRY']._serialized_options = b'8\001' _globals['_SCENARIO_TAGSENTRY']._loaded_options = None _globals['_SCENARIO_TAGSENTRY']._serialized_options = b'8\001' - _globals['_SIMULATIONMODE']._serialized_start=8283 - _globals['_SIMULATIONMODE']._serialized_end=8385 + _globals['_SIMULATIONMODE']._serialized_start=8393 + _globals['_SIMULATIONMODE']._serialized_end=8495 _globals['_SIMULATIONRUNSUMMARY']._serialized_start=193 _globals['_SIMULATIONRUNSUMMARY']._serialized_end=555 _globals['_SIMULATIONRUNSUMMARY_CHATHISTORYENTRY']._serialized_start=412 @@ -50,7 +50,7 @@ _globals['_SIMULATIONRUNSUMMARY_ISSUE']._serialized_start=492 _globals['_SIMULATIONRUNSUMMARY_ISSUE']._serialized_end=555 _globals['_SIMULATIONRUN']._serialized_start=558 - _globals['_SIMULATIONRUN']._serialized_end=7751 + _globals['_SIMULATIONRUN']._serialized_end=7861 _globals['_SIMULATIONRUN_JOB']._serialized_start=1168 _globals['_SIMULATIONRUN_JOB']._serialized_end=1742 _globals['_SIMULATIONRUN_JOB_USAGE']._serialized_start=1563 @@ -70,63 +70,63 @@ _globals['_SIMULATIONRUN_JOBMETRICS_TURN']._serialized_start=4528 _globals['_SIMULATIONRUN_JOBMETRICS_TURN']._serialized_end=5260 _globals['_SIMULATIONRUN_RUNMETRICS']._serialized_start=5467 - _globals['_SIMULATIONRUN_RUNMETRICS']._serialized_end=5861 - _globals['_SIMULATIONRUN_CREATE']._serialized_start=5864 - _globals['_SIMULATIONRUN_CREATE']._serialized_end=6324 - _globals['_SIMULATIONRUN_CREATE_REQUEST']._serialized_start=5875 - _globals['_SIMULATIONRUN_CREATE_REQUEST']._serialized_end=6222 - _globals['_SIMULATIONRUN_CREATE_RESPONSE']._serialized_start=6224 - _globals['_SIMULATIONRUN_CREATE_RESPONSE']._serialized_end=6324 - _globals['_SIMULATIONRUN_CONFIRMSOURCEUPLOAD']._serialized_start=6326 - _globals['_SIMULATIONRUN_CONFIRMSOURCEUPLOAD']._serialized_end=6442 - _globals['_SIMULATIONRUN_CONFIRMSOURCEUPLOAD_REQUEST']._serialized_start=6349 - _globals['_SIMULATIONRUN_CONFIRMSOURCEUPLOAD_REQUEST']._serialized_end=6430 - _globals['_SIMULATIONRUN_CONFIRMSOURCEUPLOAD_RESPONSE']._serialized_start=6224 - _globals['_SIMULATIONRUN_CONFIRMSOURCEUPLOAD_RESPONSE']._serialized_end=6234 - _globals['_SIMULATIONRUN_GET']._serialized_start=6444 - _globals['_SIMULATIONRUN_GET']._serialized_end=6556 - _globals['_SIMULATIONRUN_GET_REQUEST']._serialized_start=6349 - _globals['_SIMULATIONRUN_GET_REQUEST']._serialized_end=6405 - _globals['_SIMULATIONRUN_GET_RESPONSE']._serialized_start=6509 - _globals['_SIMULATIONRUN_GET_RESPONSE']._serialized_end=6556 - _globals['_SIMULATIONRUN_LIST']._serialized_start=6559 - _globals['_SIMULATIONRUN_LIST']._serialized_end=6999 - _globals['_SIMULATIONRUN_LIST_REQUEST']._serialized_start=6568 - _globals['_SIMULATIONRUN_LIST_REQUEST']._serialized_end=6898 - _globals['_SIMULATIONRUN_LIST_RESPONSE']._serialized_start=6900 - _globals['_SIMULATIONRUN_LIST_RESPONSE']._serialized_end=6999 - _globals['_SIMULATIONRUN_COUNTS']._serialized_start=7002 - _globals['_SIMULATIONRUN_COUNTS']._serialized_end=7426 - _globals['_SIMULATIONRUN_COUNTS_REQUEST']._serialized_start=7013 - _globals['_SIMULATIONRUN_COUNTS_REQUEST']._serialized_end=7239 - _globals['_SIMULATIONRUN_COUNTS_RESPONSE']._serialized_start=7241 - _globals['_SIMULATIONRUN_COUNTS_RESPONSE']._serialized_end=7351 - _globals['_SIMULATIONRUN_COUNTS_BUCKET']._serialized_start=7353 - _globals['_SIMULATIONRUN_COUNTS_BUCKET']._serialized_end=7426 - _globals['_SIMULATIONRUN_CANCEL']._serialized_start=7428 - _globals['_SIMULATIONRUN_CANCEL']._serialized_end=7506 - _globals['_SIMULATIONRUN_CANCEL_REQUEST']._serialized_start=6349 - _globals['_SIMULATIONRUN_CANCEL_REQUEST']._serialized_end=6405 - _globals['_SIMULATIONRUN_CANCEL_RESPONSE']._serialized_start=6224 - _globals['_SIMULATIONRUN_CANCEL_RESPONSE']._serialized_end=6234 + _globals['_SIMULATIONRUN_RUNMETRICS']._serialized_end=5971 + _globals['_SIMULATIONRUN_CREATE']._serialized_start=5974 + _globals['_SIMULATIONRUN_CREATE']._serialized_end=6434 + _globals['_SIMULATIONRUN_CREATE_REQUEST']._serialized_start=5985 + _globals['_SIMULATIONRUN_CREATE_REQUEST']._serialized_end=6332 + _globals['_SIMULATIONRUN_CREATE_RESPONSE']._serialized_start=6334 + _globals['_SIMULATIONRUN_CREATE_RESPONSE']._serialized_end=6434 + _globals['_SIMULATIONRUN_CONFIRMSOURCEUPLOAD']._serialized_start=6436 + _globals['_SIMULATIONRUN_CONFIRMSOURCEUPLOAD']._serialized_end=6552 + _globals['_SIMULATIONRUN_CONFIRMSOURCEUPLOAD_REQUEST']._serialized_start=6459 + _globals['_SIMULATIONRUN_CONFIRMSOURCEUPLOAD_REQUEST']._serialized_end=6540 + _globals['_SIMULATIONRUN_CONFIRMSOURCEUPLOAD_RESPONSE']._serialized_start=6334 + _globals['_SIMULATIONRUN_CONFIRMSOURCEUPLOAD_RESPONSE']._serialized_end=6344 + _globals['_SIMULATIONRUN_GET']._serialized_start=6554 + _globals['_SIMULATIONRUN_GET']._serialized_end=6666 + _globals['_SIMULATIONRUN_GET_REQUEST']._serialized_start=6459 + _globals['_SIMULATIONRUN_GET_REQUEST']._serialized_end=6515 + _globals['_SIMULATIONRUN_GET_RESPONSE']._serialized_start=6619 + _globals['_SIMULATIONRUN_GET_RESPONSE']._serialized_end=6666 + _globals['_SIMULATIONRUN_LIST']._serialized_start=6669 + _globals['_SIMULATIONRUN_LIST']._serialized_end=7109 + _globals['_SIMULATIONRUN_LIST_REQUEST']._serialized_start=6678 + _globals['_SIMULATIONRUN_LIST_REQUEST']._serialized_end=7008 + _globals['_SIMULATIONRUN_LIST_RESPONSE']._serialized_start=7010 + _globals['_SIMULATIONRUN_LIST_RESPONSE']._serialized_end=7109 + _globals['_SIMULATIONRUN_COUNTS']._serialized_start=7112 + _globals['_SIMULATIONRUN_COUNTS']._serialized_end=7536 + _globals['_SIMULATIONRUN_COUNTS_REQUEST']._serialized_start=7123 + _globals['_SIMULATIONRUN_COUNTS_REQUEST']._serialized_end=7349 + _globals['_SIMULATIONRUN_COUNTS_RESPONSE']._serialized_start=7351 + _globals['_SIMULATIONRUN_COUNTS_RESPONSE']._serialized_end=7461 + _globals['_SIMULATIONRUN_COUNTS_BUCKET']._serialized_start=7463 + _globals['_SIMULATIONRUN_COUNTS_BUCKET']._serialized_end=7536 + _globals['_SIMULATIONRUN_CANCEL']._serialized_start=7538 + _globals['_SIMULATIONRUN_CANCEL']._serialized_end=7616 + _globals['_SIMULATIONRUN_CANCEL_REQUEST']._serialized_start=6459 + _globals['_SIMULATIONRUN_CANCEL_REQUEST']._serialized_end=6515 + _globals['_SIMULATIONRUN_CANCEL_RESPONSE']._serialized_start=6334 + _globals['_SIMULATIONRUN_CANCEL_RESPONSE']._serialized_end=6344 _globals['_SIMULATIONRUN_USAGE']._serialized_start=1563 _globals['_SIMULATIONRUN_USAGE']._serialized_end=1623 - _globals['_SIMULATIONRUN_STATUS']._serialized_start=7571 - _globals['_SIMULATIONRUN_STATUS']._serialized_end=7736 - _globals['_SCENARIO']._serialized_start=7754 - _globals['_SCENARIO']._serialized_end=8070 - _globals['_SCENARIO_CREATEFROMSESSION']._serialized_start=7893 - _globals['_SCENARIO_CREATEFROMSESSION']._serialized_end=8025 - _globals['_SCENARIO_CREATEFROMSESSION_REQUEST']._serialized_start=7914 - _globals['_SCENARIO_CREATEFROMSESSION_REQUEST']._serialized_end=7976 - _globals['_SCENARIO_CREATEFROMSESSION_RESPONSE']._serialized_start=7978 - _globals['_SCENARIO_CREATEFROMSESSION_RESPONSE']._serialized_end=8025 - _globals['_SCENARIO_TAGSENTRY']._serialized_start=8027 - _globals['_SCENARIO_TAGSENTRY']._serialized_end=8070 - _globals['_SCENARIOGROUP']._serialized_start=8072 - _globals['_SCENARIOGROUP']._serialized_end=8139 - _globals['_SIMULATIONDISPATCH']._serialized_start=8142 - _globals['_SIMULATIONDISPATCH']._serialized_end=8281 - _globals['_AGENTSIMULATION']._serialized_start=8388 - _globals['_AGENTSIMULATION']._serialized_end=9160 + _globals['_SIMULATIONRUN_STATUS']._serialized_start=7681 + _globals['_SIMULATIONRUN_STATUS']._serialized_end=7846 + _globals['_SCENARIO']._serialized_start=7864 + _globals['_SCENARIO']._serialized_end=8180 + _globals['_SCENARIO_CREATEFROMSESSION']._serialized_start=8003 + _globals['_SCENARIO_CREATEFROMSESSION']._serialized_end=8135 + _globals['_SCENARIO_CREATEFROMSESSION_REQUEST']._serialized_start=8024 + _globals['_SCENARIO_CREATEFROMSESSION_REQUEST']._serialized_end=8086 + _globals['_SCENARIO_CREATEFROMSESSION_RESPONSE']._serialized_start=8088 + _globals['_SCENARIO_CREATEFROMSESSION_RESPONSE']._serialized_end=8135 + _globals['_SCENARIO_TAGSENTRY']._serialized_start=8137 + _globals['_SCENARIO_TAGSENTRY']._serialized_end=8180 + _globals['_SCENARIOGROUP']._serialized_start=8182 + _globals['_SCENARIOGROUP']._serialized_end=8249 + _globals['_SIMULATIONDISPATCH']._serialized_start=8252 + _globals['_SIMULATIONDISPATCH']._serialized_end=8391 + _globals['_AGENTSIMULATION']._serialized_start=8498 + _globals['_AGENTSIMULATION']._serialized_end=9270 # @@protoc_insertion_point(module_scope) diff --git a/livekit-protocol/livekit/protocol/agent_simulation.pyi b/livekit-protocol/livekit/protocol/agent_simulation.pyi index 6d1249de..aed4ad74 100644 --- a/livekit-protocol/livekit/protocol/agent_simulation.pyi +++ b/livekit-protocol/livekit/protocol/agent_simulation.pyi @@ -289,7 +289,7 @@ class SimulationRun(_message.Message): conversation_progression: float def __init__(self, accuracy_score: _Optional[float] = ..., experience_score: _Optional[float] = ..., task_completion: _Optional[float] = ..., stt: _Optional[_Union[SimulationRun.JobMetrics.STT, _Mapping]] = ..., llm: _Optional[_Union[SimulationRun.JobMetrics.LLM, _Mapping]] = ..., tts: _Optional[_Union[SimulationRun.JobMetrics.TTS, _Mapping]] = ..., conversation: _Optional[_Union[SimulationRun.JobMetrics.Conversation, _Mapping]] = ..., turns: _Optional[_Iterable[_Union[SimulationRun.JobMetrics.Turn, _Mapping]]] = ..., judge_model: _Optional[str] = ..., has_remote_session: bool = ..., t0: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., conciseness: _Optional[float] = ..., unnecessary_tool_calls: bool = ..., information_loss: bool = ..., redundant_statements: bool = ..., poor_question_quality: bool = ..., conversation_progression: _Optional[float] = ...) -> None: ... class RunMetrics(_message.Message): - __slots__ = ("accuracy_score", "experience_score", "scenario_pass_rate", "stt", "llm", "tts", "conversation") + __slots__ = ("accuracy_score", "experience_score", "scenario_pass_rate", "stt", "llm", "tts", "conversation", "conciseness", "conversation_progression") ACCURACY_SCORE_FIELD_NUMBER: _ClassVar[int] EXPERIENCE_SCORE_FIELD_NUMBER: _ClassVar[int] SCENARIO_PASS_RATE_FIELD_NUMBER: _ClassVar[int] @@ -297,6 +297,8 @@ class SimulationRun(_message.Message): LLM_FIELD_NUMBER: _ClassVar[int] TTS_FIELD_NUMBER: _ClassVar[int] CONVERSATION_FIELD_NUMBER: _ClassVar[int] + CONCISENESS_FIELD_NUMBER: _ClassVar[int] + CONVERSATION_PROGRESSION_FIELD_NUMBER: _ClassVar[int] accuracy_score: float experience_score: float scenario_pass_rate: float @@ -304,7 +306,9 @@ class SimulationRun(_message.Message): llm: SimulationRun.JobMetrics.LLM tts: SimulationRun.JobMetrics.TTS conversation: SimulationRun.JobMetrics.Conversation - def __init__(self, accuracy_score: _Optional[float] = ..., experience_score: _Optional[float] = ..., scenario_pass_rate: _Optional[float] = ..., stt: _Optional[_Union[SimulationRun.JobMetrics.STT, _Mapping]] = ..., llm: _Optional[_Union[SimulationRun.JobMetrics.LLM, _Mapping]] = ..., tts: _Optional[_Union[SimulationRun.JobMetrics.TTS, _Mapping]] = ..., conversation: _Optional[_Union[SimulationRun.JobMetrics.Conversation, _Mapping]] = ...) -> None: ... + conciseness: float + conversation_progression: float + def __init__(self, accuracy_score: _Optional[float] = ..., experience_score: _Optional[float] = ..., scenario_pass_rate: _Optional[float] = ..., stt: _Optional[_Union[SimulationRun.JobMetrics.STT, _Mapping]] = ..., llm: _Optional[_Union[SimulationRun.JobMetrics.LLM, _Mapping]] = ..., tts: _Optional[_Union[SimulationRun.JobMetrics.TTS, _Mapping]] = ..., conversation: _Optional[_Union[SimulationRun.JobMetrics.Conversation, _Mapping]] = ..., conciseness: _Optional[float] = ..., conversation_progression: _Optional[float] = ...) -> None: ... class Create(_message.Message): __slots__ = () class Request(_message.Message): diff --git a/livekit-protocol/protocol b/livekit-protocol/protocol index d68a0133..dc17522d 160000 --- a/livekit-protocol/protocol +++ b/livekit-protocol/protocol @@ -1 +1 @@ -Subproject commit d68a01332d071f9fd25783f746e47246ff2d772b +Subproject commit dc17522d2e6d551ce93153059b65cce0e74b9007