From a72290acec53acddf82ab48555ee9a7d3da2648d Mon Sep 17 00:00:00 2001 From: Andrey Buzin Date: Wed, 23 Sep 2026 17:09:59 -0700 Subject: [PATCH] Change ai.ops.experimental_evaluate to ai.ops.experimental.evaluate --- .../content/docs/basics/model-operations.mdx | 26 ++-- docs/ai-python/content/docs/reference/ops.mdx | 11 +- examples/models/gateway/evaluation.py | 32 ++--- src/ai/ops/__init__.py | 24 +--- src/ai/ops/experimental/__init__.py | 27 ++++ src/ai/ops/{ => experimental}/evaluation.py | 20 +-- src/ai/providers/ai_gateway/protocol/v4.py | 22 ++-- src/ai/providers/base.py | 2 +- tests/ops/experimental/__init__.py | 0 .../ops/{ => experimental}/test_evaluation.py | 116 ++++++++++-------- .../ai_gateway/protocol/test_v4_evaluation.py | 38 +++--- 11 files changed, 170 insertions(+), 148 deletions(-) create mode 100644 src/ai/ops/experimental/__init__.py rename src/ai/ops/{ => experimental}/evaluation.py (95%) create mode 100644 tests/ops/experimental/__init__.py rename tests/ops/{ => experimental}/test_evaluation.py (77%) diff --git a/docs/ai-python/content/docs/basics/model-operations.mdx b/docs/ai-python/content/docs/basics/model-operations.mdx index e101d396..966aa68c 100644 --- a/docs/ai-python/content/docs/basics/model-operations.mdx +++ b/docs/ai-python/content/docs/basics/model-operations.mdx @@ -255,37 +255,37 @@ are ordered from the highest score to the lowest. ## Evaluate state -Use `experimental_evaluate` to ask choice, score, and boolean questions about one shared state. Define matching Pydantic models for the questions and answers: +Use `ai.ops.experimental.evaluate` to ask choice, score, and boolean questions about one shared state. Define matching Pydantic models for the questions and answers: ```python import pydantic class Questions(pydantic.BaseModel): - department: ai.ops.ChoiceQuestion - severity: ai.ops.ScoreQuestion - requests_refund: ai.ops.BooleanQuestion + department: ai.ops.experimental.ChoiceQuestion + severity: ai.ops.experimental.ScoreQuestion + requests_refund: ai.ops.experimental.BooleanQuestion class Answers(pydantic.BaseModel): - department: ai.ops.ChoiceAnswer - severity: ai.ops.ScoreAnswer - requests_refund: ai.ops.BooleanAnswer + department: ai.ops.experimental.ChoiceAnswer + severity: ai.ops.experimental.ScoreAnswer + requests_refund: ai.ops.experimental.BooleanAnswer -result = await ai.ops.experimental_evaluate( +result = await ai.ops.experimental.evaluate( ai.get_model("typesafe-ai/jev"), {"message": "Please refund the duplicate charge."}, Questions( - department=ai.ops.ChoiceQuestion( + department=ai.ops.experimental.ChoiceQuestion( instructions="Which team should handle this?", criteria={"billing": "Charges and refunds", "support": "Other"}, ), - severity=ai.ops.ScoreQuestion( + severity=ai.ops.experimental.ScoreQuestion( instructions="How severe is this?", criteria=["Cosmetic", "Has workaround", "Blocking"], ), - requests_refund=ai.ops.BooleanQuestion( + requests_refund=ai.ops.experimental.BooleanQuestion( instructions="Is the customer requesting a refund?", ), ), @@ -299,11 +299,11 @@ When question IDs are dynamic, pass a mapping and omit `output_type`. The item value is then a validated answer mapping: ```python -result = await ai.ops.experimental_evaluate( +result = await ai.ops.experimental.evaluate( ai.get_model("typesafe-ai/jev"), state, { - question_id: ai.ops.BooleanQuestion(instructions=instructions) + question_id: ai.ops.experimental.BooleanQuestion(instructions=instructions) for question_id, instructions in dynamic_questions.items() }, ) diff --git a/docs/ai-python/content/docs/reference/ops.mdx b/docs/ai-python/content/docs/reference/ops.mdx index d322f0a1..2abe8641 100644 --- a/docs/ai-python/content/docs/reference/ops.mdx +++ b/docs/ai-python/content/docs/reference/ops.mdx @@ -239,10 +239,13 @@ provider. - `index`: Position of the document in the original input list. - `score`: Relevance score for the query. -## experimental_evaluate +## experimental.evaluate + +Evaluation questions, answers, `EvaluationInput`, and `EvaluationParams` live +under `ai.ops.experimental`. ```python -await ai.ops.experimental_evaluate( +await ai.ops.experimental.evaluate( model, state, questions, @@ -268,11 +271,11 @@ Pydantic questions return `Item[Answers]`, whose value is an instance of `Item[dict[str, ChoiceAnswer | ScoreAnswer | BooleanAnswer]]`: ```python -result = await ai.ops.experimental_evaluate( +result = await ai.ops.experimental.evaluate( model, state, { - question_id: ai.ops.BooleanQuestion(instructions=instructions) + question_id: ai.ops.experimental.BooleanQuestion(instructions=instructions) for question_id, instructions in dynamic_questions.items() }, ) diff --git a/examples/models/gateway/evaluation.py b/examples/models/gateway/evaluation.py index 527d9ea2..4e661981 100644 --- a/examples/models/gateway/evaluation.py +++ b/examples/models/gateway/evaluation.py @@ -17,23 +17,23 @@ class RefundQuestions(pydantic.BaseModel): - requests_refund: ai.ops.BooleanQuestion + requests_refund: ai.ops.experimental.BooleanQuestion class RefundAnswers(pydantic.BaseModel): - requests_refund: ai.ops.BooleanAnswer + requests_refund: ai.ops.experimental.BooleanAnswer class TicketQuestions(pydantic.BaseModel): - queue: ai.ops.ChoiceQuestion - urgency: ai.ops.ScoreQuestion - refund_warranted: ai.ops.BooleanQuestion + queue: ai.ops.experimental.ChoiceQuestion + urgency: ai.ops.experimental.ScoreQuestion + refund_warranted: ai.ops.experimental.BooleanQuestion class TicketAnswers(pydantic.BaseModel): - queue: ai.ops.ChoiceAnswer - urgency: ai.ops.ScoreAnswer - refund_warranted: ai.ops.BooleanAnswer + queue: ai.ops.experimental.ChoiceAnswer + urgency: ai.ops.experimental.ScoreAnswer + refund_warranted: ai.ops.experimental.BooleanAnswer async def main() -> None: @@ -44,11 +44,11 @@ async def main() -> None: # Ask a single boolean question about plain text. Boolean answers are # probabilities rather than only true or false. - refund_result = await ai.ops.experimental_evaluate( + refund_result = await ai.ops.experimental.evaluate( model, "Please refund the duplicate charge on my account.", RefundQuestions( - requests_refund=ai.ops.BooleanQuestion( + requests_refund=ai.ops.experimental.BooleanQuestion( instructions="Is the customer asking for a refund?", ) ), @@ -62,7 +62,7 @@ async def main() -> None: # Ask several question types about the same structured state. This is useful # when related decisions should use exactly the same source information. - ticket: ai.ops.EvaluationInput = { + ticket: ai.ops.experimental.EvaluationInput = { "message": ( "I was charged $240 twice for the same renewal. The service works, " "but please refund the duplicate charge." @@ -88,11 +88,11 @@ async def main() -> None: "service_status": "operational", } - result = await ai.ops.experimental_evaluate( + result = await ai.ops.experimental.evaluate( model, ticket, TicketQuestions( - queue=ai.ops.ChoiceQuestion( + queue=ai.ops.experimental.ChoiceQuestion( instructions="Which support queue should handle this ticket?", criteria={ "billing": "Charges, duplicate payments, and refunds", @@ -101,7 +101,7 @@ async def main() -> None: "other": None, }, ), - urgency=ai.ops.ScoreQuestion( + urgency=ai.ops.experimental.ScoreQuestion( instructions="How urgent is the customer's primary problem?", criteria=[ "Low: no active customer impact", @@ -110,7 +110,7 @@ async def main() -> None: "Critical: security incident, outage, or ongoing loss", ], ), - refund_warranted=ai.ops.BooleanQuestion( + refund_warranted=ai.ops.experimental.BooleanQuestion( instructions="Does the evidence warrant a refund?", criteria={ "true": "A duplicate settlement or billing error is shown", @@ -120,7 +120,7 @@ async def main() -> None: ), output_type=TicketAnswers, # Provider options are optional and apply only to this request. - params=ai.ops.EvaluationParams( + params=ai.ops.experimental.EvaluationParams( provider_options={ "gateway": { "zeroDataRetention": True, diff --git a/src/ai/ops/__init__.py b/src/ai/ops/__init__.py index 2619e9ad..1e75c4ca 100644 --- a/src/ai/ops/__init__.py +++ b/src/ai/ops/__init__.py @@ -1,19 +1,8 @@ """Model operations beyond LLM chat: media generation and friends.""" +from . import experimental from .audio import AudioParams, AudioPrompt, generate_audio from .embeddings import EmbedParams, embed -from .evaluation import ( - BooleanAnswer, - BooleanCriteria, - BooleanQuestion, - ChoiceAnswer, - ChoiceQuestion, - EvaluationInput, - EvaluationParams, - ScoreAnswer, - ScoreQuestion, - experimental_evaluate, -) from .images import ImageParams, ImagePrompt, generate_image from .items import Item, Warning from .reranking import RankedDocument, RerankParams, rerank @@ -28,22 +17,13 @@ __all__ = [ "AudioParams", "AudioPrompt", - "BooleanAnswer", - "BooleanCriteria", - "BooleanQuestion", - "ChoiceAnswer", - "ChoiceQuestion", "EmbedParams", - "EvaluationInput", - "EvaluationParams", "FrameImage", "ImageParams", "ImagePrompt", "Item", "RankedDocument", "RerankParams", - "ScoreAnswer", - "ScoreQuestion", "TranscribeParams", "Transcription", "TranscriptionSegment", @@ -51,7 +31,7 @@ "VideoPrompt", "Warning", "embed", - "experimental_evaluate", + "experimental", "generate_audio", "generate_image", "generate_video", diff --git a/src/ai/ops/experimental/__init__.py b/src/ai/ops/experimental/__init__.py new file mode 100644 index 00000000..48b6e9ce --- /dev/null +++ b/src/ai/ops/experimental/__init__.py @@ -0,0 +1,27 @@ +"""Experimental model operations.""" + +from .evaluation import ( + BooleanAnswer, + BooleanCriteria, + BooleanQuestion, + ChoiceAnswer, + ChoiceQuestion, + EvaluationInput, + EvaluationParams, + ScoreAnswer, + ScoreQuestion, + evaluate, +) + +__all__ = [ + "BooleanAnswer", + "BooleanCriteria", + "BooleanQuestion", + "ChoiceAnswer", + "ChoiceQuestion", + "EvaluationInput", + "EvaluationParams", + "ScoreAnswer", + "ScoreQuestion", + "evaluate", +] diff --git a/src/ai/ops/evaluation.py b/src/ai/ops/experimental/evaluation.py similarity index 95% rename from src/ai/ops/evaluation.py rename to src/ai/ops/experimental/evaluation.py index 1b78d0d8..157a075f 100644 --- a/src/ai/ops/evaluation.py +++ b/src/ai/ops/experimental/evaluation.py @@ -6,16 +6,16 @@ import pydantic class Questions(pydantic.BaseModel): - requests_refund: ai.ops.BooleanQuestion + requests_refund: ai.ops.experimental.BooleanQuestion class Answers(pydantic.BaseModel): - requests_refund: ai.ops.BooleanAnswer + requests_refund: ai.ops.experimental.BooleanAnswer - result = await ai.ops.experimental_evaluate( + result = await ai.ops.experimental.evaluate( ai.get_model("typesafe-ai/jev"), {"message": "Please refund the duplicate charge."}, Questions( - requests_refund=ai.ops.BooleanQuestion( + requests_refund=ai.ops.experimental.BooleanQuestion( instructions="Is the customer requesting a refund?", ), ), @@ -32,11 +32,11 @@ class Answers(pydantic.BaseModel): import pydantic -from .. import experimental_telemetry as telemetry -from . import items +from ... import experimental_telemetry as telemetry +from .. import items if TYPE_CHECKING: - from ..models.core import model as model_ + from ...models.core import model as model_ type EvaluationInput = ( @@ -160,7 +160,7 @@ class EvaluationParams: @overload -async def experimental_evaluate[AnswerT: pydantic.BaseModel]( +async def evaluate[AnswerT: pydantic.BaseModel]( model: model_.Model, state: EvaluationInput, questions: pydantic.BaseModel, @@ -171,7 +171,7 @@ async def experimental_evaluate[AnswerT: pydantic.BaseModel]( @overload -async def experimental_evaluate( +async def evaluate( model: model_.Model, state: EvaluationInput, questions: Mapping[str, pydantic.BaseModel], @@ -181,7 +181,7 @@ async def experimental_evaluate( ) -> items.Item[dict[str, _Answer]]: ... -async def experimental_evaluate( +async def evaluate( model: model_.Model, state: EvaluationInput, questions: pydantic.BaseModel | Mapping[str, pydantic.BaseModel], diff --git a/src/ai/providers/ai_gateway/protocol/v4.py b/src/ai/providers/ai_gateway/protocol/v4.py index 2933e7cc..484851e6 100644 --- a/src/ai/providers/ai_gateway/protocol/v4.py +++ b/src/ai/providers/ai_gateway/protocol/v4.py @@ -535,22 +535,22 @@ async def stream( async def evaluate( gateway: gateway_client.GatewayClient, model: models.Model, - state: ops.evaluation.EvaluationInput, + state: ops.experimental.EvaluationInput, questions: Mapping[ str, - ops.evaluation.ChoiceQuestion - | ops.evaluation.ScoreQuestion - | ops.evaluation.BooleanQuestion, + ops.experimental.ChoiceQuestion + | ops.experimental.ScoreQuestion + | ops.experimental.BooleanQuestion, ], *, - params: ops.evaluation.EvaluationParams, + params: ops.experimental.EvaluationParams, ) -> ops.items.Item[dict[str, Any]]: """Hit ``/evaluation-model`` and return raw evaluation answers.""" wire_questions: dict[str, dict[str, Any]] = {} for question_id, question in questions.items(): wire_question = question.model_dump(mode="json", by_alias=True) if ( - isinstance(question, ops.evaluation.BooleanQuestion) + isinstance(question, ops.experimental.BooleanQuestion) and question.criteria is None ): wire_question.pop("criteria") @@ -791,15 +791,15 @@ async def evaluate( self, client: gateway_client.GatewayClient, model: models.Model, - state: ops.evaluation.EvaluationInput, + state: ops.experimental.EvaluationInput, questions: Mapping[ str, - ops.evaluation.ChoiceQuestion - | ops.evaluation.ScoreQuestion - | ops.evaluation.BooleanQuestion, + ops.experimental.ChoiceQuestion + | ops.experimental.ScoreQuestion + | ops.experimental.BooleanQuestion, ], *, - params: ops.evaluation.EvaluationParams, + params: ops.experimental.EvaluationParams, provider: str, ) -> ops.items.Item[dict[str, Any]]: _ = provider diff --git a/src/ai/providers/base.py b/src/ai/providers/base.py index 016e0488..9b8ac2f6 100644 --- a/src/ai/providers/base.py +++ b/src/ai/providers/base.py @@ -27,13 +27,13 @@ from ..ops import ( audio, embeddings, - evaluation, images, items, reranking, transcriptions, videos, ) + from ..ops.experimental import evaluation from ..types import events from ..types import messages as messages_ from ..types import tools as tools_ diff --git a/tests/ops/experimental/__init__.py b/tests/ops/experimental/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/ops/test_evaluation.py b/tests/ops/experimental/test_evaluation.py similarity index 77% rename from tests/ops/test_evaluation.py rename to tests/ops/experimental/test_evaluation.py index a57b8002..84cf6b5a 100644 --- a/tests/ops/test_evaluation.py +++ b/tests/ops/experimental/test_evaluation.py @@ -1,4 +1,4 @@ -"""Tests for ``ai.ops.evaluation`` dispatch and input validation.""" +"""Tests for ``ai.ops.experimental.evaluate`` dispatch and input validation.""" from __future__ import annotations @@ -11,23 +11,25 @@ import ai from ai import models, ops -from .. import conftest +from ... import conftest type EvaluationQuestion = ( - ops.ChoiceQuestion | ops.ScoreQuestion | ops.BooleanQuestion + ops.experimental.ChoiceQuestion + | ops.experimental.ScoreQuestion + | ops.experimental.BooleanQuestion ) class Questions(pydantic.BaseModel): - department: ops.ChoiceQuestion - severity: ops.ScoreQuestion - refund: ops.BooleanQuestion + department: ops.experimental.ChoiceQuestion + severity: ops.experimental.ScoreQuestion + refund: ops.experimental.BooleanQuestion class Answers(pydantic.BaseModel): - department: ops.ChoiceAnswer - severity: ops.ScoreAnswer - refund: ops.BooleanAnswer + department: ops.experimental.ChoiceAnswer + severity: ops.experimental.ScoreAnswer + refund: ops.experimental.BooleanAnswer class EvaluationProvider(models.Provider): @@ -44,10 +46,10 @@ async def list_models(self) -> list[str]: async def evaluate( self, model: models.Model, - state: ops.EvaluationInput, + state: ops.experimental.EvaluationInput, questions: Mapping[str, EvaluationQuestion], *, - params: ops.EvaluationParams, + params: ops.experimental.EvaluationParams, ) -> ops.Item[dict[str, Any]]: assert state == {"message": "refund me"} assert set(questions) == {"department", "severity", "refund"} @@ -89,28 +91,28 @@ async def list_models(self) -> list[str]: async def evaluate( self, model: models.Model, - state: ops.EvaluationInput, + state: ops.experimental.EvaluationInput, questions: Mapping[str, EvaluationQuestion], *, - params: ops.EvaluationParams, + params: ops.experimental.EvaluationParams, ) -> ops.Item[dict[str, Any]]: return ops.Item(value=self.answers) def questions() -> Questions: return Questions( - department=ops.ChoiceQuestion( + department=ops.experimental.ChoiceQuestion( instructions="Which team should handle this?", criteria={ "billing": {"includes": ["charges", "refunds"]}, "support": None, }, ), - severity=ops.ScoreQuestion( + severity=ops.experimental.ScoreQuestion( instructions={"task": "Rate severity"}, criteria=["Cosmetic", "Workaround exists", "Blocking"], ), - refund=ops.BooleanQuestion( + refund=ops.experimental.BooleanQuestion( instructions="Is the customer requesting a refund?", criteria={"true": "Refund requested", "false": None}, ), @@ -122,12 +124,12 @@ async def test_evaluate_dispatch_and_span(recorder: conftest.Recorder) -> None: id="mock-evaluation-model", provider=EvaluationProvider() ) - result = await ops.experimental_evaluate( + result = await ops.experimental.evaluate( model, {"message": "refund me"}, questions(), output_type=Answers, - params=ops.EvaluationParams( + params=ops.experimental.EvaluationParams( provider_options={"gateway": {"zeroDataRetention": True}} ), ) @@ -156,11 +158,11 @@ async def test_evaluate_dynamic_questions() -> None: "refund": questions().refund, } - result = await ops.experimental_evaluate( + result = await ops.experimental.evaluate( model, {"message": "refund me"}, dynamic_questions, - params=ops.EvaluationParams( + params=ops.experimental.EvaluationParams( provider_options={"gateway": {"zeroDataRetention": True}} ), ) @@ -170,15 +172,17 @@ async def test_evaluate_dynamic_questions() -> None: ops.Item[ dict[ str, - ops.ChoiceAnswer | ops.ScoreAnswer | ops.BooleanAnswer, + ops.experimental.ChoiceAnswer + | ops.experimental.ScoreAnswer + | ops.experimental.BooleanAnswer, ] ], ) - assert isinstance(result.value["department"], ops.ChoiceAnswer) + assert isinstance(result.value["department"], ops.experimental.ChoiceAnswer) assert result.value["department"].choice == "billing" - assert isinstance(result.value["severity"], ops.ScoreAnswer) + assert isinstance(result.value["severity"], ops.experimental.ScoreAnswer) assert result.value["severity"].score == 1.5 - assert isinstance(result.value["refund"], ops.BooleanAnswer) + assert isinstance(result.value["refund"], ops.experimental.BooleanAnswer) assert result.value["refund"].probability == 0.98 assert result.metadata == {"rounding": {"probabilityDecimals": 2}} @@ -188,17 +192,19 @@ async def test_evaluate_raises_not_implemented() -> None: model = ai.Model(id="evaluation-test", provider=provider) class BooleanQuestions(pydantic.BaseModel): - answer: ops.BooleanQuestion + answer: ops.experimental.BooleanQuestion class BooleanAnswers(pydantic.BaseModel): - answer: ops.BooleanAnswer + answer: ops.experimental.BooleanAnswer with pytest.raises(NotImplementedError, match="evaluate"): - await ops.experimental_evaluate( + await ops.experimental.evaluate( model, "state", BooleanQuestions( - answer=ops.BooleanQuestion(instructions="Is this valid?") + answer=ops.experimental.BooleanQuestion( + instructions="Is this valid?" + ) ), output_type=BooleanAnswers, ) @@ -206,26 +212,30 @@ class BooleanAnswers(pydantic.BaseModel): def test_choice_question_requires_criteria() -> None: with pytest.raises(pydantic.ValidationError): - ops.ChoiceQuestion(instructions="Choose", criteria={}) + ops.experimental.ChoiceQuestion(instructions="Choose", criteria={}) def test_score_question_requires_two_levels() -> None: with pytest.raises(pydantic.ValidationError): - ops.ScoreQuestion(instructions="Score", criteria=["only one"]) + ops.experimental.ScoreQuestion( + instructions="Score", criteria=["only one"] + ) def test_boolean_question_rejects_unknown_criteria() -> None: with pytest.raises(pydantic.ValidationError): - ops.BooleanQuestion( + ops.experimental.BooleanQuestion( instructions="Decide", - criteria=cast("ops.BooleanCriteria", {"maybe": "Maybe"}), + criteria=cast( + "ops.experimental.BooleanCriteria", {"maybe": "Maybe"} + ), ) def test_question_rejects_non_json_instructions() -> None: with pytest.raises(pydantic.ValidationError): - ops.BooleanQuestion( - instructions=cast("ops.EvaluationInput", object()), + ops.experimental.BooleanQuestion( + instructions=cast("ops.experimental.EvaluationInput", object()), ) @@ -236,9 +246,9 @@ async def test_evaluate_validates_state(state: Any) -> None: ) with pytest.raises(pydantic.ValidationError): - await ops.experimental_evaluate( + await ops.experimental.evaluate( model, - cast("ops.EvaluationInput", state), + cast("ops.experimental.EvaluationInput", state), questions(), output_type=Answers, ) @@ -250,7 +260,7 @@ async def test_evaluate_requires_question_values() -> None: ) with pytest.raises(TypeError, match="must contain a question"): - await ops.experimental_evaluate( + await ops.experimental.evaluate( model, "state", cast( @@ -272,7 +282,7 @@ class EmptyAnswers(pydantic.BaseModel): ) with pytest.raises(ValueError, match="questions must not be empty"): - await ops.experimental_evaluate( + await ops.experimental.evaluate( model, "state", EmptyQuestions(), @@ -281,14 +291,14 @@ class EmptyAnswers(pydantic.BaseModel): empty: dict[str, EvaluationQuestion] = {} with pytest.raises(ValueError, match="questions must not be empty"): - await ops.experimental_evaluate(model, "state", empty) + await ops.experimental.evaluate(model, "state", empty) async def test_evaluate_rejects_mixed_modes() -> None: model = models.Model( id="mock-evaluation-model", provider=EvaluationProvider() ) - evaluate = cast("Any", ops.experimental_evaluate) + evaluate = cast("Any", ops.experimental.evaluate) with pytest.raises(TypeError, match="required for Pydantic"): await evaluate(model, "state", questions()) @@ -307,14 +317,14 @@ class InvalidQuestions(pydantic.BaseModel): answer: str class BooleanAnswers(pydantic.BaseModel): - answer: ops.BooleanAnswer + answer: ops.experimental.BooleanAnswer model = models.Model( id="mock-evaluation-model", provider=EvaluationProvider() ) with pytest.raises(TypeError, match="must contain a question"): - await ops.experimental_evaluate( + await ops.experimental.evaluate( model, "state", InvalidQuestions(answer="invalid"), @@ -324,14 +334,14 @@ class BooleanAnswers(pydantic.BaseModel): async def test_evaluate_requires_matching_output_fields() -> None: class MissingAnswers(pydantic.BaseModel): - department: ops.ChoiceAnswer + department: ops.experimental.ChoiceAnswer model = models.Model( id="mock-evaluation-model", provider=EvaluationProvider() ) with pytest.raises(TypeError, match="fields must match"): - await ops.experimental_evaluate( + await ops.experimental.evaluate( model, "state", questions(), @@ -341,16 +351,16 @@ class MissingAnswers(pydantic.BaseModel): async def test_evaluate_requires_matching_answer_types() -> None: class WrongAnswers(pydantic.BaseModel): - department: ops.ChoiceAnswer - severity: ops.BooleanAnswer - refund: ops.BooleanAnswer + department: ops.experimental.ChoiceAnswer + severity: ops.experimental.BooleanAnswer + refund: ops.experimental.BooleanAnswer model = models.Model( id="mock-evaluation-model", provider=EvaluationProvider() ) with pytest.raises(TypeError, match="ScoreAnswer"): - await ops.experimental_evaluate( + await ops.experimental.evaluate( model, "state", questions(), @@ -371,7 +381,7 @@ async def test_evaluate_validates_provider_output() -> None: ) with pytest.raises(pydantic.ValidationError): - await ops.experimental_evaluate( + await ops.experimental.evaluate( model, "state", questions(), @@ -384,7 +394,7 @@ async def test_evaluate_validates_provider_output() -> None: "refund": questions().refund, } with pytest.raises(pydantic.ValidationError): - await ops.experimental_evaluate(model, "state", dynamic_questions) + await ops.experimental.evaluate(model, "state", dynamic_questions) async def test_evaluate_requires_matching_answer_fields() -> None: @@ -394,7 +404,7 @@ async def test_evaluate_requires_matching_answer_fields() -> None: ) with pytest.raises(ValueError, match="answer fields must match"): - await ops.experimental_evaluate( + await ops.experimental.evaluate( model, "state", questions(), @@ -410,9 +420,9 @@ async def test_evaluate_rejects_cyclic_state() -> None: ) with pytest.raises(pydantic.ValidationError): - await ops.experimental_evaluate( + await ops.experimental.evaluate( model, - cast("ops.EvaluationInput", state), + cast("ops.experimental.EvaluationInput", state), questions(), output_type=Answers, ) diff --git a/tests/providers/ai_gateway/protocol/test_v4_evaluation.py b/tests/providers/ai_gateway/protocol/test_v4_evaluation.py index 06db0e6b..c9f64c31 100644 --- a/tests/providers/ai_gateway/protocol/test_v4_evaluation.py +++ b/tests/providers/ai_gateway/protocol/test_v4_evaluation.py @@ -18,36 +18,36 @@ class Questions(pydantic.BaseModel): - department: ops.ChoiceQuestion - severity: ops.ScoreQuestion - refund: ops.BooleanQuestion + department: ops.experimental.ChoiceQuestion + severity: ops.experimental.ScoreQuestion + refund: ops.experimental.BooleanQuestion class Answers(pydantic.BaseModel): - department: ops.ChoiceAnswer - severity: ops.ScoreAnswer - refund: ops.BooleanAnswer + department: ops.experimental.ChoiceAnswer + severity: ops.experimental.ScoreAnswer + refund: ops.experimental.BooleanAnswer class BooleanQuestions(pydantic.BaseModel): - answer: ops.BooleanQuestion + answer: ops.experimental.BooleanQuestion class BooleanAnswers(pydantic.BaseModel): - answer: ops.BooleanAnswer + answer: ops.experimental.BooleanAnswer def questions() -> Questions: return Questions( - department=ops.ChoiceQuestion( + department=ops.experimental.ChoiceQuestion( instructions="Which team should handle this?", criteria={"billing": "Charges", "support": "Other requests"}, ), - severity=ops.ScoreQuestion( + severity=ops.experimental.ScoreQuestion( instructions="How severe is this?", criteria=["Cosmetic", "Workaround exists", "Blocking"], ), - refund=ops.BooleanQuestion( + refund=ops.experimental.BooleanQuestion( instructions="Is a refund requested?", criteria={"true": "A refund is requested", "false": None}, ), @@ -95,7 +95,7 @@ def handler(request: httpx.Request) -> httpx.Response: }, ) - result = await ops.experimental_evaluate( + result = await ops.experimental.evaluate( mock_model( httpx.MockTransport(handler), api_key="sk-test", @@ -104,7 +104,7 @@ def handler(request: httpx.Request) -> httpx.Response: {"message": "Please refund the duplicate charge."}, questions(), output_type=Answers, - params=ops.EvaluationParams( + params=ops.experimental.EvaluationParams( provider_options={ "gateway": { "zeroDataRetention": True, @@ -206,18 +206,18 @@ def handler(request: httpx.Request) -> httpx.Response: }, ) - result = await ops.experimental_evaluate( + result = await ops.experimental.evaluate( mock_model(httpx.MockTransport(handler), model_id=_MODEL_ID), "state", { - "answer": ops.ChoiceQuestion( + "answer": ops.experimental.ChoiceQuestion( instructions="Choose", criteria={"yes": None, "no": None}, ) }, ) - assert isinstance(result.value["answer"], ops.ChoiceAnswer) + assert isinstance(result.value["answer"], ops.experimental.ChoiceAnswer) assert result.value["answer"].choice == "yes" assert result.warnings == [ ops.Warning(kind="unsupported", feature="providerOptions.test"), @@ -240,11 +240,13 @@ def handler(request: httpx.Request) -> httpx.Response: ) with pytest.raises(ai.ProviderAuthenticationError): - await ops.experimental_evaluate( + await ops.experimental.evaluate( mock_model(httpx.MockTransport(handler), model_id=_MODEL_ID), "state", BooleanQuestions( - answer=ops.BooleanQuestion(instructions="Is this true?") + answer=ops.experimental.BooleanQuestion( + instructions="Is this true?" + ) ), output_type=BooleanAnswers, )