diff --git a/client.go b/client.go index a437d51..6c499a5 100644 --- a/client.go +++ b/client.go @@ -39,10 +39,23 @@ type Message struct { // Request is a single completion request. type Request struct { - Model string - Messages []Message - MaxTokens int - Temperature float64 + Model string + Messages []Message + MaxTokens int + + // Temperature is the sampling temperature. nil means "leave it to the + // provider", which is NOT the same as zero — provider defaults sit + // around 1.0. Pass Float64(0) for deterministic sampling; a bare 0 + // would be indistinguishable from unset. + Temperature *float64 + + // TopP is nucleus-sampling cutoff. nil leaves it unset. Same + // zero-is-meaningful reasoning as Temperature. + TopP *float64 + + // Seed requests deterministic sampling where the provider supports + // it. nil leaves it unset. Best-effort — no provider guarantees it. + Seed *int // JSONMode asks the provider to return a JSON object that conforms to // JSONSchema. Providers that don't support structured outputs natively @@ -89,6 +102,12 @@ type Response struct { Model string FinishReason string + // ReasoningContent is the model's thinking / reasoning trace when the + // provider exposes it separately from the answer (Claude extended + // thinking, OpenAI o-series). Empty when the model didn't reason or + // the provider folds it into Content. + ReasoningContent string + // Usage is the normalized accounting for this call. Usage Usage @@ -102,6 +121,18 @@ type Response struct { ToolCalls []ToolCall } +// Ptr returns a pointer to v. It exists so optional Request fields can be +// set inline: Temperature: llmgate.Ptr(0.7). +func Ptr[T any](v T) *T { return &v } + +// Float64 returns a pointer to v. Prefer it over Ptr for the sampling +// fields, where the untyped-constant form reads better: +// Temperature: llmgate.Float64(0). +func Float64(v float64) *float64 { return &v } + +// Int returns a pointer to v, for Request.Seed. +func Int(v int) *int { return &v } + // Client is the provider-agnostic contract. type Client interface { // Complete runs a single completion. diff --git a/examples/smoke/main.go b/examples/smoke/main.go index 787f7ad..1a36291 100644 --- a/examples/smoke/main.go +++ b/examples/smoke/main.go @@ -38,7 +38,7 @@ func main() { {Role: llmgate.RoleUser, Content: "In one sentence: what is vectorless retrieval?"}, }, MaxTokens: 1024, - Temperature: 0.2, + Temperature: llmgate.Float64(0.2), }) if err != nil { fmt.Fprintln(os.Stderr, "complete:", err) diff --git a/go.mod b/go.mod index 68cf79f..017543c 100644 --- a/go.mod +++ b/go.mod @@ -2,45 +2,48 @@ module github.com/hallelx2/llmgate go 1.25.0 -require github.com/tmc/langchaingo v0.1.14 +require ( + github.com/pkoukk/tiktoken-go v0.1.6 + github.com/tmc/langchaingo v0.1.14 +) require ( - cloud.google.com/go v0.116.0 // indirect + cloud.google.com/go v0.123.0 // indirect cloud.google.com/go/ai v0.7.0 // indirect - cloud.google.com/go/aiplatform v1.69.0 // indirect - cloud.google.com/go/auth v0.14.0 // indirect - cloud.google.com/go/auth/oauth2adapt v0.2.7 // indirect - cloud.google.com/go/compute/metadata v0.6.0 // indirect - cloud.google.com/go/iam v1.2.2 // indirect - cloud.google.com/go/longrunning v0.6.2 // indirect + cloud.google.com/go/aiplatform v1.114.0 // indirect + cloud.google.com/go/auth v0.18.2 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect + cloud.google.com/go/compute/metadata v0.9.0 // indirect + cloud.google.com/go/iam v1.5.3 // indirect + cloud.google.com/go/longrunning v0.8.0 // indirect cloud.google.com/go/vertexai v0.12.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/dlclark/regexp2 v1.10.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/logr v1.4.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/google/generative-ai-go v0.15.1 // indirect github.com/google/s2a-go v0.1.9 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect - github.com/googleapis/gax-go/v2 v2.14.1 // indirect - github.com/pkoukk/tiktoken-go v0.1.6 // indirect - go.opentelemetry.io/auto/sdk v1.1.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.11 // indirect + github.com/googleapis/gax-go/v2 v2.17.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect - go.opentelemetry.io/otel v1.36.0 // indirect - go.opentelemetry.io/otel/metric v1.36.0 // indirect - go.opentelemetry.io/otel/trace v1.36.0 // indirect - golang.org/x/crypto v0.41.0 // indirect - golang.org/x/net v0.43.0 // indirect - golang.org/x/oauth2 v0.30.0 // indirect - golang.org/x/sync v0.16.0 // indirect - golang.org/x/sys v0.35.0 // indirect - golang.org/x/text v0.28.0 // indirect - golang.org/x/time v0.9.0 // indirect - google.golang.org/api v0.218.0 // indirect - google.golang.org/genproto v0.0.0-20241118233622-e639e219e697 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250122153221-138b5a5a4fd4 // indirect - google.golang.org/grpc v1.70.0 // indirect - google.golang.org/protobuf v1.36.3 // indirect + go.opentelemetry.io/otel v1.44.0 // indirect + go.opentelemetry.io/otel/metric v1.44.0 // indirect + go.opentelemetry.io/otel/trace v1.44.0 // indirect + golang.org/x/crypto v0.54.0 // indirect + golang.org/x/net v0.57.0 // indirect + golang.org/x/oauth2 v0.36.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.40.0 // indirect + golang.org/x/time v0.14.0 // indirect + google.golang.org/api v0.264.0 // indirect + google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260729162451-8efbd57d26e0 // indirect + google.golang.org/grpc v1.83.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect ) diff --git a/go.sum b/go.sum index 52d39d2..afa4526 100644 --- a/go.sum +++ b/go.sum @@ -1,30 +1,39 @@ -cloud.google.com/go v0.116.0 h1:B3fRrSDkLRt5qSHWe40ERJvhvnQwdZiHu0bJOpldweE= -cloud.google.com/go v0.116.0/go.mod h1:cEPSRWPzZEswwdr9BxE6ChEn01dWlTaF05LiC2Xs70U= +cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= +cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= cloud.google.com/go/ai v0.7.0 h1:P6+b5p4gXlza5E+u7uvcgYlzZ7103ACg70YdZeC6oGE= cloud.google.com/go/ai v0.7.0/go.mod h1:7ozuEcraovh4ABsPbrec3o4LmFl9HigNI3D5haxYeQo= -cloud.google.com/go/aiplatform v1.69.0 h1:XvBzK8e6/6ufbi/i129Vmn/gVqFwbNPmRQ89K+MGlgc= -cloud.google.com/go/aiplatform v1.69.0/go.mod h1:nUsIqzS3khlnWvpjfJbP+2+h+VrFyYsTm7RNCAViiY8= -cloud.google.com/go/auth v0.14.0 h1:A5C4dKV/Spdvxcl0ggWwWEzzP7AZMJSEIgrkngwhGYM= -cloud.google.com/go/auth v0.14.0/go.mod h1:CYsoRL1PdiDuqeQpZE0bP2pnPrGqFcOkI0nldEQis+A= -cloud.google.com/go/auth/oauth2adapt v0.2.7 h1:/Lc7xODdqcEw8IrZ9SvwnlLX6j9FHQM74z6cBk9Rw6M= -cloud.google.com/go/auth/oauth2adapt v0.2.7/go.mod h1:NTbTTzfvPl1Y3V1nPpOgl2w6d/FjO7NNUQaWSox6ZMc= -cloud.google.com/go/compute/metadata v0.6.0 h1:A6hENjEsCDtC1k8byVsgwvVcioamEHvZ4j01OwKxG9I= -cloud.google.com/go/compute/metadata v0.6.0/go.mod h1:FjyFAW1MW0C203CEOMDTu3Dk1FlqW3Rga40jzHL4hfg= -cloud.google.com/go/iam v1.2.2 h1:ozUSofHUGf/F4tCNy/mu9tHLTaxZFLOUiKzjcgWHGIA= -cloud.google.com/go/iam v1.2.2/go.mod h1:0Ys8ccaZHdI1dEUilwzqng/6ps2YB6vRsjIe00/+6JY= -cloud.google.com/go/longrunning v0.6.2 h1:xjDfh1pQcWPEvnfjZmwjKQEcHnpz6lHjfy7Fo0MK+hc= -cloud.google.com/go/longrunning v0.6.2/go.mod h1:k/vIs83RN4bE3YCswdXC5PFfWVILjm3hpEUlSko4PiI= +cloud.google.com/go/aiplatform v1.114.0 h1:TCrSLci+NFEAx0PZMv8btGe5j68RivArmDJbBLIc/3o= +cloud.google.com/go/aiplatform v1.114.0/go.mod h1:W5yMrpIuHG/CSK8iF7XnwIfCJu6dcLRQ0cTqGR5vwwE= +cloud.google.com/go/auth v0.18.2 h1:+Nbt5Ev0xEqxlNjd6c+yYUeosQ5TtEUaNcN/3FozlaM= +cloud.google.com/go/auth v0.18.2/go.mod h1:xD+oY7gcahcu7G2SG2DsBerfFxgPAJz17zz2joOFF3M= +cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= +cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= +cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= +cloud.google.com/go/iam v1.5.3 h1:+vMINPiDF2ognBJ97ABAYYwRgsaqxPbQDlMnbHMjolc= +cloud.google.com/go/iam v1.5.3/go.mod h1:MR3v9oLkZCTlaqljW6Eb2d3HGDGK5/bDv93jhfISFvU= +cloud.google.com/go/longrunning v0.8.0 h1:LiKK77J3bx5gDLi4SMViHixjD2ohlkwBi+mKA7EhfW8= +cloud.google.com/go/longrunning v0.8.0/go.mod h1:UmErU2Onzi+fKDg2gR7dusz11Pe26aknR4kHmJJqIfk= cloud.google.com/go/vertexai v0.12.0 h1:zTadEo/CtsoyRXNx3uGCncoWAP1H2HakGqwznt+iMo8= cloud.google.com/go/vertexai v0.12.0/go.mod h1:8u+d0TsvBfAAd2x5R6GMgbYhsLgo3J7lmP4bR8g2ig8= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0= github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= +github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= +github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= +github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= +github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= -github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= @@ -37,62 +46,66 @@ github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.3.4 h1:XYIDZApgAnrN1c855gTgghdIA6Stxb52D5RnLI1SLyw= -github.com/googleapis/enterprise-certificate-proxy v0.3.4/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA= -github.com/googleapis/gax-go/v2 v2.14.1 h1:hb0FFeiPaQskmvakKu5EbCbpntQn48jyHuvrkurSS/Q= -github.com/googleapis/gax-go/v2 v2.14.1/go.mod h1:Hb/NubMaVM88SrNkvl8X/o8XWwDJEPqouaLeN2IUxoA= +github.com/googleapis/enterprise-certificate-proxy v0.3.11 h1:vAe81Msw+8tKUxi2Dqh/NZMz7475yUvmRIkXr4oN2ao= +github.com/googleapis/enterprise-certificate-proxy v0.3.11/go.mod h1:RFV7MUdlb7AgEq2v7FmMCfeSMCllAzWxFgRdusoGks8= +github.com/googleapis/gax-go/v2 v2.17.0 h1:RksgfBpxqff0EZkDWYuz9q/uWsTVz+kf43LsZ1J6SMc= +github.com/googleapis/gax-go/v2 v2.17.0/go.mod h1:mzaqghpQp4JDh3HvADwrat+6M3MOIDp5YKHhb9PAgDY= github.com/pkoukk/tiktoken-go v0.1.6 h1:JF0TlJzhTbrI30wCvFuiw6FzP2+/bR+FIxUdgEAcUsw= github.com/pkoukk/tiktoken-go v0.1.6/go.mod h1:9NiV+i9mJKGj1rYOT+njbv+ZwA/zJxYdewGl6qVatpg= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/tmc/langchaingo v0.1.14 h1:o1qWBPigAIuFvrG6cjTFo0cZPFEZ47ZqpOYMjM15yZc= github.com/tmc/langchaingo v0.1.14/go.mod h1:aKKYXYoqhIDEv7WKdpnnCLRaqXic69cX9MnDUk72378= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 h1:r6I7RJCN86bpD/FQwedZ0vSixDpwuWREjW9oRMsmqDc= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0/go.mod h1:B9yO6b04uB80CzjedvewuqDhxJxi11s7/GtiGa8bAjI= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 h1:q4XOmH/0opmeuJtPsbFNivyl7bCt7yRBbeEm2sC/XtQ= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0/go.mod h1:snMWehoOh2wsEwnvvwtDyFCxVeDAODenXHtn5vzrKjo= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= -go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg= -go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E= -go.opentelemetry.io/otel/metric v1.36.0 h1:MoWPKVhQvJ+eeXWHFBOPoBOi20jh6Iq2CcCREuTYufE= -go.opentelemetry.io/otel/metric v1.36.0/go.mod h1:zC7Ks+yeyJt4xig9DEw9kuUFe5C3zLbVjV2PzT6qzbs= -go.opentelemetry.io/otel/sdk v1.36.0 h1:b6SYIuLRs88ztox4EyrvRti80uXIFy+Sqzoh9kFULbs= -go.opentelemetry.io/otel/sdk v1.36.0/go.mod h1:+lC+mTgD+MUWfjJubi2vvXWcVxyr9rmlshZni72pXeY= -go.opentelemetry.io/otel/sdk/metric v1.36.0 h1:r0ntwwGosWGaa0CrSt8cuNuTcccMXERFwHX4dThiPis= -go.opentelemetry.io/otel/sdk/metric v1.36.0/go.mod h1:qTNOhFDfKRwX0yXOqJYegL5WRaW376QbB7P4Pb0qva4= -go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w= -go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA= -golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= -golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= -golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= -golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= -golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= -golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= -golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= -golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= -golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= -golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= -golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= -golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= -golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= -google.golang.org/api v0.218.0 h1:x6JCjEWeZ9PFCRe9z0FBrNwj7pB7DOAqT35N+IPnAUA= -google.golang.org/api v0.218.0/go.mod h1:5VGHBAkxrA/8EFjLVEYmMUJ8/8+gWWQ3s4cFH0FxG2M= -google.golang.org/genproto v0.0.0-20241118233622-e639e219e697 h1:ToEetK57OidYuqD4Q5w+vfEnPvPpuTwedCNVohYJfNk= -google.golang.org/genproto v0.0.0-20241118233622-e639e219e697/go.mod h1:JJrvXBWRZaFMxBufik1a4RpFw4HhgVtBBWQeQgUj2cc= -google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 h1:CkkIfIt50+lT6NHAVoRYEyAvQGFM7xEwXUUywFvEb3Q= -google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576/go.mod h1:1R3kvZ1dtP3+4p4d3G8uJ8rFk/fWlScl38vanWACI08= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250122153221-138b5a5a4fd4 h1:yrTuav+chrF0zF/joFGICKTzYv7mh/gr9AgEXrVU8ao= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250122153221-138b5a5a4fd4/go.mod h1:+2Yz8+CLJbIfL9z73EW45avw8Lmge3xVElCP9zEKi50= -google.golang.org/grpc v1.70.0 h1:pWFv03aZoHzlRKHWicjsZytKAiYCtNS0dHbXnIdq7jQ= -google.golang.org/grpc v1.70.0/go.mod h1:ofIJqVKDXx/JiXrwr2IG4/zwdH9txy3IlF40RmcJSQw= -google.golang.org/protobuf v1.36.3 h1:82DV7MYdb8anAVi3qge1wSnMDrnKK7ebr+I0hHRN1BU= -google.golang.org/protobuf v1.36.3/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= +golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= +golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= +golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/api v0.264.0 h1:+Fo3DQXBK8gLdf8rFZ3uLu39JpOnhvzJrLMQSoSYZJM= +google.golang.org/api v0.264.0/go.mod h1:fAU1xtNNisHgOF5JooAs8rRaTkl2rT3uaoNGo9NS3R8= +google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 h1:VQZ/yAbAtjkHgH80teYd2em3xtIkkHd7ZhqfH2N9CsM= +google.golang.org/genproto v0.0.0-20260128011058-8636f8732409/go.mod h1:rxKD3IEILWEu3P44seeNOAwZN4SaoKaQ/2eTg4mM6EM= +google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8= +google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260729162451-8efbd57d26e0 h1:mJiOtnGp0k/BcSgdu03G2NwnscCfCH+h2QKUBZr18KI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260729162451-8efbd57d26e0/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.83.0 h1:JeNZEKJFbQxArAMl+hiytHauacDNqJUllNfmIMmpqnQ= +google.golang.org/grpc v1.83.0/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/internal/adapter/adapter.go b/internal/adapter/adapter.go index 54ee2e9..e1eb90f 100644 --- a/internal/adapter/adapter.go +++ b/internal/adapter/adapter.go @@ -58,8 +58,17 @@ func (a *Adapter) Complete(ctx context.Context, req llmgate.Request) (*llmgate.R if req.MaxTokens > 0 { opts = append(opts, llms.WithMaxTokens(req.MaxTokens)) } - if req.Temperature != 0 { - opts = append(opts, llms.WithTemperature(req.Temperature)) + // Nil, not zero, means "unset" for the sampling knobs — a caller asking + // for temperature 0 wants determinism, and dropping it silently leaves + // the request at the provider default (~1.0). + if req.Temperature != nil { + opts = append(opts, llms.WithTemperature(*req.Temperature)) + } + if req.TopP != nil { + opts = append(opts, llms.WithTopP(*req.TopP)) + } + if req.Seed != nil { + opts = append(opts, llms.WithSeed(*req.Seed)) } if len(req.Tools) > 0 { opts = append(opts, llms.WithTools(toLangchainTools(req.Tools))) @@ -84,22 +93,23 @@ func (a *Adapter) Complete(ctx context.Context, req llmgate.Request) (*llmgate.R return nil, fmt.Errorf("%s: empty response", a.provider) } - choice := resp.Choices[0] + folded := foldChoices(resp.Choices) model := req.Model if model == "" { model = a.model } out := &llmgate.Response{ - Content: choice.Content, - Model: model, - FinishReason: choice.StopReason, - ToolCalls: fromLangchainToolCalls(choice), + Content: folded.content, + ReasoningContent: folded.reasoning, + Model: model, + FinishReason: folded.finishReason, + ToolCalls: folded.toolCalls, } // Token usage is reported provider-by-provider under slightly different // keys. Try the common ones. - in := getInt(choice.GenerationInfo, "InputTokens", "PromptTokens", "input_tokens", "prompt_tokens") - outTok := getInt(choice.GenerationInfo, "OutputTokens", "CompletionTokens", "output_tokens", "completion_tokens") + in := getInt(folded.genInfo, "InputTokens", "PromptTokens", "input_tokens", "prompt_tokens") + outTok := getInt(folded.genInfo, "OutputTokens", "CompletionTokens", "output_tokens", "completion_tokens") out.InputTokens = in out.OutputTokens = outTok cost, priced := pricing.ComputeWithOK(model, in, outTok) @@ -114,6 +124,96 @@ func (a *Adapter) Complete(ctx context.Context, req llmgate.Request) (*llmgate.R return out, nil } +// folded is one logical reply assembled from every choice a provider +// returned. +type folded struct { + content string + reasoning string + toolCalls []llmgate.ToolCall + finishReason string + genInfo map[string]any +} + +// usageKeys are every key any provider uses to report token counts. A +// choice carrying none of them has no usage to contribute. +var usageKeys = []string{ + "InputTokens", "PromptTokens", "input_tokens", "prompt_tokens", + "OutputTokens", "CompletionTokens", "output_tokens", "completion_tokens", +} + +// foldChoices collapses a provider response into a single reply. +// +// langchaingo's Anthropic adapter returns one ContentChoice per *content +// block*, not per completion candidate — a reply of [thinking, text] or +// [text, tool_use] arrives as two choices. Reading Choices[0] alone +// therefore returned an empty answer whenever the model thought first, +// and silently dropped tool calls whenever it narrated before calling. +// +// Usage is taken from the first choice that reports any, never summed: +// every block carries a copy of the same response-level usage, so adding +// them up would multiply the bill by the block count. +func foldChoices(choices []*llms.ContentChoice) folded { + var f folded + var content, reasoning strings.Builder + seenReasoning := map[string]bool{} + + appendReasoning := func(s string) { + // Anthropic repeats ThinkingContent on the text block as well as + // the thinking block; dedupe so it isn't emitted twice. + if s == "" || seenReasoning[s] { + return + } + seenReasoning[s] = true + reasoning.WriteString(s) + } + + for _, c := range choices { + if c == nil { + continue + } + content.WriteString(c.Content) + appendReasoning(c.ReasoningContent) + if s, ok := c.GenerationInfo["ThinkingContent"].(string); ok { + appendReasoning(s) + } + f.toolCalls = append(f.toolCalls, fromLangchainToolCalls(c)...) + if c.StopReason != "" { + f.finishReason = c.StopReason + } + if f.genInfo == nil && hasUsage(c.GenerationInfo) { + f.genInfo = c.GenerationInfo + } + } + + // No choice reported usage — keep the first non-nil map so any other + // metadata is still reachable downstream. + if f.genInfo == nil { + for _, c := range choices { + if c != nil && c.GenerationInfo != nil { + f.genInfo = c.GenerationInfo + break + } + } + } + + f.content = content.String() + f.reasoning = reasoning.String() + return f +} + +// hasUsage reports whether m carries any recognised token-count key. +func hasUsage(m map[string]any) bool { + if m == nil { + return false + } + for _, k := range usageKeys { + if _, ok := m[k]; ok { + return true + } + } + return false +} + // toLangchainTools converts our provider-agnostic tool declarations into // langchaingo's []llms.Tool. The InputSchema bytes are unmarshalled into // the generic structure langchaingo forwards as the function parameters; diff --git a/internal/adapter/adapter_fold_test.go b/internal/adapter/adapter_fold_test.go new file mode 100644 index 0000000..af85a22 --- /dev/null +++ b/internal/adapter/adapter_fold_test.go @@ -0,0 +1,246 @@ +package adapter + +import ( + "context" + "testing" + + "github.com/tmc/langchaingo/llms" + + "github.com/hallelx2/llmgate" +) + +// unsetTemp is a sentinel written into the fake's CallOptions before the +// adapter applies its options. llms.CallOptions zero-values Temperature to +// 0, so without a sentinel "never set" and "explicitly set to 0" are +// indistinguishable — which is the exact bug under test. +const unsetTemp = -999.0 + +// completeWith runs one Complete against a canned single-text response and +// returns the CallOptions the adapter actually built. +func completeWith(t *testing.T, req llmgate.Request) llms.CallOptions { + t.Helper() + fm := newFake(textResponse("ok")) + fm.gotOpts.Temperature = unsetTemp + fm.gotOpts.TopP = unsetTemp + + a := NewAdapter(fm, llmgate.ProviderOpenAI, "gpt-4o-mini", true) + if len(req.Messages) == 0 { + req.Messages = []llmgate.Message{{Role: llmgate.RoleUser, Content: "hi"}} + } + if _, err := a.Complete(context.Background(), req); err != nil { + t.Fatalf("Complete: %v", err) + } + return fm.gotOpts +} + +// TestTemperatureZeroReachesProvider is the regression test for HAL-524: +// every vectorless call asks for temperature 0 and the old `!= 0` guard +// dropped all of them, silently running the whole engine at the provider +// default. +func TestTemperatureZeroReachesProvider(t *testing.T) { + got := completeWith(t, llmgate.Request{Temperature: llmgate.Float64(0)}) + if got.Temperature != 0 { + t.Fatalf("Temperature = %v, want 0 — an explicit zero must reach the provider", got.Temperature) + } +} + +func TestTemperatureUnsetIsNotForwarded(t *testing.T) { + got := completeWith(t, llmgate.Request{}) + if got.Temperature != unsetTemp { + t.Fatalf("Temperature = %v, want untouched (%v) — nil must leave the provider default alone", got.Temperature, unsetTemp) + } +} + +func TestTemperatureNonZeroForwarded(t *testing.T) { + got := completeWith(t, llmgate.Request{Temperature: llmgate.Float64(0.7)}) + if got.Temperature != 0.7 { + t.Fatalf("Temperature = %v, want 0.7", got.Temperature) + } +} + +func TestTopPAndSeedOptional(t *testing.T) { + unset := completeWith(t, llmgate.Request{}) + if unset.TopP != unsetTemp { + t.Fatalf("TopP = %v, want untouched", unset.TopP) + } + if unset.Seed != 0 { + t.Fatalf("Seed = %v, want 0 when unset", unset.Seed) + } + + set := completeWith(t, llmgate.Request{TopP: llmgate.Float64(0), Seed: llmgate.Int(42)}) + if set.TopP != 0 { + t.Fatalf("TopP = %v, want explicit 0", set.TopP) + } + if set.Seed != 42 { + t.Fatalf("Seed = %v, want 42", set.Seed) + } +} + +// --- choice folding (HAL-525) ------------------------------------------- + +// anthropicUsage mirrors the GenerationInfo langchaingo's Anthropic adapter +// stamps onto *every* content block of a single response. +func anthropicUsage() map[string]any { + return map[string]any{ + "InputTokens": 1000, + "OutputTokens": 200, + } +} + +func textBlock(s string) *llms.ContentChoice { + return &llms.ContentChoice{ + Content: s, + StopReason: "end_turn", + GenerationInfo: anthropicUsage(), + } +} + +func thinkingBlock(thought string) *llms.ContentChoice { + gi := anthropicUsage() + gi["ThinkingContent"] = thought + return &llms.ContentChoice{ + Content: "", // langchaingo: "Thinking content is not included in output" + StopReason: "end_turn", + GenerationInfo: gi, + } +} + +func toolBlock(id, name, args string) *llms.ContentChoice { + return &llms.ContentChoice{ + ToolCalls: []llms.ToolCall{{ + ID: id, + Type: "function", + FunctionCall: &llms.FunctionCall{Name: name, Arguments: args}, + }}, + StopReason: "tool_use", + GenerationInfo: anthropicUsage(), + } +} + +// TestFoldChoices covers the shapes langchaingo's Anthropic adapter +// produces. It builds one ContentChoice per *content block*, so reading +// Choices[0] returned an empty answer whenever the model thought first and +// dropped tool calls whenever it narrated before calling. +func TestFoldChoices(t *testing.T) { + tests := []struct { + name string + choices []*llms.ContentChoice + wantContent string + wantReasoning string + wantToolCalls []string // tool names, in order + }{ + { + name: "text only", + choices: []*llms.ContentChoice{textBlock("the answer")}, + wantContent: "the answer", + }, + { + name: "thinking then text", + choices: []*llms.ContentChoice{thinkingBlock("let me consider"), textBlock("the answer")}, + wantContent: "the answer", + wantReasoning: "let me consider", + }, + { + name: "text then tool use", + choices: []*llms.ContentChoice{textBlock("looking that up"), toolBlock("t1", "get_pages", `{"start":1}`)}, + wantContent: "looking that up", + wantToolCalls: []string{"get_pages"}, + }, + { + name: "two tool calls", + choices: []*llms.ContentChoice{toolBlock("t1", "get_pages", `{}`), toolBlock("t2", "search", `{}`)}, + wantToolCalls: []string{"get_pages", "search"}, + }, + { + name: "thinking, text and tool use", + choices: []*llms.ContentChoice{ + thinkingBlock("plan it out"), + textBlock("fetching"), + toolBlock("t1", "get_pages", `{}`), + }, + wantContent: "fetching", + wantReasoning: "plan it out", + wantToolCalls: []string{"get_pages"}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + fm := newFake(&llms.ContentResponse{Choices: tc.choices}) + a := NewAdapter(fm, llmgate.ProviderAnthropic, "claude-sonnet-4-5", true) + + resp, err := a.Complete(context.Background(), llmgate.Request{ + Messages: []llmgate.Message{{Role: llmgate.RoleUser, Content: "hi"}}, + }) + if err != nil { + t.Fatalf("Complete: %v", err) + } + + if resp.Content != tc.wantContent { + t.Errorf("Content = %q, want %q", resp.Content, tc.wantContent) + } + if resp.ReasoningContent != tc.wantReasoning { + t.Errorf("ReasoningContent = %q, want %q", resp.ReasoningContent, tc.wantReasoning) + } + if len(resp.ToolCalls) != len(tc.wantToolCalls) { + t.Fatalf("got %d tool calls, want %d (%v)", len(resp.ToolCalls), len(tc.wantToolCalls), tc.wantToolCalls) + } + for i, want := range tc.wantToolCalls { + if resp.ToolCalls[i].Name != want { + t.Errorf("ToolCalls[%d].Name = %q, want %q", i, resp.ToolCalls[i].Name, want) + } + } + }) + } +} + +// TestUsageCountedOnceAcrossBlocks guards the subtle half of the fold: +// every content block carries a copy of the same response-level usage, so +// summing across blocks would multiply the reported bill by the block +// count. +func TestUsageCountedOnceAcrossBlocks(t *testing.T) { + fm := newFake(&llms.ContentResponse{Choices: []*llms.ContentChoice{ + thinkingBlock("thinking"), + textBlock("answer"), + toolBlock("t1", "search", `{}`), + }}) + a := NewAdapter(fm, llmgate.ProviderAnthropic, "claude-sonnet-4-5", true) + + resp, err := a.Complete(context.Background(), llmgate.Request{ + Messages: []llmgate.Message{{Role: llmgate.RoleUser, Content: "hi"}}, + }) + if err != nil { + t.Fatalf("Complete: %v", err) + } + + if resp.Usage.InputTokens != 1000 { + t.Errorf("InputTokens = %d, want 1000 (3 blocks must not triple it)", resp.Usage.InputTokens) + } + if resp.Usage.OutputTokens != 200 { + t.Errorf("OutputTokens = %d, want 200 (3 blocks must not triple it)", resp.Usage.OutputTokens) + } + if resp.Usage.TotalTokens != 1200 { + t.Errorf("TotalTokens = %d, want 1200", resp.Usage.TotalTokens) + } +} + +// TestFoldPrefersChoiceWithUsage: a leading block without usage keys must +// not shadow a later block that has them. +func TestFoldPrefersChoiceWithUsage(t *testing.T) { + noUsage := &llms.ContentChoice{Content: "partial", GenerationInfo: map[string]any{"Citations": "x"}} + fm := newFake(&llms.ContentResponse{Choices: []*llms.ContentChoice{noUsage, textBlock(" rest")}}) + a := NewAdapter(fm, llmgate.ProviderAnthropic, "claude-sonnet-4-5", true) + + resp, err := a.Complete(context.Background(), llmgate.Request{ + Messages: []llmgate.Message{{Role: llmgate.RoleUser, Content: "hi"}}, + }) + if err != nil { + t.Fatalf("Complete: %v", err) + } + if resp.Content != "partial rest" { + t.Errorf("Content = %q, want %q", resp.Content, "partial rest") + } + if resp.Usage.InputTokens != 1000 { + t.Errorf("InputTokens = %d, want 1000 — usage must be picked up from a later block", resp.Usage.InputTokens) + } +} diff --git a/middleware/cache/cache.go b/middleware/cache/cache.go index f900f75..b6b9732 100644 --- a/middleware/cache/cache.go +++ b/middleware/cache/cache.go @@ -8,6 +8,7 @@ import ( "crypto/sha256" "encoding/binary" "encoding/hex" + "hash" "math" "sync" "time" @@ -91,8 +92,17 @@ func cacheKey(req llmgate.Request) string { var buf [8]byte binary.LittleEndian.PutUint64(buf[:], uint64(req.MaxTokens)) h.Write(buf[:]) - binary.LittleEndian.PutUint64(buf[:], math.Float64bits(req.Temperature)) - h.Write(buf[:]) + // Sampling knobs are optional: an unset value must hash differently + // from an explicit one, so write a presence byte before the bits. + writeOptFloat(h, buf[:], req.Temperature) + writeOptFloat(h, buf[:], req.TopP) + if req.Seed != nil { + h.Write([]byte{1}) + binary.LittleEndian.PutUint64(buf[:], uint64(*req.Seed)) + h.Write(buf[:]) + } else { + h.Write([]byte{0}) + } if req.JSONMode { h.Write([]byte{1}) } else { @@ -111,6 +121,19 @@ func cacheKey(req llmgate.Request) string { return hex.EncodeToString(h.Sum(nil)) } +// writeOptFloat hashes an optional float as a presence byte plus, when +// set, its IEEE-754 bits. Without the presence byte an unset field and an +// explicit 0 would collide. +func writeOptFloat(h hash.Hash, buf []byte, v *float64) { + if v == nil { + h.Write([]byte{0}) + return + } + h.Write([]byte{1}) + binary.LittleEndian.PutUint64(buf, math.Float64bits(*v)) + h.Write(buf) +} + type lruCache struct { cap int ttl time.Duration diff --git a/pricing/pricing.go b/pricing/pricing.go index 350920e..624b29d 100644 --- a/pricing/pricing.go +++ b/pricing/pricing.go @@ -14,26 +14,26 @@ type Price struct { // Source: official pricing pages for Anthropic, OpenAI, and Google. var defaultPrices = map[string]Price{ // ── Anthropic ───────────────────────────────────────────────── - "claude-sonnet-4-5": {InputPerMTok: 3.00, OutputPerMTok: 15.00}, + "claude-sonnet-4-5": {InputPerMTok: 3.00, OutputPerMTok: 15.00}, "claude-sonnet-4-20250514": {InputPerMTok: 3.00, OutputPerMTok: 15.00}, - "claude-opus-4-1": {InputPerMTok: 15.00, OutputPerMTok: 75.00}, - "claude-haiku-4-5": {InputPerMTok: 1.00, OutputPerMTok: 5.00}, - "claude-haiku-3-5": {InputPerMTok: 0.80, OutputPerMTok: 4.00}, + "claude-opus-4-1": {InputPerMTok: 15.00, OutputPerMTok: 75.00}, + "claude-haiku-4-5": {InputPerMTok: 1.00, OutputPerMTok: 5.00}, + "claude-haiku-3-5": {InputPerMTok: 0.80, OutputPerMTok: 4.00}, // ── OpenAI ──────────────────────────────────────────────────── - "gpt-4o": {InputPerMTok: 2.50, OutputPerMTok: 10.00}, - "gpt-4o-mini": {InputPerMTok: 0.15, OutputPerMTok: 0.60}, - "gpt-4.1": {InputPerMTok: 2.00, OutputPerMTok: 8.00}, - "gpt-4.1-mini": {InputPerMTok: 0.40, OutputPerMTok: 1.60}, - "gpt-4.1-nano": {InputPerMTok: 0.10, OutputPerMTok: 0.40}, - "o3": {InputPerMTok: 2.00, OutputPerMTok: 8.00}, - "o3-mini": {InputPerMTok: 1.10, OutputPerMTok: 4.40}, - "o4-mini": {InputPerMTok: 1.10, OutputPerMTok: 4.40}, + "gpt-4o": {InputPerMTok: 2.50, OutputPerMTok: 10.00}, + "gpt-4o-mini": {InputPerMTok: 0.15, OutputPerMTok: 0.60}, + "gpt-4.1": {InputPerMTok: 2.00, OutputPerMTok: 8.00}, + "gpt-4.1-mini": {InputPerMTok: 0.40, OutputPerMTok: 1.60}, + "gpt-4.1-nano": {InputPerMTok: 0.10, OutputPerMTok: 0.40}, + "o3": {InputPerMTok: 2.00, OutputPerMTok: 8.00}, + "o3-mini": {InputPerMTok: 1.10, OutputPerMTok: 4.40}, + "o4-mini": {InputPerMTok: 1.10, OutputPerMTok: 4.40}, // ── Google ──────────────────────────────────────────────────── - "gemini-2.5-flash": {InputPerMTok: 0.15, OutputPerMTok: 0.60}, - "gemini-2.5-pro": {InputPerMTok: 1.25, OutputPerMTok: 10.00}, - "gemini-2.0-flash": {InputPerMTok: 0.10, OutputPerMTok: 0.40}, + "gemini-2.5-flash": {InputPerMTok: 0.15, OutputPerMTok: 0.60}, + "gemini-2.5-pro": {InputPerMTok: 1.25, OutputPerMTok: 10.00}, + "gemini-2.0-flash": {InputPerMTok: 0.10, OutputPerMTok: 0.40}, // ── Zhipu / Z.ai GLM (public Z.ai API list prices, added May 2026) ─ "glm-4.6": {InputPerMTok: 0.60, OutputPerMTok: 2.20}, @@ -65,7 +65,7 @@ func Register(model string, p Price) { prices[model] = p } -// WarnFunc, if non-nil, is invoked once per distinct model that has no +// WarnFunc is invoked, when non-nil, once per distinct model that has no // price-book entry, the first time a cost is computed for it. Wire it to a // logger to surface "$0 because the model is unpriced" — otherwise an // unpriced model is silently accounted as free, which is the failure mode