diff --git a/Sources/AnyLanguageModel/LanguageModelSession.swift b/Sources/AnyLanguageModel/LanguageModelSession.swift index a4484318..ccf362ba 100644 --- a/Sources/AnyLanguageModel/LanguageModelSession.swift +++ b/Sources/AnyLanguageModel/LanguageModelSession.swift @@ -123,19 +123,19 @@ public final class LanguageModelSession: @unchecked Sendable { state.withLock { $0.endResponding() } } } - + nonisolated func growStreamingTranscript(text: String) { withMutation(keyPath: \.transcript) { state.withLock { $0.transcript.appendStreamingResponse(text) } } } - + nonisolated func appendTranscriptEntry(_ entry: Transcript.Entry) { withMutation(keyPath: \.transcript) { state.withLock { $0.transcript.append(entry) } } } - + nonisolated private func wrapRespond(_ operation: () async throws -> T) async throws -> T { beginResponding() do { @@ -176,7 +176,9 @@ public final class LanguageModelSession: @unchecked Sendable { } session.withMutation(keyPath: \.transcript) { - session.state.withLock { $0.transcript.finalizeStreamedTranscript(textContent, assetIDs: []) } + session.state.withLock { + $0.transcript.finalizeStreamedTranscript(textContent, assetIDs: []) + } } } } catch { diff --git a/Sources/AnyLanguageModel/Models/AnthropicLanguageModel.swift b/Sources/AnyLanguageModel/Models/AnthropicLanguageModel.swift index 1958b3d9..ef9e9fd7 100644 --- a/Sources/AnyLanguageModel/Models/AnthropicLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/AnthropicLanguageModel.swift @@ -98,7 +98,7 @@ public struct AnthropicLanguageModel: LanguageModel { /// These parameters are merged into the top-level request JSON, /// allowing you to pass additional options not explicitly modeled. public var extraBody: [String: JSONValue]? - + public var effort: Effort? // MARK: - Nested Types @@ -206,7 +206,7 @@ public struct AnthropicLanguageModel: LanguageModel { /// Availability: All Models case low } - + /// Configuration for extended thinking. public struct Thinking: Hashable, Codable, Sendable { /// The type of thinking to use. @@ -218,7 +218,7 @@ public struct AnthropicLanguageModel: LanguageModel { /// internal reasoning process. Larger budgets can improve response quality /// for complex tasks but increase latency and cost. public var budgetTokens: Int? - + /// How thinking should be displayed. public var display: ThinkingDisplay? @@ -229,7 +229,7 @@ public struct AnthropicLanguageModel: LanguageModel { /// Enables adaptive thinking. case adaptive } - + /// How thinking should be returned during generation. public enum ThinkingDisplay: String, Hashable, Codable, Sendable { /// Thinking will be summarized. @@ -245,7 +245,7 @@ public struct AnthropicLanguageModel: LanguageModel { } /// Creates a thinking configuration. - /// + /// /// - Parameters: /// - type: The type of thinking to perform. /// - budgetTokens: The maximum number of tokens to use for thinking. Only required when `type` == `.enabled`. @@ -255,12 +255,12 @@ public struct AnthropicLanguageModel: LanguageModel { self.budgetTokens = budgetTokens self.display = display } - + /// Convenience function for enabling adaptive thinking on supported models. public static func adaptive(display: ThinkingDisplay?) -> Thinking { return Thinking.init(type: .adaptive, budgetTokens: nil, display: display) } - + /// Convenience function for enabling thinking with a token budget on supported models. public static func enabled(budgetTokens: Int, display: ThinkingDisplay?) -> Thinking { return Thinking.init(type: .enabled, budgetTokens: budgetTokens, display: display) @@ -383,13 +383,13 @@ public struct AnthropicLanguageModel: LanguageModel { let anthropicTools: [AnthropicTool] = try session.tools.map { tool in try convertToolToAnthropicFormat(tool) } - + let responseSchema = type == String.self ? nil : try convertSchemaToAnthropicFormat(Content.generationSchema) - + var entries: [Transcript.Entry] = [] var runningText = "" var messages: [AnthropicMessage] = session.transcript.toAnthropicMessages() - + // Loop until no more tool calls are found. while true { let params = try createMessageParams( @@ -400,7 +400,7 @@ public struct AnthropicLanguageModel: LanguageModel { responseSchema: responseSchema, options: options ) - + let body = try JSONEncoder().encode(params) let message: AnthropicMessageResponse = try await httpSession.fetch( @@ -412,7 +412,7 @@ public struct AnthropicLanguageModel: LanguageModel { // Append to messages for future response loops. messages.append(AnthropicMessage(role: .assistant, content: message.content)) - + // Handle tool calls, if present let toolUses: [AnthropicToolUse] = message.content.compactMap { content in if case .toolUse(let u) = content { return u } @@ -447,11 +447,11 @@ public struct AnthropicLanguageModel: LanguageModel { ) ) } - + messages.append(AnthropicMessage(role: .user, content: toolResultBlocks)) entries.append(.toolCalls(Transcript.ToolCalls(invocations.map(\.call)))) - - continue // Keep going through the loop + + continue // Keep going through the loop } } } @@ -463,8 +463,8 @@ public struct AnthropicLanguageModel: LanguageModel { default: return nil } }.joined() - - break // Break the loop + + break // Break the loop } if type == String.self { @@ -483,11 +483,10 @@ public struct AnthropicLanguageModel: LanguageModel { transcriptEntries: ArraySlice(entries) ) } - - + struct ContentAccumulationBlocks: Hashable, Codable, Sendable { enum Kind: Hashable, Codable, Sendable { case toolUse, thinking, text } - + var kind: Kind var text: String var partialJSON: String? @@ -518,9 +517,10 @@ public struct AnthropicLanguageModel: LanguageModel { try convertToolToAnthropicFormat(tool) } - let responseSchema = type == String.self ? nil : try convertSchemaToAnthropicFormat(Content.generationSchema) + let responseSchema = + type == String.self ? nil : try convertSchemaToAnthropicFormat(Content.generationSchema) let expectsStructuredResponse = type != String.self - + var messages: [AnthropicMessage] = session.transcript.toAnthropicMessages() while true { @@ -536,22 +536,22 @@ public struct AnthropicLanguageModel: LanguageModel { options: options, stream: true ) - + let body = try JSONEncoder().encode(params) - + // Stream server-sent events from Anthropic API let events: AsyncThrowingStream = - httpSession + httpSession .fetchEventStream( .post, url: url, headers: headers, body: body ) - + // Accumulating content blocks keyed by their index. var contentBlocks: [Int: ContentAccumulationBlocks] = [:] - + eventStream: for try await event in events { switch event { case .contentBlockStart(let start): @@ -563,7 +563,7 @@ public struct AnthropicLanguageModel: LanguageModel { text: start.contentBlock.text ?? "", id: start.contentBlock.id, name: start.contentBlock.name - ) + ) case "thinking": contentBlocks[start.index] = ContentAccumulationBlocks( kind: .thinking, @@ -580,19 +580,19 @@ public struct AnthropicLanguageModel: LanguageModel { case .textDelta(let textDelta): // Accumulate text delta for streaming // Make sure the block has even been started. - guard contentBlocks[delta.index] != nil else { continue } - + guard contentBlocks[delta.index] != nil else { continue } + // Set default text if contentBlocks[delta.index]?.text == nil { contentBlocks[delta.index]?.text = "" } - + contentBlocks[delta.index]?.text += textDelta.text accumulatedText += textDelta.text - + // Grow the observable transcript so a Transcript-driven UI updates live. session.growStreamingTranscript(text: accumulatedText) - + // Send text back normally if expectsStructuredResponse { if let snapshot: LanguageModelSession.ResponseStream.Snapshot = @@ -610,7 +610,7 @@ public struct AnthropicLanguageModel: LanguageModel { if contentBlocks[delta.index]?.partialJSON == nil { contentBlocks[delta.index]?.partialJSON = "" } - + contentBlocks[delta.index]?.partialJSON? += jsonDelta.partialJson case .thinkingDelta(let thinkingDelta): contentBlocks[delta.index]?.text += thinkingDelta.thinking @@ -632,11 +632,11 @@ public struct AnthropicLanguageModel: LanguageModel { continue } } - + // Assemble assistant content from the streamed content blocks var assistantContent: [AnthropicContent] = [] var toolUses: [AnthropicToolUse] = [] - + for block in contentBlocks.sorted(by: { $0.key < $1.key }).map(\.value) { switch block.kind { case .text: @@ -646,22 +646,25 @@ public struct AnthropicLanguageModel: LanguageModel { case .thinking: // Ensure there is a signature. Needed for claude to reconstruct the thought on the server. guard let signature = block.signature else { continue } - + assistantContent.append( - AnthropicContent.thinking(AnthropicThinking(thinking: block.text, signature: signature)) + AnthropicContent.thinking( + AnthropicThinking(thinking: block.text, signature: signature) + ) ) case .toolUse: - guard let id = block.id, let name = block.name, let jsonString = block.partialJSON else { continue } + guard let id = block.id, let name = block.name, let jsonString = block.partialJSON + else { continue } guard let json = fromPartialJSON(jsonString) else { continue } - + let toolUse = AnthropicToolUse(id: id, name: name, input: json) assistantContent.append(AnthropicContent.toolUse(toolUse)) toolUses.append(toolUse) } } - + messages.append(AnthropicMessage(role: .assistant, content: assistantContent)) - + // Process the tool calls var appendedToolResults = false if !toolUses.isEmpty { @@ -676,32 +679,34 @@ public struct AnthropicLanguageModel: LanguageModel { case .invocations(let invocations): if !invocations.isEmpty { var toolResultBlocks: [AnthropicContent] = [] - + // Need to append tool calls before tool results session.appendTranscriptEntry( .toolCalls(Transcript.ToolCalls(invocations.map(\.call))) ) - + for invocation in invocations { // Save the tool outputs into the transcript. session.appendTranscriptEntry(.toolOutput(invocation.output)) - + toolResultBlocks.append( .toolResult( AnthropicToolResult( toolUseId: invocation.call.id, - content: convertSegmentsToAnthropicContent(invocation.output.segments) + content: convertSegmentsToAnthropicContent( + invocation.output.segments + ) ) ) ) } - + messages.append(AnthropicMessage(role: .user, content: toolResultBlocks)) appendedToolResults = true } } } - + // Continue only if we responded with tool call results, if we didn't the turn is complete. if stopReason == "tool_use" && appendedToolResults { continue @@ -709,7 +714,7 @@ public struct AnthropicLanguageModel: LanguageModel { break } } - + continuation.finish() } catch { continuation.finish(throwing: error) @@ -838,7 +843,7 @@ private func createMessageParams( if let display = thinking.display { thinkingObject["display"] = .string(display.rawValue) } - + params["thinking"] = .object(thinkingObject) } if let serviceTier = customOptions.serviceTier { @@ -1026,7 +1031,9 @@ private func toGeneratedContent(_ value: [String: JSONValue]?) throws -> Generat } private func fromGeneratedContent(_ content: GeneratedContent) throws -> [String: JSONValue] { - let data = try JSONEncoder().encode(content) + // `GeneratedContent`'s `Codable` conformance is a lossless persistence format for + // round-tripping a `Transcript`, not a wire format, so go through `jsonString`. + let data = Data(content.jsonString.utf8) let jsonValue = try JSONDecoder().decode(JSONValue.self, from: data) guard case .object(let dict) = jsonValue else { @@ -1047,7 +1054,6 @@ private func fromPartialJSON(_ json: String) -> [String: JSONValue]? { return dict } - // MARK: - Supporting Types extension Transcript { @@ -1088,7 +1094,7 @@ extension Transcript { ) ) } - + print("Tool use block \(toolUseBlocks)") messages.append( .init( @@ -1387,7 +1393,7 @@ private enum AnthropicStreamEvent: Codable, Sendable { struct ContentBlock: Codable, Sendable { let type: String let text: String? - + // Used by tool use content blocks. let id: String? let name: String? @@ -1453,14 +1459,14 @@ private enum AnthropicStreamEvent: Codable, Sendable { case partialJson = "partial_json" } } - + struct ThinkingDelta: Codable, SendableMetatype { let type: String let thinking: String } - + /// Cryptographic signature for a completed thinking block. - /// + /// /// Emitted at the end of a thinking block, even when ``CustomGenerationOptions/Thinking/display`` is set to `omitted`. /// The signature must be preserved verbatim for thought to be recovered in the transcript. Otherwise the Claude API will throw out any text provided in thinking blocks. struct SignatureDelta: Codable, Sendable { diff --git a/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift b/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift index b074ddca..bf617afb 100644 --- a/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift @@ -723,7 +723,9 @@ private func toGeneratedContent(_ value: [String: JSONValue]?) throws -> Generat } private func fromGeneratedContent(_ content: GeneratedContent) throws -> [String: JSONValue] { - let data = try JSONEncoder().encode(content) + // `GeneratedContent`'s `Codable` conformance is a lossless persistence format for + // round-tripping a `Transcript`, not a wire format, so go through `jsonString`. + let data = Data(content.jsonString.utf8) let jsonValue = try JSONDecoder().decode(JSONValue.self, from: data) guard case .object(let dict) = jsonValue else { diff --git a/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift b/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift index 6bcd8045..90e28de0 100644 --- a/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift @@ -672,9 +672,9 @@ import Foundation // Force CPU-only execution to avoid Metal GPU issues params.n_gpu_layers = 0 - // Try to reduce memory usage - params.use_mmap = true - params.use_mlock = false + // Try to reduce memory usage by memory-mapping the weights without locking them + // in RAM. Replaces the separate `use_mmap` / `use_mlock` flags removed upstream. + params.load_mode = LLAMA_LOAD_MODE_MMAP return params } diff --git a/Sources/AnyLanguageModel/Models/OllamaLanguageModel.swift b/Sources/AnyLanguageModel/Models/OllamaLanguageModel.swift index 82be5cc7..c7671c47 100644 --- a/Sources/AnyLanguageModel/Models/OllamaLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/OllamaLanguageModel.swift @@ -76,11 +76,6 @@ public struct OllamaLanguageModel: LanguageModel { includeSchemaInPrompt: Bool, options: GenerationOptions ) async throws -> LanguageModelSession.Response where Content: Generable { - let userSegments = extractPromptSegments(from: session, fallbackText: prompt.description) - let (ollamaText, ollamaImages) = convertSegmentsToOllama(userSegments) - let messages = [ - OllamaMessage(role: .user, content: ollamaText) - ] let ollamaOptions = convertOptions(options) let ollamaTools = try session.tools.map { tool in try convertToolToOllamaFormat(tool) @@ -93,50 +88,76 @@ public struct OllamaLanguageModel: LanguageModel { ollamaFormat = try JSONValue(schema) } - let params = try createChatParams( - model: model, - messages: messages, - tools: ollamaTools.isEmpty ? nil : ollamaTools, - options: ollamaOptions, - stream: false, - images: ollamaImages.isEmpty ? nil : ollamaImages, - format: ollamaFormat - ) - let url = baseURL.appendingPathComponent("api/chat") - let body = try JSONEncoder().encode(params) - let chatResponse: ChatResponse = try await httpSession.fetch( - .post, - url: url, - body: body, - dateDecodingStrategy: .iso8601WithFractionalSeconds - ) var entries: [Transcript.Entry] = [] + var messages = session.transcript.toOllamaMessages() + var text = "" + + // Loop until the model responds without requesting more tool calls. + while true { + let params = try createChatParams( + model: model, + messages: messages, + tools: ollamaTools.isEmpty ? nil : ollamaTools, + options: ollamaOptions, + stream: false, + format: ollamaFormat + ) - if let toolCalls = chatResponse.message.toolCalls, !toolCalls.isEmpty { - let resolution = try await resolveToolCalls(toolCalls, session: session) - switch resolution { - case .stop(let calls): - if !calls.isEmpty { - entries.append(.toolCalls(Transcript.ToolCalls(calls))) - } - return LanguageModelSession.Response( - content: "" as! Content, - rawContent: GeneratedContent(""), - transcriptEntries: ArraySlice(entries) + let body = try JSONEncoder().encode(params) + let chatResponse: ChatResponse = try await httpSession.fetch( + .post, + url: url, + body: body, + dateDecodingStrategy: .iso8601WithFractionalSeconds + ) + + let message = chatResponse.message + if let toolCalls = message.toolCalls, !toolCalls.isEmpty { + // Echo the assistant turn back so the model can see what it asked for. + messages.append( + OllamaMessage( + role: .assistant, + content: message.content ?? "", + toolCalls: toolCalls.map { $0.asRequestToolCall() } + ) ) - case .invocations(let invocations): - if !invocations.isEmpty { - entries.append(.toolCalls(Transcript.ToolCalls(invocations.map(\.call)))) - for invocation in invocations { - entries.append(.toolOutput(invocation.output)) + + let resolution = try await resolveToolCalls(toolCalls, session: session) + switch resolution { + case .stop(let calls): + if !calls.isEmpty { + entries.append(.toolCalls(Transcript.ToolCalls(calls))) + } + let empty = try emptyResponseContent(for: type) + return LanguageModelSession.Response( + content: empty.content, + rawContent: empty.rawContent, + transcriptEntries: ArraySlice(entries) + ) + case .invocations(let invocations): + if !invocations.isEmpty { + entries.append(.toolCalls(Transcript.ToolCalls(invocations.map(\.call)))) + for invocation in invocations { + entries.append(.toolOutput(invocation.output)) + messages.append( + OllamaMessage( + role: .tool, + content: convertSegmentsToToolContentString(invocation.output.segments), + toolName: invocation.call.toolName + ) + ) + } + continue } } } + + text = message.content ?? "" + break } - let text = chatResponse.message.content ?? "" if type == String.self { return LanguageModelSession.Response( content: text as! Content, @@ -161,43 +182,40 @@ public struct OllamaLanguageModel: LanguageModel { includeSchemaInPrompt: Bool, options: GenerationOptions ) -> sending LanguageModelSession.ResponseStream where Content: Generable { - let userSegments = extractPromptSegments(from: session, fallbackText: prompt.description) - let (ollamaText, ollamaImages) = convertSegmentsToOllama(userSegments) - let messages = [ - OllamaMessage(role: .user, content: ollamaText) - ] let ollamaOptions = convertOptions(options) let url = baseURL.appendingPathComponent("api/chat") // Transform the newline-delimited JSON stream from Ollama into ResponseStream snapshots let stream: AsyncThrowingStream.Snapshot, any Error> = AsyncThrowingStream { continuation in - do { - let ollamaTools = try session.tools.map { tool in - try convertToolToOllamaFormat(tool) - } - let ollamaFormat: JSONValue? - if type == String.self { - ollamaFormat = nil - } else { - let schema = try convertSchemaToOllamaFormat(type.generationSchema) - ollamaFormat = try JSONValue(schema) - } - - let params = try createChatParams( - model: model, - messages: messages, - tools: ollamaTools.isEmpty ? nil : ollamaTools, - options: ollamaOptions, - stream: true, - images: (ollamaImages.isEmpty ? nil : ollamaImages), - format: ollamaFormat - ) - let body = try JSONEncoder().encode(params) + let task = Task { + do { + let ollamaTools = try session.tools.map { tool in + try convertToolToOllamaFormat(tool) + } + let ollamaFormat: JSONValue? + if type == String.self { + ollamaFormat = nil + } else { + let schema = try convertSchemaToOllamaFormat(type.generationSchema) + ollamaFormat = try JSONValue(schema) + } - let task = Task { - // Reuse ChatResponse as each streamed line shares the same shape - do { + var messages = session.transcript.toOllamaMessages() + + // Loop until the model responds without requesting more tool calls. + while true { + let params = try createChatParams( + model: model, + messages: messages, + tools: ollamaTools.isEmpty ? nil : ollamaTools, + options: ollamaOptions, + stream: true, + format: ollamaFormat + ) + let body = try JSONEncoder().encode(params) + + // Reuse ChatResponse as each streamed line shares the same shape let chunks = httpSession.fetchStream( .post, @@ -207,10 +225,19 @@ public struct OllamaLanguageModel: LanguageModel { ) as AsyncThrowingStream var partialText = "" + var streamedToolCalls: [OllamaToolCall] = [] for try await chunk in chunks { - if let piece = chunk.message.content { + if let calls = chunk.message.toolCalls, !calls.isEmpty { + streamedToolCalls.append(contentsOf: calls) + } + + if let piece = chunk.message.content, !piece.isEmpty { partialText += piece + + // Grow the observable transcript so a Transcript-driven UI updates live. + session.growStreamingTranscript(text: partialText) + if type == String.self { let snapshot = LanguageModelSession.ResponseStream.Snapshot( content: (partialText as! Content).asPartiallyGenerated(), @@ -236,17 +263,58 @@ public struct OllamaLanguageModel: LanguageModel { } } - continuation.finish() - } catch { - continuation.finish(throwing: error) + guard !streamedToolCalls.isEmpty else { break } + + // Echo the assistant turn back so the model can see what it asked for. + messages.append( + OllamaMessage( + role: .assistant, + content: partialText, + toolCalls: streamedToolCalls.map { $0.asRequestToolCall() } + ) + ) + + let resolution = try await resolveToolCalls(streamedToolCalls, session: session) + switch resolution { + case .stop(let calls): + if !calls.isEmpty { + session.appendTranscriptEntry(.toolCalls(Transcript.ToolCalls(calls))) + } + continuation.finish() + return + case .invocations(let invocations): + // Nothing resolved means there is nothing to feed back, so the turn is over. + guard !invocations.isEmpty else { + continuation.finish() + return + } + + // Tool calls must land in the transcript before their outputs. + session.appendTranscriptEntry( + .toolCalls(Transcript.ToolCalls(invocations.map(\.call))) + ) + + for invocation in invocations { + session.appendTranscriptEntry(.toolOutput(invocation.output)) + messages.append( + OllamaMessage( + role: .tool, + content: convertSegmentsToToolContentString(invocation.output.segments), + toolName: invocation.call.toolName + ) + ) + } + } } - } - continuation.onTermination = { _ in - task.cancel() + continuation.finish() + } catch { + continuation.finish(throwing: error) } - } catch { - continuation.finish(throwing: error) + } + + continuation.onTermination = { _ in + task.cancel() } } @@ -441,6 +509,20 @@ private func convertSchemaToOllamaFormat(_ schema: GenerationSchema) throws -> J return try JSONDecoder().decode(JSONSchema.self, from: data) } +/// Converts generated content to the plain JSON Ollama expects for tool call arguments. +/// +/// `GeneratedContent`'s `Codable` conformance is a lossless persistence format: it writes +/// `kind`/`orderedKeys` so that a `Transcript` can be serialized and read back exactly. +/// That is deliberate, and not the shape a provider expects for tool arguments, so go +/// through `jsonString` at the wire boundary instead. +private func fromGeneratedContent(_ content: GeneratedContent) -> JSONValue { + let json = content.jsonString + guard let value = try? JSONDecoder().decode(JSONValue.self, from: Data(json.utf8)) else { + return .object([:]) + } + return value +} + private func toGeneratedContent(_ value: JSONValue?) throws -> GeneratedContent { guard let value else { return GeneratedContent(properties: [:]) } let data = try JSONEncoder().encode(value) @@ -454,7 +536,6 @@ private func createChatParams( tools: [[String: JSONValue]]?, options: [String: JSONValue]?, stream: Bool, - images: [String]?, format: JSONValue? ) throws -> [String: JSONValue] { var params: [String: JSONValue] = [ @@ -471,10 +552,6 @@ private func createChatParams( params["options"] = .object(options) } - if let images, !images.isEmpty { - params["images"] = .array(images.map { .string($0) }) - } - if let format { params["format"] = format } @@ -482,6 +559,19 @@ private func createChatParams( return params } +private func emptyResponseContent( + for type: Content.Type +) throws -> (content: Content, rawContent: GeneratedContent) { + if type == String.self { + let raw = GeneratedContent("") + return ("" as! Content, raw) + } + + let raw = GeneratedContent(properties: [:]) + let content = try type.init(raw) + return (content, raw) +} + // MARK: - Supporting Types private struct OllamaMessage: Hashable, Codable, Sendable { @@ -494,6 +584,97 @@ private struct OllamaMessage: Hashable, Codable, Sendable { let role: Role let content: String + /// Base64-encoded images. `/api/chat` carries these per message rather than at the top level. + var images: [String]? + var toolCalls: [OllamaRequestToolCall]? + /// Names the tool a `.tool` message is answering. + var toolName: String? + + private enum CodingKeys: String, CodingKey { + case role + case content + case images + case toolCalls = "tool_calls" + case toolName = "tool_name" + } +} + +/// The request-side shape of a tool call, echoed back to Ollama in the assistant turn. +private struct OllamaRequestToolCall: Hashable, Codable, Sendable { + struct Function: Hashable, Codable, Sendable { + let name: String + let arguments: JSONValue + } + + let function: Function +} + +extension Transcript { + fileprivate func toOllamaMessages() -> [OllamaMessage] { + var messages: [OllamaMessage] = [] + for entry in self { + switch entry { + case .instructions(let instructions): + let (text, images) = convertSegmentsToOllama(instructions.segments) + messages.append( + OllamaMessage(role: .system, content: text, images: images.isEmpty ? nil : images) + ) + case .prompt(let prompt): + let (text, images) = convertSegmentsToOllama(prompt.segments) + messages.append( + OllamaMessage(role: .user, content: text, images: images.isEmpty ? nil : images) + ) + case .response(let response): + let (text, images) = convertSegmentsToOllama(response.segments) + messages.append( + OllamaMessage(role: .assistant, content: text, images: images.isEmpty ? nil : images) + ) + case .toolCalls(let toolCalls): + messages.append( + OllamaMessage( + role: .assistant, + content: "", + toolCalls: toolCalls.map { call in + OllamaRequestToolCall( + function: .init( + name: call.toolName, + arguments: fromGeneratedContent(call.arguments) + ) + ) + } + ) + ) + case .toolOutput(let toolOutput): + messages.append( + OllamaMessage( + role: .tool, + content: convertSegmentsToToolContentString(toolOutput.segments), + toolName: toolOutput.toolName + ) + ) + } + } + return messages + } +} + +/// Flattens transcript segments into the string content a `.tool` message carries. +/// +/// Image segments are dropped because Ollama tool results are text-only. +private func convertSegmentsToToolContentString(_ segments: [Transcript.Segment]) -> String { + segments.compactMap { segment in + switch segment { + case .text(let textSegment): + return textSegment.content + case .structure(let structuredSegment): + switch structuredSegment.content.kind { + case .string(let text): return text + default: return structuredSegment.content.jsonString + } + case .image: + return nil + } + }.joined(separator: "\n") } private func convertSegmentsToOllama(_ segments: [Transcript.Segment]) -> (String, [String]) { @@ -518,15 +699,6 @@ private func convertSegmentsToOllama(_ segments: [Transcript.Segment]) -> (Strin return (textParts.joined(separator: "\n"), images) } -private func extractPromptSegments(from session: LanguageModelSession, fallbackText: String) -> [Transcript.Segment] { - for entry in session.transcript.reversed() { - if case .prompt(let p) = entry { - return p.segments - } - } - return [.text(.init(content: fallbackText))] -} - private struct ChatResponse: Decodable, Sendable { let model: String let createdAt: Date @@ -557,6 +729,12 @@ private struct OllamaToolCall: Decodable, Sendable { let id: String? let type: String? let function: OllamaToolFunction + + func asRequestToolCall() -> OllamaRequestToolCall { + OllamaRequestToolCall( + function: .init(name: function.name, arguments: function.arguments ?? .object([:])) + ) + } } private struct OllamaToolFunction: Decodable, Sendable { diff --git a/Sources/AnyLanguageModel/Models/OpenAILanguageModel.swift b/Sources/AnyLanguageModel/Models/OpenAILanguageModel.swift index 20bca5eb..b062065c 100644 --- a/Sources/AnyLanguageModel/Models/OpenAILanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/OpenAILanguageModel.swift @@ -1267,14 +1267,9 @@ extension Transcript { case .toolCalls(let toolCalls): // Add assistant message with tool calls let openAIToolCalls: [JSONValue] = toolCalls.map { call in - let argumentsJSON: String - if let data = try? JSONEncoder().encode(call.arguments), - let jsonString = String(data: data, encoding: .utf8) - { - argumentsJSON = jsonString - } else { - argumentsJSON = "{}" - } + // `GeneratedContent`'s `Codable` conformance is a lossless persistence format + // for round-tripping a `Transcript`, not a wire format, so use `jsonString`. + let argumentsJSON = call.arguments.jsonString return .object([ "id": .string(call.id), diff --git a/Sources/AnyLanguageModel/Models/OpenResponsesLanguageModel.swift b/Sources/AnyLanguageModel/Models/OpenResponsesLanguageModel.swift index 8370e14f..9198cb9d 100644 --- a/Sources/AnyLanguageModel/Models/OpenResponsesLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/OpenResponsesLanguageModel.swift @@ -816,9 +816,9 @@ extension Transcript { ) case .toolCalls(let toolCalls): let rawCalls: [JSONValue] = toolCalls.map { call in - let argsStr = - (try? JSONEncoder().encode(call.arguments)).flatMap { String(data: $0, encoding: .utf8) } - ?? "{}" + // `GeneratedContent`'s `Codable` conformance is a lossless persistence format + // for round-tripping a `Transcript`, not a wire format, so use `jsonString`. + let argsStr = call.arguments.jsonString return .object([ "id": .string(call.id), "type": .string("function_call"), diff --git a/Sources/AnyLanguageModel/Transcript.swift b/Sources/AnyLanguageModel/Transcript.swift index 9716623f..f256d046 100644 --- a/Sources/AnyLanguageModel/Transcript.swift +++ b/Sources/AnyLanguageModel/Transcript.swift @@ -21,11 +21,11 @@ public struct Transcript: Sendable, Equatable, Codable { mutating func append(contentsOf newEntries: S) where S: Sequence, S.Element == Entry { entries.append(contentsOf: newEntries) } - + mutating private func replace(index: Int, with entry: Entry) { entries[index] = entry } - + /// Updates a transcript with temporary text that is being streamed from a model. /// Appends the assistant response to the end of entries, if the last entry is a response then that response is updated with the newest streamed text. /// @@ -33,24 +33,33 @@ public struct Transcript: Sendable, Equatable, Codable { mutating func appendStreamingResponse(_ text: String) { // Make sure the last entry in the transcript is a response. If it is not, create a new response and append it to the end of the transcript. guard case .response(var response) = entries.last else { - append(Entry.response(Response(assetIDs: [], segments: [ - Transcript.Segment.text(Transcript.TextSegment(content: text)) - ]))) + append( + Entry.response( + Response( + assetIDs: [], + segments: [ + Transcript.Segment.text(Transcript.TextSegment(content: text)) + ] + ) + ) + ) return } - + // If the last segment in the last response is text, replace it with the new content. if case .text(let last)? = response.segments.last { // Keep the same ID as the last segment. - response.segments[response.segments.count - 1] = Transcript.Segment.text(Transcript.TextSegment(id: last.id, content: text)) + response.segments[response.segments.count - 1] = Transcript.Segment.text( + Transcript.TextSegment(id: last.id, content: text) + ) } else { response.segments.append(Transcript.Segment.text(Transcript.TextSegment(content: text))) } - + // Replace the latest entry with the one we just updated. replace(index: entries.count - 1, with: .response(response)) } - + /// Replaces the trailing response entry's text with the final text, or appends a new response entry if the last entry isn't a response. /// Prevents streamed responses from having duplicate entries on completion. /// @@ -60,28 +69,41 @@ public struct Transcript: Sendable, Equatable, Codable { mutating func finalizeStreamedTranscript(_ text: String, assetIDs: [String]) { // Make sure the last entry in the transcript is a response. If it is not, create a new response and append it to the end of the transcript. guard case .response(let response) = entries.last else { - append(Entry.response(Response(assetIDs: assetIDs, segments: [ - Transcript.Segment.text(Transcript.TextSegment(content: text)) - ]))) + append( + Entry.response( + Response( + assetIDs: assetIDs, + segments: [ + Transcript.Segment.text(Transcript.TextSegment(content: text)) + ] + ) + ) + ) return } - + // If the last segment is text we want to carry its ID over to the new text segment. Otherwise generate a new ID for it. - let id = switch response.segments.last { - case .text(let last): - last.id - default: - UUID().uuidString - } - - let newResponse: Entry = Entry.response(Response(id: response.id, assetIDs: assetIDs, segments: [ - Transcript.Segment.text(Transcript.TextSegment(id: id, content: text)) - ])) - + let id = + switch response.segments.last { + case .text(let last): + last.id + default: + UUID().uuidString + } + + let newResponse: Entry = Entry.response( + Response( + id: response.id, + assetIDs: assetIDs, + segments: [ + Transcript.Segment.text(Transcript.TextSegment(id: id, content: text)) + ] + ) + ) + replace(index: entries.count - 1, with: newResponse) } - /// An entry in a transcript. public enum Entry: Sendable, Identifiable, Equatable, Codable { /// Instructions, typically provided by you, the developer. diff --git a/Tests/AnyLanguageModelTests/AnthropicLanguageModelTests.swift b/Tests/AnyLanguageModelTests/AnthropicLanguageModelTests.swift index b0fd1bd1..8646aaf0 100644 --- a/Tests/AnyLanguageModelTests/AnthropicLanguageModelTests.swift +++ b/Tests/AnyLanguageModelTests/AnthropicLanguageModelTests.swift @@ -84,16 +84,16 @@ struct AnthropicLanguageModelTests { #expect(!snapshots.last!.rawContent.jsonString.isEmpty) #expect(!(snapshots.last!.content.summary ?? "").isEmpty) } - + @Test func streamingTranscript() async throws { let session = LanguageModelSession(model: model) - + let stream = session.streamResponse(to: "Say 'Hello' slowly") - + var snapshots: [LanguageModelSession.ResponseStream.Snapshot] = [] for try await snapshot in stream { snapshots.append(snapshot) - + // Make sure the snapshot is also in the transcript. let hasTranscriptEntry = session.transcript.contains(where: { entry in switch entry { @@ -110,14 +110,16 @@ struct AnthropicLanguageModelTests { return false } }) - #expect(hasTranscriptEntry, "Expected the string snapshot to also appear in the transcript during streaming.") + #expect( + hasTranscriptEntry, + "Expected the string snapshot to also appear in the transcript during streaming." + ) } - + #expect(!snapshots.isEmpty) #expect(!snapshots.last!.rawContent.jsonString.isEmpty) } - @Test func withGenerationOptions() async throws { let session = LanguageModelSession(model: model) @@ -158,9 +160,9 @@ struct AnthropicLanguageModelTests { @Test func withTools() async throws { let weatherTool = WeatherTool() let session = LanguageModelSession(model: model, tools: [weatherTool]) - + let response = try await session.respond(to: "How's the weather in San Francisco?") - + var foundToolOutput = false for case let .toolOutput(toolOutput) in response.transcriptEntries { #expect(!toolOutput.id.isEmpty) @@ -170,21 +172,20 @@ struct AnthropicLanguageModelTests { #expect(foundToolOutput) } - @Test func streamWithTools() async throws { let weatherTool = WeatherTool() let session = LanguageModelSession(model: model, tools: [weatherTool]) - + let stream = session.streamResponse(to: "How's the weather in San Francisco?") - + var snapshots: [LanguageModelSession.ResponseStream.Snapshot] = [] - + var toolAppearedInTranscript: Bool = false var toolResponseAppearedInTranscript: Bool = false - + for try await snapshot in stream { snapshots.append(snapshot) - + for entry in session.transcript { switch entry { case .toolCalls: @@ -195,10 +196,13 @@ struct AnthropicLanguageModelTests { } } } - + #expect(toolAppearedInTranscript, "Expected a tool call to appear in the transcript during streaming.") - #expect(toolResponseAppearedInTranscript, "Expected a tool output to appear in the transcript during streaming.") - + #expect( + toolResponseAppearedInTranscript, + "Expected a tool output to appear in the transcript during streaming." + ) + var foundToolOutput = false for case let .toolOutput(toolOutput) in session.transcript { #expect(!toolOutput.id.isEmpty) @@ -207,7 +211,7 @@ struct AnthropicLanguageModelTests { } #expect(foundToolOutput, "Expected the 'getWeather' tool to exist in the final transcript.") } - + @Test func multimodalWithImageURL() async throws { let session = LanguageModelSession(model: model) let response = try await session.respond( diff --git a/Tests/AnyLanguageModelTests/CustomGenerationOptionsTests.swift b/Tests/AnyLanguageModelTests/CustomGenerationOptionsTests.swift index b0f53662..88bf50f0 100644 --- a/Tests/AnyLanguageModelTests/CustomGenerationOptionsTests.swift +++ b/Tests/AnyLanguageModelTests/CustomGenerationOptionsTests.swift @@ -294,7 +294,11 @@ struct AnthropicCustomOptionsTests { } @Test func thinkingCodable() throws { - let thinking = AnthropicLanguageModel.CustomGenerationOptions.Thinking(type: .enabled, budgetTokens: 8192, display: .omitted) + let thinking = AnthropicLanguageModel.CustomGenerationOptions.Thinking( + type: .enabled, + budgetTokens: 8192, + display: .omitted + ) let encoder = JSONEncoder() let data = try encoder.encode(thinking) diff --git a/Tests/AnyLanguageModelTests/TranscriptTests.swift b/Tests/AnyLanguageModelTests/TranscriptTests.swift index 60e9122e..037c70a0 100644 --- a/Tests/AnyLanguageModelTests/TranscriptTests.swift +++ b/Tests/AnyLanguageModelTests/TranscriptTests.swift @@ -58,12 +58,14 @@ struct TranscriptTests { @Test func sessionRestoresInstructionsFromTranscript() throws { let instructions = "First\n\nSecond trailing spaces " let transcript = Transcript(entries: [ - .instructions(.init( - id: "instructions-id", - segments: [.text(.init(content: instructions))], - toolDefinitions: [] - )), - .prompt(.init(segments: [.text(.init(content: "Hello"))])) + .instructions( + .init( + id: "instructions-id", + segments: [.text(.init(content: instructions))], + toolDefinitions: [] + ) + ), + .prompt(.init(segments: [.text(.init(content: "Hello"))])), ]) let session = LanguageModelSession(model: MockLanguageModel(), transcript: transcript)