Skip to content
Merged
10 changes: 6 additions & 4 deletions Sources/AnyLanguageModel/LanguageModelSession.swift
Original file line number Diff line number Diff line change
Expand Up @@ -123,19 +123,19 @@ public final class LanguageModelSession: @unchecked Sendable {
state.withLock { $0.endResponding() }
}
}

nonisolated func growStreamingTranscript(text: String) {
withMutation(keyPath: \.transcript) {
state.withLock { $0.transcript.appendStreamingResponse(text) }
}
}

nonisolated func appendTranscriptEntry(_ entry: Transcript.Entry) {
withMutation(keyPath: \.transcript) {
state.withLock { $0.transcript.append(entry) }
}
}

nonisolated private func wrapRespond<T>(_ operation: () async throws -> T) async throws -> T {
beginResponding()
do {
Expand Down Expand Up @@ -176,7 +176,9 @@ public final class LanguageModelSession: @unchecked Sendable {
}

session.withMutation(keyPath: \.transcript) {
session.state.withLock { $0.transcript.finalizeStreamedTranscript(textContent, assetIDs: []) }
session.state.withLock {
$0.transcript.finalizeStreamedTranscript(textContent, assetIDs: [])
}
}
}
} catch {
Expand Down
120 changes: 63 additions & 57 deletions Sources/AnyLanguageModel/Models/AnthropicLanguageModel.swift

Large diffs are not rendered by default.

4 changes: 3 additions & 1 deletion Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift
Original file line number Diff line number Diff line change
Expand Up @@ -723,7 +723,9 @@ private func toGeneratedContent(_ value: [String: JSONValue]?) throws -> Generat
}

private func fromGeneratedContent(_ content: GeneratedContent) throws -> [String: JSONValue] {
let data = try JSONEncoder().encode(content)
// `GeneratedContent`'s `Codable` conformance is a lossless persistence format for
// round-tripping a `Transcript`, not a wire format, so go through `jsonString`.
let data = Data(content.jsonString.utf8)
let jsonValue = try JSONDecoder().decode(JSONValue.self, from: data)

guard case .object(let dict) = jsonValue else {
Expand Down
6 changes: 3 additions & 3 deletions Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift
Original file line number Diff line number Diff line change
Expand Up @@ -672,9 +672,9 @@ import Foundation
// Force CPU-only execution to avoid Metal GPU issues
params.n_gpu_layers = 0

// Try to reduce memory usage
params.use_mmap = true
params.use_mlock = false
// Try to reduce memory usage by memory-mapping the weights without locking them
// in RAM. Replaces the separate `use_mmap` / `use_mlock` flags removed upstream.
params.load_mode = LLAMA_LOAD_MODE_MMAP
return params
}

Expand Down
Loading