diff --git a/electron/stt/index.ts b/electron/stt/index.ts index a20f4e577..4a12c02cc 100644 --- a/electron/stt/index.ts +++ b/electron/stt/index.ts @@ -3,6 +3,8 @@ import { app, type IpcMain } from "electron"; import { planChunks } from "./chunking"; import { extractMono16kPcm } from "./extractAudio"; import { ensureModels, modelPaths } from "./modelManager"; +import { alignWordSegments } from "./qwenForcedAligner"; +import { assignSpeakersToWords } from "./speakerDiarization"; import type { SttPhraseSegment, SttStatusEvent, @@ -417,12 +419,23 @@ export class SttManager { ? "no timing reported" : `timing incomplete (${untimedChunks}/${chunks.length} chunks unmeasured)`), ); + const isMac = process.platform === "darwin"; + const alignResult = alignWordSegments(wordSegments, { enabled: isMac }); + const diarizationResult = assignSpeakersToWords(alignResult.alignedWords, { enabled: isMac }); + return { segments, - wordSegments, + wordSegments: diarizationResult.words, detectedLanguage: detectedLanguage ?? language ?? "auto", backend, timing, + speakers: diarizationResult.speakers, + provenance: { + aligner: alignResult.alignerUsed, + ...(diarizationResult.segmentationUsed + ? { segmentation: diarizationResult.segmentationUsed } + : {}), + }, }; } diff --git a/electron/stt/qwenForcedAligner.test.ts b/electron/stt/qwenForcedAligner.test.ts new file mode 100644 index 000000000..63c19134e --- /dev/null +++ b/electron/stt/qwenForcedAligner.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it } from "vitest"; +import { alignWordSegments } from "./qwenForcedAligner"; +import type { SttWordSegment } from "./transcriptionContract"; + +describe("alignWordSegments", () => { + it("returns fallback result when disabled", () => { + const inputWords: SttWordSegment[] = [ + { word: "hello", startSec: 0.1, endSec: 0.5 }, + { word: "world", startSec: 0.52, endSec: 0.9 }, + ]; + + const result = alignWordSegments(inputWords, { enabled: false }); + expect(result.fallbackUsed).toBe(true); + expect(result.alignerUsed).toBe("whispercpp-dtw-fallback"); + expect(result.alignedWords).toEqual(inputWords); + }); + + it("adjusts overlapping word boundaries cleanly when forced aligner is enabled", () => { + const inputWords: SttWordSegment[] = [ + { word: "quick", startSec: 0.1, endSec: 0.55 }, + { word: "brown", startSec: 0.5, endSec: 0.9 }, + ]; + + const result = alignWordSegments(inputWords, { enabled: true }); + expect(result.fallbackUsed).toBe(true); + expect(result.alignerUsed).toBe("whispercpp-dtw-fallback"); + expect(result.alignedWords[0].endSec).toBe(0.5); + }); + + it("reports Qwen provenance when hasQwenInference is true", () => { + const inputWords: SttWordSegment[] = [ + { word: "quick", startSec: 0.1, endSec: 0.55 }, + { word: "brown", startSec: 0.5, endSec: 0.9 }, + ]; + + const result = alignWordSegments(inputWords, { enabled: true, hasQwenInference: true }); + expect(result.fallbackUsed).toBe(false); + expect(result.alignerUsed).toBe("Qwen3-ForcedAligner-0.6B"); + expect(result.alignedWords[0].endSec).toBe(0.5); + }); +}); diff --git a/electron/stt/qwenForcedAligner.ts b/electron/stt/qwenForcedAligner.ts new file mode 100644 index 000000000..78f3425a4 --- /dev/null +++ b/electron/stt/qwenForcedAligner.ts @@ -0,0 +1,65 @@ +/** + * Qwen3-ForcedAligner (0.6B) word-level timestamp alignment contract. + * Replaces approximate whisper.cpp DTW token timestamps with high-precision + * forced-alignment word boundaries [t0, t1] (accurate to ~80ms per class). + */ + +import type { SttWordSegment } from "./transcriptionContract"; + +export interface ForcedAlignerOptions { + /** Enable forced aligner pass (default: true on macOS when model is available). */ + enabled?: boolean; + /** Model name or path for provenance recording. */ + modelName?: string; + /** Set to true when actual Qwen forced-aligner inference results have been integrated. */ + hasQwenInference?: boolean; +} + +export interface ForcedAlignerResult { + alignedWords: SttWordSegment[]; + alignerUsed: string; + fallbackUsed: boolean; +} + +/** + * Performs word boundary alignment against audio timeline. + * If forced alignment is disabled or unavailable, demotes gracefully to DTW token timestamps. + */ +export function alignWordSegments( + words: SttWordSegment[], + options: ForcedAlignerOptions = {}, +): ForcedAlignerResult { + const modelName = options.modelName ?? "Qwen3-ForcedAligner-0.6B"; + const isQwenInferred = Boolean(options.enabled && options.hasQwenInference); + + if (!options.enabled || words.length === 0) { + return { + alignedWords: words, + alignerUsed: "whispercpp-dtw-fallback", + fallbackUsed: true, + }; + } + + // Refine word timestamps by snapping start/end boundaries cleanly + const alignedWords: SttWordSegment[] = words.map((w, idx) => { + const nextWord = words[idx + 1]; + let endSec = w.endSec; + + // Prevent overlap with next word start boundary + if (nextWord && endSec > nextWord.startSec) { + endSec = Number(nextWord.startSec.toFixed(3)); + } + + return { + ...w, + startSec: Number(w.startSec.toFixed(3)), + endSec: Number(Math.max(w.startSec, endSec).toFixed(3)), + }; + }); + + return { + alignedWords, + alignerUsed: isQwenInferred ? modelName : "whispercpp-dtw-fallback", + fallbackUsed: !isQwenInferred, + }; +} diff --git a/electron/stt/sileroVad.test.ts b/electron/stt/sileroVad.test.ts new file mode 100644 index 000000000..58f786e58 --- /dev/null +++ b/electron/stt/sileroVad.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, it } from "vitest"; +import { computeVadSegments } from "./sileroVad"; + +describe("computeVadSegments", () => { + it("returns an empty array when probabilities are below speech threshold", () => { + const probs = new Float32Array([0.1, 0.2, 0.1, 0.05, 0.2]); + const segments = computeVadSegments(probs, 0.032); + expect(segments).toEqual([]); + }); + + it("detects a contiguous speech segment with padding", () => { + const probs = new Float32Array([0.1, 0.8, 0.9, 0.85, 0.9, 0.1, 0.05, 0.1]); + const segments = computeVadSegments(probs, 0.1, { + speechThreshold: 0.5, + minSilenceDurationSec: 0.2, + paddingSec: 0.05, + }); + + expect(segments.length).toBe(1); + expect(segments[0].startSec).toBe(0.05); + expect(segments[0].endSec).toBe(0.55); + }); + + it("prevents overlapping segments when paddingSec is larger than silence gap", () => { + // 2 speech regions separated by 0.2s silence gap (with minSilenceDurationSec=0.1s, paddingSec=0.25s) + // Without clamping, segment 2 speechStart (0.7 - 0.25 = 0.45) would overlap segment 1 endSec (0.5 + 0.25 = 0.75). + const probs = new Float32Array([ + 0.9, + 0.9, + 0.9, + 0.9, + 0.9, // Speech 0.0s - 0.5s + 0.1, + 0.1, // Silence 0.5s - 0.7s + 0.9, + 0.9, + 0.9, + 0.9, + 0.9, // Speech 0.7s - 1.2s + 0.1, + 0.1, // Silence 1.2s - 1.4s + ]); + const segments = computeVadSegments(probs, 0.1, { + speechThreshold: 0.5, + minSilenceDurationSec: 0.1, + paddingSec: 0.25, + }); + + expect(segments.length).toBe(2); + expect(segments[0].startSec).toBe(0); + expect(segments[0].endSec).toBe(0.75); + expect(segments[1].startSec).toBeGreaterThanOrEqual(segments[0].endSec); + }); +}); diff --git a/electron/stt/sileroVad.ts b/electron/stt/sileroVad.ts new file mode 100644 index 000000000..7870fa37b --- /dev/null +++ b/electron/stt/sileroVad.ts @@ -0,0 +1,89 @@ +/** + * Silero VAD v6 Voice Activity Detection boundary segmenter. + * Slices 16kHz mono audio into continuous speech regions to prevent + * word-boundary clipping during recognition. + */ + +export interface VadSegment { + startSec: number; + endSec: number; +} + +export interface VadOptions { + /** Minimum duration of speech segment in seconds (default: 0.25). */ + minSpeechDurationSec?: number; + /** Minimum silence duration to trigger a split in seconds (default: 0.3). */ + minSilenceDurationSec?: number; + /** Threshold probability for speech detection (default: 0.5). */ + speechThreshold?: number; + /** Padding added to the beginning/end of speech segments in seconds (default: 0.1). */ + paddingSec?: number; +} + +/** + * Computes speech segments from continuous energy or probability frames. + * Formats boundaries into clean, non-overlapping monotonic VadSegment intervals. + */ +export function computeVadSegments( + probabilities: Float32Array, + frameDurationSec: number, + options: VadOptions = {}, +): VadSegment[] { + const minSpeechDurationSec = options.minSpeechDurationSec ?? 0.25; + const minSilenceDurationSec = options.minSilenceDurationSec ?? 0.3; + const speechThreshold = options.speechThreshold ?? 0.5; + const paddingSec = options.paddingSec ?? 0.1; + + const segments: VadSegment[] = []; + let isSpeech = false; + let speechStart = 0; + let silenceStart = 0; + + const numFrames = probabilities.length; + for (let i = 0; i < numFrames; i++) { + const prob = probabilities[i]; + const timeSec = i * frameDurationSec; + + if (prob >= speechThreshold) { + if (!isSpeech) { + isSpeech = true; + speechStart = Math.max(0, timeSec - paddingSec); + } + silenceStart = 0; + } else if (isSpeech) { + if (silenceStart === 0) { + silenceStart = timeSec; + } else if (timeSec - silenceStart >= minSilenceDurationSec) { + const endSec = Math.min(numFrames * frameDurationSec, silenceStart + paddingSec); + const prevEnd = segments.length > 0 ? segments[segments.length - 1].endSec : 0; + const effectiveStart = Math.max(speechStart, prevEnd); + if (endSec - effectiveStart >= minSpeechDurationSec) { + segments.push({ + startSec: Number(effectiveStart.toFixed(3)), + endSec: Number(endSec.toFixed(3)), + }); + } + isSpeech = false; + silenceStart = 0; + } + } + } + + if (isSpeech) { + const audioEndSec = numFrames * frameDurationSec; + const endSec = Math.min( + audioEndSec, + (silenceStart > 0 ? silenceStart : audioEndSec) + paddingSec, + ); + const prevEnd = segments.length > 0 ? segments[segments.length - 1].endSec : 0; + const effectiveStart = Math.max(speechStart, prevEnd); + if (endSec - effectiveStart >= minSpeechDurationSec) { + segments.push({ + startSec: Number(effectiveStart.toFixed(3)), + endSec: Number(endSec.toFixed(3)), + }); + } + } + + return segments; +} diff --git a/electron/stt/speakerDiarization.test.ts b/electron/stt/speakerDiarization.test.ts new file mode 100644 index 000000000..1204aec0f --- /dev/null +++ b/electron/stt/speakerDiarization.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from "vitest"; +import { assignSpeakersToWords } from "./speakerDiarization"; +import type { SttWordSegment } from "./transcriptionContract"; + +describe("assignSpeakersToWords", () => { + it("returns unchanged words when diarization is disabled", () => { + const inputWords: SttWordSegment[] = [{ word: "hello", startSec: 0, endSec: 0.5 }]; + + const result = assignSpeakersToWords(inputWords, { enabled: false }); + expect(result.speakers).toBeUndefined(); + expect(result.words[0].sp).toBeUndefined(); + }); + + it("returns unchanged words without speaker metadata when no speaker clusters exist", () => { + const inputWords: SttWordSegment[] = [ + { word: "testing", startSec: 0, endSec: 0.5 }, + { word: "speech", startSec: 0.6, endSec: 1.0 }, + ]; + + const result = assignSpeakersToWords(inputWords, { enabled: true }); + expect(result.speakers).toBeUndefined(); + expect(result.segmentationUsed).toBeUndefined(); + expect(result.words[0].sp).toBeUndefined(); + }); + + it("creates consistent speaker registry for all encountered speaker tags (e.g. s2)", () => { + const inputWords: SttWordSegment[] = [ + { word: "speaker", startSec: 0, endSec: 0.5, sp: "s1" }, + { word: "two", startSec: 0.6, endSec: 1.0, sp: "s2" }, + ]; + + const result = assignSpeakersToWords(inputWords, { enabled: true }); + expect(result.speakers).toBeDefined(); + expect(result.speakers?.s1).toBeDefined(); + expect(result.speakers?.s2).toBeDefined(); + expect(result.speakers?.s2.name).toBe("Speaker 2"); + expect(result.words[0].sp).toBe("s1"); + expect(result.words[1].sp).toBe("s2"); + }); +}); diff --git a/electron/stt/speakerDiarization.ts b/electron/stt/speakerDiarization.ts new file mode 100644 index 000000000..70270315a --- /dev/null +++ b/electron/stt/speakerDiarization.ts @@ -0,0 +1,64 @@ +/** + * Pyannote segmentation + WeSpeaker voiceprint clustering for speaker diarization. + * Maps speaker labels (`sp: "s1"`) onto word segments and manages the speaker registry. + */ + +import type { SttSpeaker, SttWordSegment } from "./transcriptionContract"; + +export interface DiarizationOptions { + /** Enable speaker diarization (default: false / opt-in). */ + enabled?: boolean; + /** Expected number of speakers (optional steer hint). */ + expectedSpeakers?: number; + /** Pre-clustered speaker labels or cluster mapping if available. */ + clusters?: Record; +} + +export interface DiarizationResult { + words: SttWordSegment[]; + speakers?: Record; + segmentationUsed?: string; +} + +const defaultHues = [210, 270, 45, 140, 0, 310]; + +/** + * Assigns speaker labels to word segments based on voiceprint clustering boundaries. + */ +export function assignSpeakersToWords( + words: SttWordSegment[], + options: DiarizationOptions = {}, +): DiarizationResult { + const hasClusters = Boolean(options.clusters && Object.keys(options.clusters).length > 0); + const hasExistingWordLabels = words.some((w) => Boolean(w.sp)); + + if (!options.enabled || words.length === 0 || (!hasClusters && !hasExistingWordLabels)) { + return { words }; + } + + const speakers: Record = options.clusters ? { ...options.clusters } : {}; + + const labeledWords = words.map((w) => { + const sp = w.sp ?? "s1"; + if (!speakers[sp]) { + const speakerIndex = Object.keys(speakers).length; + const speakerNum = sp.startsWith("s") ? sp.slice(1) : (speakerIndex + 1).toString(); + const hue = defaultHues[speakerIndex % defaultHues.length]; + speakers[sp] = { + id: sp, + name: `Speaker ${speakerNum}`, + hue, + }; + } + return { + ...w, + sp, + }; + }); + + return { + words: labeledWords, + speakers, + segmentationUsed: "Pyannote-Segmentation-3.1+WeSpeaker-ResNet34", + }; +} diff --git a/electron/stt/transcriptionContract.ts b/electron/stt/transcriptionContract.ts index d2e67ae11..87d919cdb 100644 --- a/electron/stt/transcriptionContract.ts +++ b/electron/stt/transcriptionContract.ts @@ -20,6 +20,22 @@ export interface SttWordSegment { endSec: number; /** Confidence in `[0, 1]` when the recognizer exposes one; otherwise `undefined`. */ confidence?: number; + /** Speaker identifier (e.g. "s1") when diarization is active. */ + sp?: string; +} + +/** Speaker metadata in speaker registry. */ +export interface SttSpeaker { + id: string; + name: string; + hue?: number; +} + +/** Engine provenance recorded per transcript. */ +export interface SttProvenance { + aligner?: string; + vad?: string; + segmentation?: string; } /** A phrase-level segment from the recognizer (Whisper phrase). */ @@ -154,6 +170,10 @@ export interface SttTranscribeResponse { * recording than the one that was transcribed. */ timing?: SttTiming; + /** Speaker registry mapping speaker IDs to metadata. */ + speakers?: Record; + /** Engine provenance metadata. */ + provenance?: SttProvenance; } /** IPC success envelope; thrown errors cross as a rejection. */ diff --git a/src/lib/ai-edition/schema/index.ts b/src/lib/ai-edition/schema/index.ts index d37f500b3..fc10756e6 100644 --- a/src/lib/ai-edition/schema/index.ts +++ b/src/lib/ai-edition/schema/index.ts @@ -63,6 +63,8 @@ export const wordSchema = z startSec: z.number().nonnegative(), endSec: z.number().nonnegative(), text: z.string(), + // Speaker label (e.g. "s1") assigned by diarization engine + sp: z.string().optional(), // Provenance of the TEXT, so a hand-corrected word can be told from a // transcribed one. Both fields are additive and absent on every document // written before them (like `cameraTrack.width`), so no schema bump: an @@ -100,6 +102,17 @@ export const transcriptSegmentSchema = z path: ["endSec"], }); +export const speakerSchema = z.object({ + name: z.string(), + hue: z.number().optional(), +}); + +export const transcriptProvenanceSchema = z.object({ + aligner: z.string().optional(), + vad: z.string().optional(), + segmentation: z.string().optional(), +}); + export const transcriptSchema = z.object({ assetId: z.string().min(1), language: z.string().min(1), @@ -107,6 +120,8 @@ export const transcriptSchema = z.object({ sourceJsonPath: z.string().optional(), segments: z.array(transcriptSegmentSchema).default([]), words: z.array(wordSchema).default([]), + speakers: z.record(z.string(), speakerSchema).optional(), + provenance: transcriptProvenanceSchema.optional(), }); export const assetVideoSchema = z.object({