diff --git a/app/api/v1/routes/call_import_evaluations.py b/app/api/v1/routes/call_import_evaluations.py index ef316f2e..f73744ad 100644 --- a/app/api/v1/routes/call_import_evaluations.py +++ b/app/api/v1/routes/call_import_evaluations.py @@ -242,17 +242,32 @@ def _evaluated_transcript_source_label( evaluation: CallImportEvaluation, source_row: CallImportRow, ) -> str: - """Label whether this row had a diarised transcript for scoring.""" - del evaluation + """Label which transcript source this row was scored against.""" + source = (evaluation.transcript_source or "diarised").strip().lower() + if source == "production": + if not (source_row.transcript or "").strip(): + return "" + return "Production" if not (source_row.diarised_transcript or "").strip(): return "" return "Diarised" -def _pick_evaluation_row_transcript(source_row: Optional[CallImportRow]) -> Optional[str]: - """Transcript shown in evaluation row detail — diarised when present.""" +def _pick_evaluation_row_transcript( + source_row: Optional[CallImportRow], + evaluation: Optional[CallImportEvaluation] = None, +) -> Optional[str]: + """Transcript shown in evaluation row detail for the run's source.""" if source_row is None: return None + source = ( + (evaluation.transcript_source or "diarised").strip().lower() + if evaluation is not None + else "diarised" + ) + if source == "production": + raw = (source_row.transcript or "").strip() + return raw or None diarised = (source_row.diarised_transcript or "").strip() if diarised: return diarised @@ -263,6 +278,7 @@ def _pick_evaluation_row_transcript(source_row: Optional[CallImportRow]) -> Opti def _to_evaluation_row_response( eval_row_obj: CallImportEvaluationRow, source_row: Optional[CallImportRow], + evaluation: Optional[CallImportEvaluation] = None, ) -> CallImportEvaluationRowResponse: """Serialize one evaluation row plus joined source-row metadata.""" return CallImportEvaluationRowResponse( @@ -271,7 +287,7 @@ def _to_evaluation_row_response( call_import_row_id=eval_row_obj.call_import_row_id, row_index=source_row.row_index if source_row else None, conversation_id=source_row.conversation_id if source_row else None, - transcript=_pick_evaluation_row_transcript(source_row), + transcript=_pick_evaluation_row_transcript(source_row, evaluation), raw_columns=source_row.raw_columns if source_row else None, recording_url=source_row.recording_url if source_row else None, recording_date=source_row.recording_date if source_row else None, @@ -811,125 +827,127 @@ async def create_call_import_evaluation( metric_overrides_payload[leaf_id] = override_dict # ----- Validate auto-transcribe settings ----- - # Every evaluation run scores the diarised transcript and - # auto-diarises rows that don't already have one, so STT - # provider+model are mandatory on every request (the - # ``auto_transcribe`` flag is preserved on the schema for API - # compatibility but is effectively always true at this point). - # ``transcribe_mode`` controls whether STT is required: the - # ``llm_only`` path skips STT entirely and feeds audio directly to - # the diariser LLM, so STT fields must be absent. The ``stt_llm`` - # path (default) keeps the original behaviour. - transcribe_mode_norm = (payload.transcribe_mode or "stt_llm").strip().lower() - if transcribe_mode_norm not in {"stt_llm", "llm_only"}: - raise HTTPException( - status_code=400, - detail=( - f"Unknown transcribe_mode '{payload.transcribe_mode}'. " - "Expected 'stt_llm' or 'llm_only'." - ), - ) + # Diarised runs auto-diarise rows missing a diarised transcript and + # require STT + diariser LLM config. Production runs score the CSV + # transcript directly and skip diarisation entirely. + use_diarised = payload.transcript_sources[0] == "diarised" + auto_transcribe = use_diarised - auto_transcribe = True + transcribe_mode_norm: Optional[str] = None stt_provider_norm: Optional[str] = None stt_model_norm: Optional[str] = None - if transcribe_mode_norm == "stt_llm": - if not payload.stt_provider: + diarisation_llm_provider_norm: Optional[str] = None + diarisation_llm_model_norm: Optional[str] = None + diarisation_prompt_norm: Optional[str] = None + + if use_diarised: + transcribe_mode_norm = (payload.transcribe_mode or "stt_llm").strip().lower() + if transcribe_mode_norm not in {"stt_llm", "llm_only"}: + raise HTTPException( + status_code=400, + detail=( + f"Unknown transcribe_mode '{payload.transcribe_mode}'. " + "Expected 'stt_llm' or 'llm_only'." + ), + ) + + if transcribe_mode_norm == "stt_llm": + if not payload.stt_provider: + raise HTTPException( + status_code=400, + detail=( + "stt_provider is required when " + "transcribe_mode='stt_llm': every evaluation run " + "auto-diarises rows that are missing a diarised " + "transcript." + ), + ) + if not payload.stt_model: + raise HTTPException( + status_code=400, + detail=( + "stt_model is required when transcribe_mode='stt_llm'." + ), + ) + try: + stt_provider_norm = ModelProvider( + payload.stt_provider.lower() + ).value + except ValueError: + raise HTTPException( + status_code=400, + detail=f"Unknown STT provider '{payload.stt_provider}'.", + ) + stt_model_norm = payload.stt_model.strip() or None + if not stt_model_norm: + raise HTTPException( + status_code=400, detail="stt_model cannot be empty." + ) + else: + # llm_only — explicitly reject lingering STT inputs so the + # contract is unambiguous (the worker would ignore them but + # silent acceptance hides accidental misconfiguration). + if (payload.stt_provider or "").strip() or ( + payload.stt_model or "" + ).strip(): + raise HTTPException( + status_code=400, + detail=( + "stt_provider / stt_model must be omitted when " + "transcribe_mode='llm_only'; the LLM consumes the " + "audio directly." + ), + ) + + # --- Validate LLM diariser settings ----- + if not payload.diarization_llm_provider: raise HTTPException( status_code=400, detail=( - "stt_provider is required when " - "transcribe_mode='stt_llm': every evaluation run " - "auto-diarises rows that are missing a diarised " - "transcript." + "diarization_llm_provider is required: every evaluation " + "run diarises STT output with an LLM." ), ) - if not payload.stt_model: + if not payload.diarization_llm_model: raise HTTPException( status_code=400, detail=( - "stt_model is required when transcribe_mode='stt_llm'." + "diarization_llm_model is required: every evaluation " + "run diarises STT output with an LLM." ), ) try: - stt_provider_norm = ModelProvider( - payload.stt_provider.lower() + diarisation_llm_provider_norm = ModelProvider( + payload.diarization_llm_provider.lower() ).value except ValueError: raise HTTPException( status_code=400, - detail=f"Unknown STT provider '{payload.stt_provider}'.", - ) - stt_model_norm = payload.stt_model.strip() or None - if not stt_model_norm: - raise HTTPException( - status_code=400, detail="stt_model cannot be empty." + detail=( + f"Unknown diarisation LLM provider " + f"'{payload.diarization_llm_provider}'." + ), ) - else: - # llm_only — explicitly reject lingering STT inputs so the - # contract is unambiguous (the worker would ignore them but - # silent acceptance hides accidental misconfiguration). - if (payload.stt_provider or "").strip() or ( - payload.stt_model or "" - ).strip(): + diarisation_llm_model_norm = ( + payload.diarization_llm_model.strip() or None + ) + if not diarisation_llm_model_norm: raise HTTPException( status_code=400, - detail=( - "stt_provider / stt_model must be omitted when " - "transcribe_mode='llm_only'; the LLM consumes the " - "audio directly." - ), + detail="diarization_llm_model cannot be empty.", ) + diarisation_prompt_norm = ( + payload.diarization_prompt.strip() + if isinstance(payload.diarization_prompt, str) + else None + ) or None - # --- Validate LLM diariser settings ----- - # The post-STT diariser is mandatory now that pyannote is no longer - # in the loop. We reject the request up-front (instead of letting - # individual rows fail at task time) so the operator gets a clean - # 400 in the modal. - if not payload.diarization_llm_provider: - raise HTTPException( - status_code=400, - detail=( - "diarization_llm_provider is required: every evaluation " - "run diarises STT output with an LLM." - ), - ) - if not payload.diarization_llm_model: - raise HTTPException( - status_code=400, - detail=( - "diarization_llm_model is required: every evaluation " - "run diarises STT output with an LLM." - ), - ) - try: - diarisation_llm_provider_norm: Optional[str] = ModelProvider( - payload.diarization_llm_provider.lower() - ).value - except ValueError: - raise HTTPException( - status_code=400, - detail=( - f"Unknown diarisation LLM provider " - f"'{payload.diarization_llm_provider}'." - ), - ) - diarisation_llm_model_norm: Optional[str] = ( - payload.diarization_llm_model.strip() or None + from app.models.enums import CallImportParameterType, CallImportStatus + from app.services.call_imports.bulk_ops import ( + count_all_source_rows, + count_completed_source_rows, + count_source_rows_with_production_transcript, ) - if not diarisation_llm_model_norm: - raise HTTPException( - status_code=400, - detail="diarization_llm_model cannot be empty.", - ) - diarisation_prompt_norm: Optional[str] = ( - payload.diarization_prompt.strip() - if isinstance(payload.diarization_prompt, str) - else None - ) or None - - from app.models.enums import CallImportStatus - from app.services.call_imports.bulk_ops import count_completed_source_rows starting_from_mapped = False if call_import.status == CallImportStatus.MAPPED: @@ -958,6 +976,21 @@ async def create_call_import_evaluation( db, organization_id, workspace_id, call_import.schema_id ) parameters = list(schema.parameters) + if not use_diarised: + transcript_mapped = any( + param.type == CallImportParameterType.TRANSCRIPT + and (call_import.parameter_mapping or {}).get(param.name) + for param in parameters + ) + if not transcript_mapped: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + "No transcript column is mapped in this batch. " + "Map a schema transcript parameter to a CSV column, " + "or choose 'Diarize then evaluate'." + ), + ) if payload.telephony_integration_id is not None: integration = _resolve_telephony_integration( db, @@ -989,14 +1022,31 @@ async def create_call_import_evaluation( db.refresh(call_import) starting_from_mapped = True - total_row_count = count_completed_source_rows(db, call_import.id) + if use_diarised: + total_row_count = count_completed_source_rows(db, call_import.id) + else: + # Production runs score CSV text — rows need not wait for + # recording fetch to finish before they are evaluable. + total_row_count = count_source_rows_with_production_transcript( + db, call_import.id + ) + + requested_sources: List[str] = list(payload.transcript_sources) - # Every evaluation run scores the diarised transcript. The - # ``transcript_sources`` field on the schema has already been - # normalized to ``['diarised']`` by the validator; we still iterate - # the list below so the loop machinery stays generic in case future - # sources are reintroduced. - requested_sources: List[str] = ["diarised"] + if ( + not use_diarised + and not starting_from_mapped + and count_all_source_rows(db, call_import.id) > 0 + and total_row_count == 0 + ): + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + "No rows have a production transcript. " + "Choose 'Diarize then evaluate' or import rows with " + "a transcript column." + ), + ) base_name = _normalize_name(payload.name) @@ -1036,7 +1086,7 @@ def _name_for_source(source: str) -> Optional[str]: diarisation_llm_provider=diarisation_llm_provider_norm, diarisation_llm_model=diarisation_llm_model_norm, diarisation_llm_credential_id=( - payload.diarization_llm_credential_id + payload.diarization_llm_credential_id if auto_transcribe else None ), diarisation_prompt=diarisation_prompt_norm, transcribe_mode=transcribe_mode_norm, @@ -1658,9 +1708,9 @@ def _build_query(session: Session): total = query.count() rows = query.offset((page - 1) * page_size).limit(page_size).all() - # Row detail shows the diarised transcript that normal metrics score. + # Row detail shows the transcript for this run's chosen source. items: List[CallImportEvaluationRowResponse] = [ - _to_evaluation_row_response(eval_row_obj, source_row) + _to_evaluation_row_response(eval_row_obj, source_row, eval_row) for eval_row_obj, source_row in rows ] @@ -3981,7 +4031,7 @@ async def cancel_call_import_evaluation_row( _rollup_evaluation_status(evaluation, db) db.commit() row_db.refresh(eval_row) - return _to_evaluation_row_response(eval_row, source_row) + return _to_evaluation_row_response(eval_row, source_row, evaluation) except LookupError as exc: raise HTTPException( status_code=404, detail="Evaluation row not found in this run" @@ -4012,7 +4062,7 @@ async def cancel_call_import_evaluation_row( .first() ) - return _to_evaluation_row_response(eval_row, source_row) + return _to_evaluation_row_response(eval_row, source_row, evaluation) @router.delete( @@ -8808,11 +8858,11 @@ async def retry_call_import_evaluation_row( source_row, _shard_id, ): - return _to_evaluation_row_response(eval_row, source_row) + return _to_evaluation_row_response(eval_row, source_row, evaluation) db.refresh(eval_row) source_row = targets[0][1] - return _to_evaluation_row_response(eval_row, source_row) + return _to_evaluation_row_response(eval_row, source_row, evaluation) from app.core.auth.capabilities import EVALS_RUN, EVALS_VIEW, REPORTS_GENERATE diff --git a/app/api/v1/routes/call_imports.py b/app/api/v1/routes/call_imports.py index 20cd2b16..e7461504 100644 --- a/app/api/v1/routes/call_imports.py +++ b/app/api/v1/routes/call_imports.py @@ -2975,6 +2975,46 @@ async def delete_call_import( ) +def _locate_call_import_row_or_404( + catalog_db: Session, + *, + call_import_id: UUID, + row_id: UUID, + organization_id: UUID, +) -> Tuple[Session, CallImportRow, Optional[Session]]: + """Find a call import row on the correct DB session for mutation. + + When sharding is enabled rows live on shard databases; ``get_db`` only + opens the catalog. Returns ``(row_db, row, extra_catalog_to_close)`` + where ``extra_catalog_to_close`` is the catalog session opened by + :func:`locate_call_import_row` (distinct from the route's catalog + session) and must be closed via :func:`close_row_sessions`. + """ + from app.db_sharding.row_ops import close_row_sessions, locate_call_import_row + + try: + row_db, located_catalog, row, _shard_id = locate_call_import_row(row_id) + except LookupError: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import row not found", + ) from None + if ( + row.call_import_id != call_import_id + or row.organization_id != organization_id + ): + close_row_sessions( + row_db, + located_catalog if located_catalog is not row_db else None, + ) + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import row not found", + ) + extra_catalog = located_catalog if located_catalog is not row_db else None + return row_db, row, extra_catalog + + @router.delete( "/{call_import_id}/rows/{row_id}", status_code=status.HTTP_204_NO_CONTENT, @@ -3009,38 +3049,35 @@ async def delete_call_import_row( detail="Call import not found", ) - row = ( - db.query(CallImportRow) - .filter( - CallImportRow.id == row_id, - CallImportRow.call_import_id == call_import.id, - ) - .first() - ) - if not row: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import row not found", - ) + from app.db_sharding.row_ops import close_row_sessions - _revoke_pending_tasks([row]) + row_db, row, extra_catalog = _locate_call_import_row_or_404( + db, + call_import_id=call_import.id, + row_id=row_id, + organization_id=organization_id, + ) + try: + _revoke_pending_tasks([row]) - if row.recording_s3_key and s3_service.is_enabled(): - try: - s3_service.delete_file_by_key(row.recording_s3_key) - except Exception as exc: # noqa: BLE001 — best-effort, DB is source of truth - logger.warning( - "Failed to delete S3 object {} for row {}: {}", - row.recording_s3_key, - row.id, - exc, - ) + if row.recording_s3_key and s3_service.is_enabled(): + try: + s3_service.delete_file_by_key(row.recording_s3_key) + except Exception as exc: # noqa: BLE001 — best-effort, DB is source of truth + logger.warning( + "Failed to delete S3 object {} for row {}: {}", + row.recording_s3_key, + row.id, + exc, + ) - db.delete(row) - db.flush() + row_db.delete(row) + row_db.commit() - _recompute_call_import_counters(db, call_import) - db.commit() + _recompute_call_import_counters(db, call_import) + db.commit() + finally: + close_row_sessions(row_db, extra_catalog) logger.info( "Deleted call_import_row {} (call_import={}, org={})", @@ -3529,24 +3566,21 @@ async def cancel_call_import_row_diarisation( detail="Call import not found", ) - row = ( - db.query(CallImportRow) - .filter( - CallImportRow.id == row_id, - CallImportRow.call_import_id == call_import_id, - ) - .first() - ) - if not row: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import row not found", - ) + from app.db_sharding.row_ops import close_row_sessions - _apply_diarisation_cancel([row]) - db.commit() - db.refresh(row) - return CallImportRowResponse.model_validate(row) + row_db, row, extra_catalog = _locate_call_import_row_or_404( + db, + call_import_id=call_import_id, + row_id=row_id, + organization_id=organization_id, + ) + try: + _apply_diarisation_cancel([row]) + row_db.commit() + row_db.refresh(row) + return CallImportRowResponse.model_validate(row) + finally: + close_row_sessions(row_db, extra_catalog) @router.post( @@ -3702,43 +3736,40 @@ async def toggle_call_import_row_speaker_swap( detail="Call import not found", ) - row = ( - db.query(CallImportRow) - .filter( - CallImportRow.id == row_id, - CallImportRow.call_import_id == call_import_id, - CallImportRow.organization_id == organization_id, - ) - .first() + from app.db_sharding.row_ops import close_row_sessions + + row_db, row, extra_catalog = _locate_call_import_row_or_404( + db, + call_import_id=call_import_id, + row_id=row_id, + organization_id=organization_id, ) - if not row: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import row not found", + try: + segments = ( + row.diarised_segments if isinstance(row.diarised_segments, list) else None ) + if not segments: + # Without structured turns the swap toggle would have nothing to + # re-render — surface a clear error rather than silently + # flipping a flag the UI never read. + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + "This row has no structured diarised segments to swap. " + "Re-run diarisation to generate per-speaker turns first." + ), + ) - segments = row.diarised_segments if isinstance(row.diarised_segments, list) else None - if not segments: - # Without structured turns the swap toggle would have nothing to - # re-render — surface a clear error rather than silently - # flipping a flag the UI never read. - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - "This row has no structured diarised segments to swap. " - "Re-run diarisation to generate per-speaker turns first." - ), + new_swap = not bool(row.diarised_speaker_swap) + row.diarised_speaker_swap = new_swap + row.diarised_transcript = ( + _render_diarised_segments_text(segments, swap=new_swap) or None ) - - new_swap = not bool(row.diarised_speaker_swap) - row.diarised_speaker_swap = new_swap - row.diarised_transcript = ( - _render_diarised_segments_text(segments, swap=new_swap) or None - ) - db.commit() - db.refresh(row) - - return CallImportRowResponse.model_validate(row) + row_db.commit() + row_db.refresh(row) + return CallImportRowResponse.model_validate(row) + finally: + close_row_sessions(row_db, extra_catalog) # --------------------------------------------------------------------------- diff --git a/app/config/models.json b/app/config/models.json index d6fb9fb1..c3324c3a 100644 --- a/app/config/models.json +++ b/app/config/models.json @@ -305,6 +305,11 @@ "featured_rank": 7, "highlights": ["GA stable", "1M context", "Agentic coding"] }, + "gemini-3.5-flash-lite": { + "provider": "google", + "model_type": "llm", + "description": "Gemini 3.5 Flash Lite \u2014 fastest, lowest-cost 3.5 model for high-throughput extraction, classification, and subagent workflows" + }, "gemini-3.1-flash-lite": { "provider": "google", "model_type": "llm", diff --git a/app/db_sharding/scatter_gather.py b/app/db_sharding/scatter_gather.py index 917acc31..931aa57a 100644 --- a/app/db_sharding/scatter_gather.py +++ b/app/db_sharding/scatter_gather.py @@ -779,6 +779,61 @@ def count_shard(db: Session, _shard_id: str) -> int: return total +def _non_empty_production_transcript_filter(): + """Rows whose CSV ``transcript`` column has non-whitespace content.""" + return ( + CallImportRow.transcript.isnot(None), + func.length(func.trim(CallImportRow.transcript)) > 0, + ) + + +def count_call_import_rows_with_production_transcript( + catalog_db: Session, + call_import_id: UUID, +) -> int: + """Import rows with a non-empty production transcript (shard-aware).""" + transcript_filters = _non_empty_production_transcript_filter() + + if not is_sharding_enabled(): + return int( + catalog_db.query(func.count(CallImportRow.id)) + .filter( + CallImportRow.call_import_id == call_import_id, + *transcript_filters, + ) + .scalar() + or 0 + ) + + total = 0 + shard_ids = shard_ids_for_import(catalog_db, call_import_id) + + def count_shard(db: Session, _shard_id: str) -> int: + return int( + db.query(func.count(CallImportRow.id)) + .filter( + CallImportRow.call_import_id == call_import_id, + *transcript_filters, + ) + .scalar() + or 0 + ) + + for part in scatter_gather_on_shards(shard_ids, count_shard): + total += int(part) + if total == 0: + total = int( + catalog_db.query(func.count(CallImportRow.id)) + .filter( + CallImportRow.call_import_id == call_import_id, + *transcript_filters, + ) + .scalar() + or 0 + ) + return total + + def count_completed_call_import_rows( catalog_db: Session, call_import_id: UUID, diff --git a/app/models/schemas.py b/app/models/schemas.py index 59c3ced2..10a4d5b7 100644 --- a/app/models/schemas.py +++ b/app/models/schemas.py @@ -3000,11 +3000,10 @@ class CallImportEvaluationCreate(BaseModel): min_length=1, max_length=1, description=( - "Which transcript to score against. Only ``diarised`` is " - "supported — every evaluation run scores the diarised " - "transcript. Pass ``['diarised']`` explicitly or omit the " - "field to take the default; any other value (including the " - "legacy ``'production'`` source) is rejected with a 400." + "Which transcript to score against. ``'diarised'`` (default) " + "auto-diarises rows missing a diarised transcript then scores " + "``diarised_transcript``. ``'production'`` scores the CSV " + "``transcript`` column directly and skips diarisation." ), ) @@ -3013,20 +3012,16 @@ class CallImportEvaluationCreate(BaseModel): def _validate_transcript_sources( cls, value: List[str] ) -> List["CallImportEvaluationTranscriptSource"]: - # The Field min_length/max_length constraints catch empty + - # over-long payloads; this validator's job is to reject any - # non-diarised source value and normalize the result to - # ``['diarised']`` for downstream code. - invalid = [src for src in value if src != "diarised"] + allowed = {"production", "diarised"} + invalid = [src for src in value if src not in allowed] if invalid: raise ValueError( - "Only the 'diarised' transcript source is supported " + "transcript_sources must be ['production'] or ['diarised'] " "(received: " + ", ".join(repr(src) for src in invalid) - + "). Remove 'production' from transcript_sources or " - "omit the field to use the default." + + ")." ) - return ["diarised"] + return value # type: ignore[return-value] # --- Run-level LLM config --- llm_provider: Optional[str] = Field( default=None, diff --git a/app/services/call_import_user_insights.py b/app/services/call_import_user_insights.py index e1947a0b..bf723e6a 100644 --- a/app/services/call_import_user_insights.py +++ b/app/services/call_import_user_insights.py @@ -143,9 +143,12 @@ def _pick_transcript( evaluation: CallImportEvaluation, source_row: CallImportRow, ) -> str: - del evaluation - diarised = (source_row.diarised_transcript or "").strip() - return diarised[:ROW_TRANSCRIPT_CHAR_CAP] + source = (evaluation.transcript_source or "diarised").strip().lower() + if source == "production": + text = (source_row.transcript or "").strip() + else: + text = (source_row.diarised_transcript or "").strip() + return text[:ROW_TRANSCRIPT_CHAR_CAP] def _metric_name_map(metrics: Sequence[Metric]) -> Dict[str, str]: diff --git a/app/services/call_imports/bulk_ops.py b/app/services/call_imports/bulk_ops.py index cbed6ceb..2d83faf5 100644 --- a/app/services/call_imports/bulk_ops.py +++ b/app/services/call_imports/bulk_ops.py @@ -199,6 +199,16 @@ def count_completed_source_rows(db: Session, call_import_id: UUID) -> int: return count_completed_call_import_rows(db, call_import_id) +def count_source_rows_with_production_transcript( + db: Session, call_import_id: UUID +) -> int: + from app.db_sharding.scatter_gather import ( + count_call_import_rows_with_production_transcript, + ) + + return count_call_import_rows_with_production_transcript(db, call_import_id) + + def _completed_source_row_ids(db: Session, call_import_id: UUID) -> List[UUID]: from app.db_sharding.scatter_gather import list_completed_source_row_ids_ordered @@ -670,7 +680,19 @@ def execute_call_import_materialization( return {"total_rows": 0, "status": "failed"} try: + logger.info( + "execute_call_import_materialization downloading staged source " + "(call_import={} key={})", + call_import_id, + call_import.source_s3_key, + ) file_bytes = s3_service.download_file_by_key(call_import.source_s3_key) + logger.info( + "execute_call_import_materialization downloaded {} bytes " + "(call_import={})", + len(file_bytes), + call_import_id, + ) except StorageError as exc: call_import.status = CallImportStatus.FAILED call_import.error_message = f"Could not read staged source file from S3: {exc}" diff --git a/app/services/reporting/call_import_evaluation_pdf_report.py b/app/services/reporting/call_import_evaluation_pdf_report.py index 1a3a3d82..64c53df7 100644 --- a/app/services/reporting/call_import_evaluation_pdf_report.py +++ b/app/services/reporting/call_import_evaluation_pdf_report.py @@ -1373,7 +1373,7 @@ def _failure_diagnostics_section_html( if not metrics_body and not discovered_section: metrics_body = "

No cluster groups in this run.

" return f""" -
+

04 Failure Diagnostics

Per-metric clustering of flagged calls with engineering gap labels (LOGIC_GAP, UNDERSPEC, EXISTS_NO_TRIGGER, MISSING) and Level-2 sub-categories.

@@ -1485,7 +1485,7 @@ def _prompt_improvements_section_html( """ ) return f""" -
+

05 Prompt Improvement Recommendations

Top {_PDF_PROMPT_IMPROVEMENTS_MAX} recommended prompt changes for {html.escape(agent_name)}, aligned with the Visualizations → Prompt / Agent Improvements view.

@@ -2005,7 +2005,7 @@ def _render_html(self, payload: dict[str, Any]) -> str: """ ) quality_panel_markup = f""" -
+

02 Quality Metric Panel

{''.join(quality_groups_markup)}
@@ -2104,7 +2104,7 @@ def _render_html(self, payload: dict[str, Any]) -> str: else "User insights are identified by analyzing patterns across LLM rationales and diarized transcripts from the evaluation run." ) business_section = f""" -
+

03 User Insights

{intro}

@@ -2137,7 +2137,7 @@ def _render_html(self, payload: dict[str, Any]) -> str: ) if notes: design_notes_section = f""" -
+

05 User Experience Design Notes

    {notes}
@@ -2169,6 +2169,11 @@ def _render_html(self, payload: dict[str, Any]) -> str: if internal_logo_uri else "" ) + report_body_class = ( + "report-body internal-report" + if is_internal + else "report-body external-report" + ) return f""" @@ -2203,17 +2208,20 @@ def _render_html(self, payload: dict[str, Any]) -> str: .eyebrow .muted {{ color: #7a756d; }} h1 {{ font-size: 28px; font-weight: 800; margin: 4px 0; letter-spacing: .1px; }} .subtitle {{ font-size: 13px; color: #666; margin-bottom: 10px; }} - h2 {{ font-size: 16px; margin: 16px 0 8px; color: #0b1220; border-bottom: 2px solid #0b1220; padding-bottom: 2px; }} - h3 {{ font-size: 13px; margin: 0; }} + h2 {{ font-size: 16px; margin: 8px 0 6px; color: #0b1220; border-bottom: 2px solid #0b1220; padding-bottom: 2px; break-after: avoid; page-break-after: avoid; }} + h3 {{ font-size: 13px; margin: 0; break-after: avoid; page-break-after: avoid; }} + .report-body section {{ margin: 0 0 10px; padding: 0; }} + .report-body section > .method:first-of-type {{ margin-top: 0; margin-bottom: 6px; }} + .metric-group {{ margin-bottom: 8px; break-inside: auto; page-break-inside: auto; }} .meta {{ display: grid; grid-template-columns: repeat(5, 1fr); gap: 10px; }} .summary {{ display: grid; grid-template-columns: repeat(4, 1fr); gap: 10px; }} - .metric-group-title {{ font-size: 13px; margin: 12px 0 8px; color: #374151; text-transform: uppercase; letter-spacing: .08em; }} + .metric-group-title {{ font-size: 13px; margin: 8px 0 6px; color: #374151; text-transform: uppercase; letter-spacing: .08em; break-after: avoid; page-break-after: avoid; }} .box {{ border: 1px solid #d9dee8; padding: 7px 8px; border-radius: 6px; background: #f7f9fc; }} .box strong {{ display: block; font-size: 13px; color: #111827; }} .box span {{ color: #667085; font-size: 9px; text-transform: uppercase; letter-spacing: .04em; }} - .metric {{ break-inside: avoid; border: 1px solid #d9dee8; border-radius: 2px; padding: 10px; margin-bottom: 10px; background: #fff; }} - .metric-compact-grid {{ display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 10px; margin-bottom: 8px; }} - .metric-compact {{ padding: 9px 10px 8px; margin-bottom: 0; min-height: 148px; }} + .metric {{ break-inside: avoid; page-break-inside: avoid; border: 1px solid #d9dee8; border-radius: 2px; padding: 10px; margin-bottom: 8px; background: #fff; }} + .metric-compact-grid {{ display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 8px; margin-bottom: 6px; break-inside: auto; page-break-inside: auto; }} + .metric-compact {{ padding: 9px 10px 8px; margin-bottom: 0; min-height: 0; break-inside: avoid; page-break-inside: avoid; }} .metric-compact-head {{ display: flex; justify-content: space-between; align-items: flex-start; gap: 8px; margin-bottom: 6px; }} .metric-compact-head .metric-title {{ font-size: 9px; letter-spacing: .08em; color: #667085; font-weight: 800; }} .metric-compact-status {{ width: 7px; height: 7px; border-radius: 50%; background: #16a34a; margin-top: 2px; flex-shrink: 0; }} @@ -2283,16 +2291,16 @@ def _render_html(self, payload: dict[str, Any]) -> str: .dist-fill {{ height: 100%; background: #c7725e; }} .dist-value {{ text-align: right; font-size: 10px; font-weight: 800; color: #111827; }} .empty-bars {{ color: #667085; font-size: 10px; font-style: italic; }} - .insight-block {{ break-inside: avoid; margin: 14px 0 18px; }} - .insight-heading {{ margin-bottom: 8px; }} + .insight-block {{ break-inside: auto; page-break-inside: auto; margin: 8px 0 10px; }} + .insight-heading {{ margin-bottom: 6px; break-after: avoid; page-break-after: avoid; }} .insight-heading h3 {{ margin: 0; }} - .insight-body {{ display: grid; grid-template-columns: 1.15fr 0.85fr; gap: 14px; align-items: start; }} - .insight-table-col {{ min-width: 0; }} - .insight-sidecol {{ display: grid; gap: 10px; min-width: 0; }} - .insight-overview-box {{ break-inside: avoid; margin: 0 0 16px; padding: 12px 14px; border: 1px solid #e7ddd1; border-radius: 8px; background: #faf7f2; }} + .insight-body {{ display: grid; grid-template-columns: 1.15fr 0.85fr; gap: 10px; align-items: start; break-inside: auto; page-break-inside: auto; }} + .insight-table-col {{ min-width: 0; break-inside: auto; page-break-inside: auto; }} + .insight-sidecol {{ display: grid; gap: 8px; min-width: 0; }} + .insight-overview-box {{ break-inside: auto; page-break-inside: auto; margin: 0 0 10px; padding: 10px 12px; border: 1px solid #e7ddd1; border-radius: 8px; background: #faf7f2; }} .insight-overview-label {{ font-size: 10px; text-transform: uppercase; letter-spacing: .08em; color: #7a756d; font-weight: 800; margin-bottom: 6px; }} .insight-overview-box p {{ margin: 0; color: #374151; line-height: 1.5; }} - .insight-callout {{ border: 1px solid #eadfce; border-radius: 8px; padding: 10px 12px; }} + .insight-callout {{ break-inside: avoid; page-break-inside: avoid; border: 1px solid #eadfce; border-radius: 8px; padding: 10px 12px; }} .insight-observation-box {{ background: #fdf3f0; border-color: #f0d4cb; }} .insight-evidence-box {{ background: #faf6ef; border-color: #eadfce; }} .insight-callout-label {{ font-size: 10px; text-transform: uppercase; letter-spacing: .08em; color: #b85f4b; font-weight: 800; margin-bottom: 6px; }} @@ -2302,7 +2310,7 @@ def _render_html(self, payload: dict[str, Any]) -> str: .insight-table td:nth-child(2), .insight-table td:nth-child(4) {{ white-space: nowrap; font-weight: 800; }} .insight-track {{ height: 9px; background: #e8edff; border: 1px solid #cfd8ff; min-width: 120px; }} .insight-fill {{ height: 100%; background: #4f46e5; }} - .failure-diagnostics {{ margin-top: 4px; max-width: 100%; }} + .failure-diagnostics {{ margin-top: 0; max-width: 100%; break-inside: auto; page-break-inside: auto; }} .failure-diagnostics-intro {{ break-inside: auto; margin-bottom: 6px; max-width: 100%; }} .failure-diagnostics-intro h2 {{ margin-bottom: 4px; }} .failure-diagnostics .method {{ margin: 0 0 6px; }} @@ -2476,27 +2484,29 @@ def _render_html(self, payload: dict[str, Any]) -> str: .design-notes {{ margin: 8px 0 0; padding-left: 18px; }} .design-notes li {{ margin-bottom: 8px; line-height: 1.5; }} .gap-badge {{ display: inline-block; font-size: 9px; font-weight: 800; letter-spacing: .04em; color: #b42318; text-transform: uppercase; }} - table {{ width: 100%; border-collapse: collapse; margin-top: 8px; }} + table {{ width: 100%; border-collapse: collapse; margin-top: 6px; break-inside: auto; page-break-inside: auto; }} + thead {{ display: table-header-group; }} + tr {{ break-inside: avoid; page-break-inside: avoid; }} th, td {{ text-align: left; border-bottom: 1px solid #e5e7eb; padding: 7px; vertical-align: top; }} th {{ background: #111827; color: #fff; font-size: 10px; text-transform: uppercase; }} .method {{ color: #475467; line-height: 1.5; }} .audit-summary p {{ margin: 0 0 7px; }} .audit-summary ul {{ margin: 7px 0 10px; padding-left: 16px; }} .audit-summary li {{ margin-bottom: 5px; }} - .audit-summary-section {{ margin-bottom: 0; break-after: avoid; page-break-after: avoid; }} + .audit-summary-section {{ margin-bottom: 0; break-after: auto; page-break-after: auto; }} .audit-summary-section + section {{ margin-top: 0; }} - .audit-summary-section + section h2 {{ margin-top: 12px; }} - .audit-stat-strip {{ display: flex; gap: 0; margin: 14px 0 0; border-top: 1px solid #111827; break-after: avoid; page-break-after: avoid; }} - .audit-stat-strip:last-child {{ margin-bottom: 18px; }} + .audit-summary-section + section h2 {{ margin-top: 8px; }} + .audit-stat-strip {{ display: flex; gap: 0; margin: 10px 0 0; border-top: 1px solid #111827; break-after: auto; page-break-after: auto; }} + .audit-stat-strip:last-child {{ margin-bottom: 10px; }} .audit-stat-card {{ flex: 1; min-width: 0; padding: 10px 12px 12px 0; }} .audit-stat-card + .audit-stat-card {{ border-left: 1px solid #d1d5db; padding-left: 12px; }} .audit-stat-rule {{ display: none; }} .audit-stat-label {{ font-size: 8px; font-weight: 800; letter-spacing: .08em; text-transform: uppercase; color: #d16532; margin-bottom: 6px; line-height: 1.3; }} .audit-stat-value {{ font-family: Georgia, 'Times New Roman', serif; font-size: 28px; font-weight: 700; color: #111827; line-height: 1; }} - .audit-delta-panel {{ break-inside: auto; page-break-inside: auto; margin: 8px 0 12px; padding-top: 8px; border-top: 1px solid #e5e7eb; }} - .audit-delta-title {{ font-size: 13px; font-weight: 900; letter-spacing: .04em; text-transform: uppercase; color: #111827; margin-bottom: 8px; }} - .audit-delta-grid {{ display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 8px; }} - .audit-delta-card {{ break-inside: auto; page-break-inside: auto; border: 1px solid #d9dee8; border-radius: 6px; padding: 8px 9px; background: #f8fafc; min-height: 96px; }} + .audit-delta-panel {{ break-inside: auto; page-break-inside: auto; margin: 6px 0 8px; padding-top: 6px; border-top: 1px solid #e5e7eb; }} + .audit-delta-title {{ font-size: 13px; font-weight: 900; letter-spacing: .04em; text-transform: uppercase; color: #111827; margin-bottom: 6px; break-after: avoid; page-break-after: avoid; }} + .audit-delta-grid {{ display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 8px; break-inside: auto; page-break-inside: auto; }} + .audit-delta-card {{ break-inside: avoid; page-break-inside: avoid; border: 1px solid #d9dee8; border-radius: 6px; padding: 8px 9px; background: #f8fafc; min-height: 0; }} .audit-delta-head {{ display: flex; justify-content: space-between; align-items: flex-start; gap: 8px; margin-bottom: 4px; }} .audit-delta-head-main {{ min-width: 0; }} .audit-delta-head span {{ display: block; color: #111827; font-size: 9px; font-weight: 900; letter-spacing: .06em; line-height: 1.25; text-transform: uppercase; overflow-wrap: anywhere; }} @@ -2509,12 +2519,17 @@ def _render_html(self, payload: dict[str, Any]) -> str: .audit-delta-chart .metric-sparkline {{ width: 100%; max-width: 100%; }} .audit-delta-empty {{ height: 50px; display: flex; align-items: center; justify-content: center; border: 1px dashed #cbd5e1; background: #fff; color: #667085; font-size: 9px; }} .audit-delta-reason {{ margin: 6px 0 0; padding-top: 5px; border-top: 1px solid #e5e7eb; color: #374151; font-size: 10px; font-weight: 600; line-height: 1.45; }} - .report-footer {{ margin-top: 24px; border-top: 1px solid #1f2937; padding-top: 10px; display: flex; justify-content: space-between; align-items: flex-start; gap: 16px; color: #4b5563; font-size: 10px; }} + .report-footer {{ margin-top: 14px; border-top: 1px solid #1f2937; padding-top: 10px; display: flex; justify-content: space-between; align-items: flex-start; gap: 16px; color: #4b5563; font-size: 10px; break-inside: avoid; page-break-inside: avoid; }} .footer-left, .footer-right {{ display: flex; flex-direction: column; gap: 3px; }} .footer-right {{ text-align: right; }} .brand-title {{ color: #111827; font-weight: 800; font-size: 11px; letter-spacing: .2px; }} .brand-link {{ color: #d16532; font-weight: 700; text-decoration: none; }} - .prompt-improvements {{ margin-top: 4px; max-width: 100%; }} + .internal-report .insight-block {{ margin: 6px 0 8px; }} + .internal-report .failure-diagnostics-intro h2, + .internal-report .prompt-improvements > h2 {{ + margin-top: 8px; + }} + .prompt-improvements {{ margin-top: 0; max-width: 100%; break-inside: auto; page-break-inside: auto; }} .prompt-improvement-card {{ margin: 0 0 10px; padding: 8px 10px; @@ -2627,7 +2642,7 @@ def _render_html(self, payload: dict[str, Any]) -> str: }} - + {repeat_header_markup}
{brand_header_markup} @@ -2644,7 +2659,7 @@ def _render_html(self, payload: dict[str, Any]) -> str:
{f''' -
+

01 Audit Summary

{audit_summary_markup}
{top_metric_strip_markup} @@ -2657,8 +2672,7 @@ def _render_html(self, payload: dict[str, Any]) -> str: {prompt_improvements_section} {design_notes_section} {f''' -
-

Methodology

+

This report is generated from completed Call Import evaluation results. Metrics are derived from the saved evaluation outputs and the transcript source configured for the run.{html.escape(weekly_methodology)}

''' if show_methodology else ''} diff --git a/app/workers/concurrency/eval_dispatch.py b/app/workers/concurrency/eval_dispatch.py index 7afbd735..89debff8 100644 --- a/app/workers/concurrency/eval_dispatch.py +++ b/app/workers/concurrency/eval_dispatch.py @@ -64,6 +64,13 @@ def _needs_transcribe_for_eval( transcribe_overwrite: bool, auto_transcribe: bool = True, ) -> bool: + if ( + (getattr(evaluation, "transcript_source", None) or "") + .strip() + .lower() + == "production" + ): + return False if not auto_transcribe: return False transcribe_mode = ( @@ -212,6 +219,13 @@ def enqueue_eval_chain_transcribe_after_import( transcribe_overwrite: bool = False, ) -> bool: """Directly chain diarisation after a successful eval-chain recording fetch.""" + if not _needs_transcribe_for_eval( + evaluation, + source_row, + transcribe_overwrite=transcribe_overwrite, + ): + return False + from app.db_sharding.row_ops import shard_row_write_context recover_eval_row_for_eval_chain(eval_row) diff --git a/app/workers/concurrency/fair_dispatch.py b/app/workers/concurrency/fair_dispatch.py index 913734ea..55e4bbc7 100644 --- a/app/workers/concurrency/fair_dispatch.py +++ b/app/workers/concurrency/fair_dispatch.py @@ -352,6 +352,12 @@ def _dispatch_batch_for_workspace( for eval_row, source_row, evaluation in pending: restricted_metric_ids = get_row_restricted_metrics(eval_row.id) transcribe_overwrite = evaluation_transcribe_overwrite(evaluation.id) + eval_auto_transcribe = ( + (getattr(evaluation, "transcript_source", None) or "diarised") + .strip() + .lower() + != "production" + ) outcome = _try_dispatch_single_row( db=db, evaluation=evaluation, @@ -359,7 +365,7 @@ def _dispatch_batch_for_workspace( source_row=source_row, restricted_metric_ids=restricted_metric_ids, transcribe_overwrite=transcribe_overwrite, - auto_transcribe=True, + auto_transcribe=eval_auto_transcribe, call_import=call_import, shard_cache=shard_cache, ) diff --git a/app/workers/tasks/call_import_bulk_ops.py b/app/workers/tasks/call_import_bulk_ops.py index d36cccde..866cb215 100644 --- a/app/workers/tasks/call_import_bulk_ops.py +++ b/app/workers/tasks/call_import_bulk_ops.py @@ -114,6 +114,12 @@ def materialize_mapped_call_import_evaluation_task( block the Run Evaluation request. """ del self + logger.info( + "materialize_mapped_call_import_evaluation starting " + "(call_import={} evaluation={})", + call_import_id, + evaluation_id, + ) db = SessionLocal() eval_uuid = UUID(evaluation_id) try: @@ -124,6 +130,14 @@ def materialize_mapped_call_import_evaluation_task( UUID(workspace_id), schedule_import_dispatch=False, ) + logger.info( + "materialize_mapped_call_import_evaluation materialization finished " + "(call_import={} evaluation={} status={} total_rows={})", + call_import_id, + evaluation_id, + mat_result.get("status"), + mat_result.get("total_rows"), + ) status = mat_result.get("status") if status == "failed": call_import = ( diff --git a/app/workers/tasks/evaluate_call_import_row.py b/app/workers/tasks/evaluate_call_import_row.py index 9ccb9dbb..29247d24 100644 --- a/app/workers/tasks/evaluate_call_import_row.py +++ b/app/workers/tasks/evaluate_call_import_row.py @@ -377,8 +377,15 @@ def evaluate_call_import_row_task( production_transcript = (source_row.transcript or "").strip() diarised_transcript = (source_row.diarised_transcript or "").strip() - transcript = diarised_transcript - missing_label = "diarised" + eval_source = ( + (evaluation.transcript_source or "diarised").strip().lower() + ) + if eval_source == "production": + transcript = production_transcript + missing_label = "production" + else: + transcript = diarised_transcript + missing_label = "diarised" recording_s3_key = (source_row.recording_s3_key or "").strip() or None has_audio = recording_s3_key is not None diff --git a/app/workers/tasks/process_call_import_row.py b/app/workers/tasks/process_call_import_row.py index 5138cc3a..09f0feae 100644 --- a/app/workers/tasks/process_call_import_row.py +++ b/app/workers/tasks/process_call_import_row.py @@ -548,6 +548,7 @@ def process_call_import_row_task( if run_eval_row_id: from app.models.database import CallImportEvaluation, CallImportEvaluationRow from app.workers.concurrency.eval_dispatch import ( + _needs_transcribe_for_eval, enqueue_eval_chain_transcribe_after_import, ) @@ -568,7 +569,11 @@ def process_call_import_row_task( .filter(CallImportEvaluation.id == eval_row.evaluation_id) .first() ) - if evaluation is not None: + if evaluation is not None and _needs_transcribe_for_eval( + evaluation, + row, + transcribe_overwrite=False, + ): enqueue_eval_chain_transcribe_after_import( db, evaluation=evaluation, diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 4f66da62..5ab03863 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -1629,9 +1629,9 @@ class ApiClient { metric_ids: string[] name?: string | null /** - * Which transcript(s) to score against. Passing both values triggers - * two evaluation runs (one per source). Defaults server-side to - * `['production']` for backwards compatibility. + * Which transcript to score against. ``'diarised'`` (default) + * auto-diarizes then scores; ``'production'`` scores the CSV + * transcript directly. */ transcript_sources?: Array<'production' | 'diarised'> /** Run-level LLM provider key. Leave undefined for legacy default. */ diff --git a/frontend/src/lib/clipboard.ts b/frontend/src/lib/clipboard.ts new file mode 100644 index 00000000..f5ca753b --- /dev/null +++ b/frontend/src/lib/clipboard.ts @@ -0,0 +1,49 @@ +/** + * Copy text to the system clipboard with a secure-context fallback. + * + * ``navigator.clipboard`` is async and requires a secure context. Falls + * back to ``document.execCommand('copy')`` so LAN dev over HTTP still + * works. + */ +export function copyTextToClipboard( + text: string, + onSuccess?: () => void, +): void { + if (!text) return + + const finalize = () => { + onSuccess?.() + } + + const fallbackCopy = () => { + try { + const ta = document.createElement('textarea') + ta.value = text + ta.style.position = 'fixed' + ta.style.opacity = '0' + document.body.appendChild(ta) + ta.select() + document.execCommand('copy') + document.body.removeChild(ta) + finalize() + } catch { + // Swallow — user can still drag-select visible text. + } + } + + if (navigator.clipboard?.writeText) { + navigator.clipboard.writeText(text).then(finalize).catch(fallbackCopy) + } else { + fallbackCopy() + } +} + +/** + * Read text from the clipboard. Requires a secure context and permission. + */ +export async function readTextFromClipboard(): Promise { + if (!navigator.clipboard?.readText) { + throw new Error('Clipboard read is not supported in this browser.') + } + return navigator.clipboard.readText() +} diff --git a/frontend/src/pages/callImports/CallImportDetail.tsx b/frontend/src/pages/callImports/CallImportDetail.tsx index 86a27dcb..bc8687b8 100644 --- a/frontend/src/pages/callImports/CallImportDetail.tsx +++ b/frontend/src/pages/callImports/CallImportDetail.tsx @@ -40,6 +40,7 @@ import { Loader2, } from 'lucide-react' import { apiClient } from '../../lib/api' +import { copyTextToClipboard } from '../../lib/clipboard' import { getApiErrorMessage } from '../../lib/apiErrors' import { useToast } from '../../hooks/useToast' import { formatDiarisationError } from '../../lib/diarisationErrors' @@ -312,62 +313,47 @@ export default function CallImportDetail() { // exact button that was clicked without affecting the rest of the // table. Auto-clears after 1.5s via a setTimeout inside the handler. const [copiedRowId, setCopiedRowId] = useState(null) + const [copiedTranscriptField, setCopiedTranscriptField] = useState<{ + rowId: string + field: 'production' | 'diarised' + } | null>(null) + const handleCopyConversationId = ( row: CallImportRow, event: React.MouseEvent, ) => { - // Stop the synthetic click from bubbling up to the expand button - // that wraps the row header — otherwise copying would also toggle - // the row open/closed. event.preventDefault() event.stopPropagation() const text = row.conversation_id || '' if (!text) return - const finalize = () => { + copyTextToClipboard(text, () => { setCopiedRowId(row.id) window.setTimeout(() => { - // Only clear if THIS row is still the active one — a quick - // double-click on two different rows shouldn't prematurely - // wipe the badge on the second. - setCopiedRowId((prev) => (prev === row.id ? null : prev)) + setCopiedRowId((current) => (current === row.id ? null : current)) }, 1500) - } - // ``navigator.clipboard`` is async + requires a secure context. - // Fall back to the legacy ``execCommand`` path so localhost-over- - // http (e.g. ``http://10.x.x.x:5173`` LAN dev) still works. - if (navigator.clipboard?.writeText) { - navigator.clipboard.writeText(text).then(finalize).catch(() => { - // Best-effort fallback if the async API rejects (permission / - // not focused / unsupported MIME, …). - try { - const ta = document.createElement('textarea') - ta.value = text - ta.style.position = 'fixed' - ta.style.opacity = '0' - document.body.appendChild(ta) - ta.select() - document.execCommand('copy') - document.body.removeChild(ta) - finalize() - } catch { - // Swallow — user can still drag-select the visible text. - } - }) - } else { - try { - const ta = document.createElement('textarea') - ta.value = text - ta.style.position = 'fixed' - ta.style.opacity = '0' - document.body.appendChild(ta) - ta.select() - document.execCommand('copy') - document.body.removeChild(ta) - finalize() - } catch { - // Swallow — drag-select still works. - } - } + }) + } + + const handleCopyTranscript = ( + row: CallImportRow, + field: 'production' | 'diarised', + event: React.MouseEvent, + ) => { + event.preventDefault() + event.stopPropagation() + const text = + field === 'production' + ? row.transcript || '' + : row.diarised_transcript || '' + if (!text) return + copyTextToClipboard(text, () => { + setCopiedTranscriptField({ rowId: row.id, field }) + window.setTimeout(() => { + setCopiedTranscriptField((current) => + current?.rowId === row.id && current?.field === field ? null : current, + ) + }, 1500) + }) } // Row search: debounce keystrokes so we don't refire the rows fetch on // every character — the backend filters by conversation_id ILIKE %q% @@ -470,6 +456,9 @@ export default function CallImportDetail() { const [evalTranscribeMode, setEvalTranscribeMode] = useState< 'stt_llm' | 'llm_only' >('llm_only') + const [evalTranscriptSource, setEvalTranscriptSource] = useState< + 'production' | 'diarised' + >('diarised') const [evalSTT, setEvalSTT] = useState({ provider: null, model: null, @@ -2624,6 +2613,29 @@ export default function CallImportDetail() { )}
+ {row.transcript && ( + + )}
{row.transcript ? ( @@ -2686,6 +2698,29 @@ export default function CallImportDetail() { )}
+ {row.diarised_transcript && ( + + )} {Array.isArray(row.diarised_segments) && row.diarised_segments.length > 0 && ( + +
+ {evalTranscriptSource === 'production' ? ( +

+ Skips diarization and scores the transcript + imported from your CSV. Recordings are still + fetched from Exotel/Plivo when a recording URL + is mapped (needed for audio metrics). +

+ ) : null} + + + {evalTranscriptSource === 'diarised' && + (() => { // In ``llm_only`` mode there is no STT — // the diariser LLM consumes the audio // directly. We adapt the "is the auto- @@ -4502,8 +4586,8 @@ export default function CallImportDetail() { {evalTranscribeMode === 'stt_llm' - ? "Every evaluation scores the diarised transcript. Rows that don't already have one are diarised first via the STT provider you pick below." - : "Every evaluation scores the diarised transcript. Rows that don't already have one are diarised by feeding the audio directly to the multimodal LLM you pick below. Calls longer than about 8 minutes need enough output headroom—if results look cut off, use STT + LLM diariser instead."} + ? "Rows that don't already have a diarised transcript are diarised first via the STT provider you pick below." + : "Rows that don't already have a diarised transcript are diarised by feeding the audio directly to the multimodal LLM you pick below. Calls longer than about 8 minutes need enough output headroom—if results look cut off, use STT + LLM diariser instead."} @@ -4732,28 +4816,27 @@ export default function CallImportDetail() { } else if (selectedMetricIds.length === 0) { disabledReasons.push('Select at least one metric to score.') } - // STT is only required in the legacy two-stage - // mode. ``llm_only`` mode feeds the audio - // straight to the diariser LLM, so the STT - // checks are gated on the active mode. - if (evalTranscribeMode === 'stt_llm') { - if (!evalSTT.provider) { - disabledReasons.push( - 'Pick an STT provider (auto-diarisation is required).', - ) - } else if (!evalSTT.model) { + // STT and diariser are only required when diarising. + if (evalTranscriptSource === 'diarised') { + if (evalTranscribeMode === 'stt_llm') { + if (!evalSTT.provider) { + disabledReasons.push( + 'Pick an STT provider (auto-diarisation is required).', + ) + } else if (!evalSTT.model) { + disabledReasons.push( + 'Pick an STT model for the selected provider.', + ) + } + } + if (!isLLMSelectionComplete(evalDiariserLLM, aiProviders)) { disabledReasons.push( - 'Pick an STT model for the selected provider.', + evalTranscribeMode === 'llm_only' + ? 'Pick a multimodal LLM provider — the recording is fed to it directly in LLM-only mode.' + : 'Pick a diariser LLM provider and model (or a Custom gateway credential).', ) } } - if (!isLLMSelectionComplete(evalDiariserLLM, aiProviders)) { - disabledReasons.push( - evalTranscribeMode === 'llm_only' - ? 'Pick a multimodal LLM provider — the recording is fed to it directly in LLM-only mode.' - : 'Pick a diariser LLM provider and model (or a Custom gateway credential).', - ) - } if ( aiProviders.length > 0 && isLLMSelectionPartial(runLLM, aiProviders) @@ -4841,9 +4924,7 @@ export default function CallImportDetail() { runEvaluationMutation.mutate({ metric_ids: selectedMetricIds, name: runDraftName.trim() || null, - // Diarised is the only supported source - // now; the backend rejects anything else. - transcript_sources: ['diarised'], + transcript_sources: [evalTranscriptSource], llm_provider: runLLMComplete ? runLLM.provider || null : null, @@ -4861,41 +4942,40 @@ export default function CallImportDetail() { metric_llm_overrides: Object.keys(overrides).length ? overrides : null, - // Auto-diarise is always on; ``transcribe_mode`` - // decides whether the STT step actually runs - // (and therefore whether the STT fields are - // sent or nulled out for the backend's - // validator). - auto_transcribe: true, - transcribe_overwrite: transcribeOverwrite, - transcribe_mode: evalTranscribeMode, - stt_provider: - evalTranscribeMode === 'stt_llm' - ? evalSTT.provider - : null, - stt_model: - evalTranscribeMode === 'stt_llm' - ? evalSTT.model - : null, - stt_credential_id: - evalTranscribeMode === 'stt_llm' - ? evalSTT.credential_id || null - : null, - stt_language: - evalTranscribeMode === 'stt_llm' - ? evalSTTLanguage.trim() || null - : null, - diarization_llm_provider: - evalDiariserLLM.provider, - diarization_llm_model: - resolveLLMModelForSubmit( - evalDiariserLLM, - aiProviders, - ) ?? evalDiariserLLM.model, - diarization_llm_credential_id: - evalDiariserLLM.credential_id || null, - diarization_prompt: - evalDiarisationPrompt.trim() || null, + auto_transcribe: evalTranscriptSource === 'diarised', + ...(evalTranscriptSource === 'diarised' + ? { + transcribe_overwrite: transcribeOverwrite, + transcribe_mode: evalTranscribeMode, + stt_provider: + evalTranscribeMode === 'stt_llm' + ? evalSTT.provider + : null, + stt_model: + evalTranscribeMode === 'stt_llm' + ? evalSTT.model + : null, + stt_credential_id: + evalTranscribeMode === 'stt_llm' + ? evalSTT.credential_id || null + : null, + stt_language: + evalTranscribeMode === 'stt_llm' + ? evalSTTLanguage.trim() || null + : null, + diarization_llm_provider: + evalDiariserLLM.provider, + diarization_llm_model: + resolveLLMModelForSubmit( + evalDiariserLLM, + aiProviders, + ) ?? evalDiariserLLM.model, + diarization_llm_credential_id: + evalDiariserLLM.credential_id || null, + diarization_prompt: + evalDiarisationPrompt.trim() || null, + } + : {}), discover_new_metrics: false, ...(data.status === 'mapped' ? (() => { diff --git a/frontend/src/pages/metrics/MetricsManagement.tsx b/frontend/src/pages/metrics/MetricsManagement.tsx index 5eefdbe8..c359679c 100644 --- a/frontend/src/pages/metrics/MetricsManagement.tsx +++ b/frontend/src/pages/metrics/MetricsManagement.tsx @@ -8,6 +8,8 @@ import type { LLMGenerationConfig } from '../../config/llmGenerationParams' import { useToast } from '../../hooks/useToast' import { useWorkspaceStore } from '../../store/workspaceStore' import { + Copy, + ClipboardPaste, Edit, Trash2, X, @@ -24,6 +26,13 @@ import { Layers, AlertTriangle, } from 'lucide-react' +import { copyTextToClipboard, readTextFromClipboard } from '../../lib/clipboard' +import { + categoryFormFromMetricClipboard, + parseMetricClipboardPayload, + serializeMetricToClipboard, + singleFormFromMetricClipboard, +} from './metricClipboardUtils' import { categoryChildrenFromPartial, createCategoryChildrenFromPartial, @@ -196,6 +205,7 @@ export default function MetricsManagement() { description: string surfaces: MetricSurface[] capture_rationale: boolean + selection_mode: 'single_choice' | 'multi_label' // CREATE-time visibility scope (same semantics as ``formData.scope``). // Inherited by every child of the new category. scope: 'workspace' | 'organization' @@ -210,6 +220,7 @@ export default function MetricsManagement() { description: '', surfaces: ['agent'], capture_rationale: false, + selection_mode: 'single_choice', scope: 'workspace', children: [ { local_id: 'c1', name: '', description: '', example: '' }, @@ -244,6 +255,7 @@ export default function MetricsManagement() { const [pendingDeleteMetric, setPendingDeleteMetric] = useState( null, ) + const [pasteMetricError, setPasteMetricError] = useState(null) // --------------------------------------------------------------------------- // Edit-category form — only used when editing an existing PARENT // (``selection_mode`` set, no ``parent_metric_id``). Reuses the @@ -889,6 +901,7 @@ export default function MetricsManagement() { description: '', surfaces: ['agent'], capture_rationale: false, + selection_mode: 'single_choice', scope: 'workspace', children: [ { local_id: 'c1', name: '', description: '', example: '' }, @@ -1065,6 +1078,45 @@ export default function MetricsManagement() { setPendingDeleteMetric(null) } + const handleCopyMetric = (metric: Metric) => { + if (metric.parent_metric_id) { + showToast('Copy the parent category metric to include all labels.', 'error') + return + } + const payload = serializeMetricToClipboard(metric) + copyTextToClipboard(JSON.stringify(payload, null, 2), () => { + showToast('Metric copied to clipboard', 'success') + }) + } + + const handlePasteMetric = async () => { + setPasteMetricError(null) + try { + const text = await readTextFromClipboard() + const payload = parseMetricClipboardPayload(text) + const targetScope = 'workspace' as const + setEditingMetric(null) + setIsEditingCategory(false) + setIsCustomMetricMode(true) + if (payload.kind === 'category') { + setCreateMode('category') + setCategoryForm(categoryFormFromMetricClipboard(payload, targetScope)) + resetForm() + } else { + setCreateMode('single') + setFormData(singleFormFromMetricClipboard(payload, targetScope)) + resetCategoryForm() + } + setShowCreateModal(true) + showToast('Metric pasted — review scope and save', 'success') + } catch (err) { + const message = + err instanceof Error ? err.message : 'Could not paste metric from clipboard' + setPasteMetricError(message) + showToast(message, 'error') + } + } + const handleEdit = (metric: Metric) => { // A parent category metric: switch to the dedicated category // editor (children appear as editable rows inside the form). @@ -1257,6 +1309,7 @@ export default function MetricsManagement() { setEditingMetric(null) setIsEditingCategory(false) setCreateMode('single') + setPasteMetricError(null) resetForm() resetAIForm() resetCategoryForm() @@ -1803,14 +1856,33 @@ export default function MetricsManagement() { ? 'Create Custom Metric' : 'Create Metric'} - +
+ {!editingMetric && ( + + )} + +
+ {pasteMetricError && !editingMetric && ( +
+ {pasteMetricError} +
+ )} + {/* Mode switcher: lets the user pick between the single metric and the parent category flow without leaving this modal. Hidden during edit because edit always @@ -2717,8 +2789,9 @@ export default function MetricsManagement() { name: categoryForm.name.trim(), description: categoryForm.description.trim() || null, - // Always single_choice for the new flow. - selection_mode: 'single_choice', + // Preserve pasted selection_mode; default flow + // remains single_choice. + selection_mode: categoryForm.selection_mode, allow_discovery: false, capture_rationale: categoryForm.capture_rationale, supported_surfaces: categoryForm.surfaces, @@ -3171,6 +3244,15 @@ export default function MetricsManagement() { > {m.enabled ? 'Disable' : 'Enable'} + {!isChild && ( + + )} {canDelete ? (