From 1a094100bc507df30fb6a27a28c136e5a5745901 Mon Sep 17 00:00:00 2001 From: Tejas Narayan Date: Mon, 3 Aug 2026 15:40:28 +0530 Subject: [PATCH] Revert "feat: updating minor features (#106)" This reverts commit 5a6e2f7078a023c8644c7790d9d0e6a47db92bde. --- app/api/v1/routes/call_import_evaluations.py | 278 +++++++--------- app/api/v1/routes/call_imports.py | 187 +++++------ app/config/models.json | 5 - app/db_sharding/scatter_gather.py | 55 ---- app/models/schemas.py | 23 +- app/services/call_import_user_insights.py | 9 +- app/services/call_imports/bulk_ops.py | 22 -- .../call_import_evaluation_pdf_report.py | 82 ++--- app/workers/concurrency/eval_dispatch.py | 14 - app/workers/concurrency/fair_dispatch.py | 8 +- app/workers/tasks/call_import_bulk_ops.py | 14 - app/workers/tasks/evaluate_call_import_row.py | 11 +- app/workers/tasks/process_call_import_row.py | 7 +- frontend/src/lib/api.ts | 6 +- frontend/src/lib/clipboard.ts | 49 --- .../pages/callImports/CallImportDetail.tsx | 302 +++++++----------- .../src/pages/metrics/MetricsManagement.tsx | 98 +----- .../src/pages/metrics/metricClipboardUtils.ts | 242 -------------- ...st_call_import_diarization_and_eval_llm.py | 21 +- .../test_call_import_evaluation_pdf_report.py | 11 +- .../test_api/test_call_import_evaluations.py | 18 +- .../test_call_import_evaluations_export.py | 3 +- .../test_call_import_sharded_row_mutations.py | 182 ----------- .../test_call_import_unified_dispatch.py | 8 - .../test_eval_dispatch_sharding.py | 30 +- .../test_evaluate_call_import_row.py | 68 ---- .../test_workers/test_fair_dispatch_retry.py | 28 -- .../test_process_call_import_row_sharding.py | 70 +--- 28 files changed, 401 insertions(+), 1450 deletions(-) delete mode 100644 frontend/src/lib/clipboard.ts delete mode 100644 frontend/src/pages/metrics/metricClipboardUtils.ts delete mode 100644 tests/test_api/test_call_import_sharded_row_mutations.py diff --git a/app/api/v1/routes/call_import_evaluations.py b/app/api/v1/routes/call_import_evaluations.py index f73744ad..ef316f2e 100644 --- a/app/api/v1/routes/call_import_evaluations.py +++ b/app/api/v1/routes/call_import_evaluations.py @@ -242,32 +242,17 @@ def _evaluated_transcript_source_label( evaluation: CallImportEvaluation, source_row: CallImportRow, ) -> str: - """Label which transcript source this row was scored against.""" - source = (evaluation.transcript_source or "diarised").strip().lower() - if source == "production": - if not (source_row.transcript or "").strip(): - return "" - return "Production" + """Label whether this row had a diarised transcript for scoring.""" + del evaluation if not (source_row.diarised_transcript or "").strip(): return "" return "Diarised" -def _pick_evaluation_row_transcript( - source_row: Optional[CallImportRow], - evaluation: Optional[CallImportEvaluation] = None, -) -> Optional[str]: - """Transcript shown in evaluation row detail for the run's source.""" +def _pick_evaluation_row_transcript(source_row: Optional[CallImportRow]) -> Optional[str]: + """Transcript shown in evaluation row detail — diarised when present.""" if source_row is None: return None - source = ( - (evaluation.transcript_source or "diarised").strip().lower() - if evaluation is not None - else "diarised" - ) - if source == "production": - raw = (source_row.transcript or "").strip() - return raw or None diarised = (source_row.diarised_transcript or "").strip() if diarised: return diarised @@ -278,7 +263,6 @@ def _pick_evaluation_row_transcript( def _to_evaluation_row_response( eval_row_obj: CallImportEvaluationRow, source_row: Optional[CallImportRow], - evaluation: Optional[CallImportEvaluation] = None, ) -> CallImportEvaluationRowResponse: """Serialize one evaluation row plus joined source-row metadata.""" return CallImportEvaluationRowResponse( @@ -287,7 +271,7 @@ def _to_evaluation_row_response( call_import_row_id=eval_row_obj.call_import_row_id, row_index=source_row.row_index if source_row else None, conversation_id=source_row.conversation_id if source_row else None, - transcript=_pick_evaluation_row_transcript(source_row, evaluation), + transcript=_pick_evaluation_row_transcript(source_row), raw_columns=source_row.raw_columns if source_row else None, recording_url=source_row.recording_url if source_row else None, recording_date=source_row.recording_date if source_row else None, @@ -827,127 +811,125 @@ async def create_call_import_evaluation( metric_overrides_payload[leaf_id] = override_dict # ----- Validate auto-transcribe settings ----- - # Diarised runs auto-diarise rows missing a diarised transcript and - # require STT + diariser LLM config. Production runs score the CSV - # transcript directly and skip diarisation entirely. - use_diarised = payload.transcript_sources[0] == "diarised" - auto_transcribe = use_diarised + # Every evaluation run scores the diarised transcript and + # auto-diarises rows that don't already have one, so STT + # provider+model are mandatory on every request (the + # ``auto_transcribe`` flag is preserved on the schema for API + # compatibility but is effectively always true at this point). + # ``transcribe_mode`` controls whether STT is required: the + # ``llm_only`` path skips STT entirely and feeds audio directly to + # the diariser LLM, so STT fields must be absent. The ``stt_llm`` + # path (default) keeps the original behaviour. + transcribe_mode_norm = (payload.transcribe_mode or "stt_llm").strip().lower() + if transcribe_mode_norm not in {"stt_llm", "llm_only"}: + raise HTTPException( + status_code=400, + detail=( + f"Unknown transcribe_mode '{payload.transcribe_mode}'. " + "Expected 'stt_llm' or 'llm_only'." + ), + ) - transcribe_mode_norm: Optional[str] = None + auto_transcribe = True stt_provider_norm: Optional[str] = None stt_model_norm: Optional[str] = None - diarisation_llm_provider_norm: Optional[str] = None - diarisation_llm_model_norm: Optional[str] = None - diarisation_prompt_norm: Optional[str] = None - - if use_diarised: - transcribe_mode_norm = (payload.transcribe_mode or "stt_llm").strip().lower() - if transcribe_mode_norm not in {"stt_llm", "llm_only"}: - raise HTTPException( - status_code=400, - detail=( - f"Unknown transcribe_mode '{payload.transcribe_mode}'. " - "Expected 'stt_llm' or 'llm_only'." - ), - ) - - if transcribe_mode_norm == "stt_llm": - if not payload.stt_provider: - raise HTTPException( - status_code=400, - detail=( - "stt_provider is required when " - "transcribe_mode='stt_llm': every evaluation run " - "auto-diarises rows that are missing a diarised " - "transcript." - ), - ) - if not payload.stt_model: - raise HTTPException( - status_code=400, - detail=( - "stt_model is required when transcribe_mode='stt_llm'." - ), - ) - try: - stt_provider_norm = ModelProvider( - payload.stt_provider.lower() - ).value - except ValueError: - raise HTTPException( - status_code=400, - detail=f"Unknown STT provider '{payload.stt_provider}'.", - ) - stt_model_norm = payload.stt_model.strip() or None - if not stt_model_norm: - raise HTTPException( - status_code=400, detail="stt_model cannot be empty." - ) - else: - # llm_only — explicitly reject lingering STT inputs so the - # contract is unambiguous (the worker would ignore them but - # silent acceptance hides accidental misconfiguration). - if (payload.stt_provider or "").strip() or ( - payload.stt_model or "" - ).strip(): - raise HTTPException( - status_code=400, - detail=( - "stt_provider / stt_model must be omitted when " - "transcribe_mode='llm_only'; the LLM consumes the " - "audio directly." - ), - ) - - # --- Validate LLM diariser settings ----- - if not payload.diarization_llm_provider: + if transcribe_mode_norm == "stt_llm": + if not payload.stt_provider: raise HTTPException( status_code=400, detail=( - "diarization_llm_provider is required: every evaluation " - "run diarises STT output with an LLM." + "stt_provider is required when " + "transcribe_mode='stt_llm': every evaluation run " + "auto-diarises rows that are missing a diarised " + "transcript." ), ) - if not payload.diarization_llm_model: + if not payload.stt_model: raise HTTPException( status_code=400, detail=( - "diarization_llm_model is required: every evaluation " - "run diarises STT output with an LLM." + "stt_model is required when transcribe_mode='stt_llm'." ), ) try: - diarisation_llm_provider_norm = ModelProvider( - payload.diarization_llm_provider.lower() + stt_provider_norm = ModelProvider( + payload.stt_provider.lower() ).value except ValueError: raise HTTPException( status_code=400, - detail=( - f"Unknown diarisation LLM provider " - f"'{payload.diarization_llm_provider}'." - ), + detail=f"Unknown STT provider '{payload.stt_provider}'.", ) - diarisation_llm_model_norm = ( - payload.diarization_llm_model.strip() or None - ) - if not diarisation_llm_model_norm: + stt_model_norm = payload.stt_model.strip() or None + if not stt_model_norm: + raise HTTPException( + status_code=400, detail="stt_model cannot be empty." + ) + else: + # llm_only — explicitly reject lingering STT inputs so the + # contract is unambiguous (the worker would ignore them but + # silent acceptance hides accidental misconfiguration). + if (payload.stt_provider or "").strip() or ( + payload.stt_model or "" + ).strip(): raise HTTPException( status_code=400, - detail="diarization_llm_model cannot be empty.", + detail=( + "stt_provider / stt_model must be omitted when " + "transcribe_mode='llm_only'; the LLM consumes the " + "audio directly." + ), ) - diarisation_prompt_norm = ( - payload.diarization_prompt.strip() - if isinstance(payload.diarization_prompt, str) - else None - ) or None - from app.models.enums import CallImportParameterType, CallImportStatus - from app.services.call_imports.bulk_ops import ( - count_all_source_rows, - count_completed_source_rows, - count_source_rows_with_production_transcript, + # --- Validate LLM diariser settings ----- + # The post-STT diariser is mandatory now that pyannote is no longer + # in the loop. We reject the request up-front (instead of letting + # individual rows fail at task time) so the operator gets a clean + # 400 in the modal. + if not payload.diarization_llm_provider: + raise HTTPException( + status_code=400, + detail=( + "diarization_llm_provider is required: every evaluation " + "run diarises STT output with an LLM." + ), + ) + if not payload.diarization_llm_model: + raise HTTPException( + status_code=400, + detail=( + "diarization_llm_model is required: every evaluation " + "run diarises STT output with an LLM." + ), + ) + try: + diarisation_llm_provider_norm: Optional[str] = ModelProvider( + payload.diarization_llm_provider.lower() + ).value + except ValueError: + raise HTTPException( + status_code=400, + detail=( + f"Unknown diarisation LLM provider " + f"'{payload.diarization_llm_provider}'." + ), + ) + diarisation_llm_model_norm: Optional[str] = ( + payload.diarization_llm_model.strip() or None ) + if not diarisation_llm_model_norm: + raise HTTPException( + status_code=400, + detail="diarization_llm_model cannot be empty.", + ) + diarisation_prompt_norm: Optional[str] = ( + payload.diarization_prompt.strip() + if isinstance(payload.diarization_prompt, str) + else None + ) or None + + from app.models.enums import CallImportStatus + from app.services.call_imports.bulk_ops import count_completed_source_rows starting_from_mapped = False if call_import.status == CallImportStatus.MAPPED: @@ -976,21 +958,6 @@ async def create_call_import_evaluation( db, organization_id, workspace_id, call_import.schema_id ) parameters = list(schema.parameters) - if not use_diarised: - transcript_mapped = any( - param.type == CallImportParameterType.TRANSCRIPT - and (call_import.parameter_mapping or {}).get(param.name) - for param in parameters - ) - if not transcript_mapped: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - "No transcript column is mapped in this batch. " - "Map a schema transcript parameter to a CSV column, " - "or choose 'Diarize then evaluate'." - ), - ) if payload.telephony_integration_id is not None: integration = _resolve_telephony_integration( db, @@ -1022,31 +989,14 @@ async def create_call_import_evaluation( db.refresh(call_import) starting_from_mapped = True - if use_diarised: - total_row_count = count_completed_source_rows(db, call_import.id) - else: - # Production runs score CSV text — rows need not wait for - # recording fetch to finish before they are evaluable. - total_row_count = count_source_rows_with_production_transcript( - db, call_import.id - ) - - requested_sources: List[str] = list(payload.transcript_sources) + total_row_count = count_completed_source_rows(db, call_import.id) - if ( - not use_diarised - and not starting_from_mapped - and count_all_source_rows(db, call_import.id) > 0 - and total_row_count == 0 - ): - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - "No rows have a production transcript. " - "Choose 'Diarize then evaluate' or import rows with " - "a transcript column." - ), - ) + # Every evaluation run scores the diarised transcript. The + # ``transcript_sources`` field on the schema has already been + # normalized to ``['diarised']`` by the validator; we still iterate + # the list below so the loop machinery stays generic in case future + # sources are reintroduced. + requested_sources: List[str] = ["diarised"] base_name = _normalize_name(payload.name) @@ -1086,7 +1036,7 @@ def _name_for_source(source: str) -> Optional[str]: diarisation_llm_provider=diarisation_llm_provider_norm, diarisation_llm_model=diarisation_llm_model_norm, diarisation_llm_credential_id=( - payload.diarization_llm_credential_id if auto_transcribe else None + payload.diarization_llm_credential_id ), diarisation_prompt=diarisation_prompt_norm, transcribe_mode=transcribe_mode_norm, @@ -1708,9 +1658,9 @@ def _build_query(session: Session): total = query.count() rows = query.offset((page - 1) * page_size).limit(page_size).all() - # Row detail shows the transcript for this run's chosen source. + # Row detail shows the diarised transcript that normal metrics score. items: List[CallImportEvaluationRowResponse] = [ - _to_evaluation_row_response(eval_row_obj, source_row, eval_row) + _to_evaluation_row_response(eval_row_obj, source_row) for eval_row_obj, source_row in rows ] @@ -4031,7 +3981,7 @@ async def cancel_call_import_evaluation_row( _rollup_evaluation_status(evaluation, db) db.commit() row_db.refresh(eval_row) - return _to_evaluation_row_response(eval_row, source_row, evaluation) + return _to_evaluation_row_response(eval_row, source_row) except LookupError as exc: raise HTTPException( status_code=404, detail="Evaluation row not found in this run" @@ -4062,7 +4012,7 @@ async def cancel_call_import_evaluation_row( .first() ) - return _to_evaluation_row_response(eval_row, source_row, evaluation) + return _to_evaluation_row_response(eval_row, source_row) @router.delete( @@ -8858,11 +8808,11 @@ async def retry_call_import_evaluation_row( source_row, _shard_id, ): - return _to_evaluation_row_response(eval_row, source_row, evaluation) + return _to_evaluation_row_response(eval_row, source_row) db.refresh(eval_row) source_row = targets[0][1] - return _to_evaluation_row_response(eval_row, source_row, evaluation) + return _to_evaluation_row_response(eval_row, source_row) from app.core.auth.capabilities import EVALS_RUN, EVALS_VIEW, REPORTS_GENERATE diff --git a/app/api/v1/routes/call_imports.py b/app/api/v1/routes/call_imports.py index e7461504..20cd2b16 100644 --- a/app/api/v1/routes/call_imports.py +++ b/app/api/v1/routes/call_imports.py @@ -2975,46 +2975,6 @@ async def delete_call_import( ) -def _locate_call_import_row_or_404( - catalog_db: Session, - *, - call_import_id: UUID, - row_id: UUID, - organization_id: UUID, -) -> Tuple[Session, CallImportRow, Optional[Session]]: - """Find a call import row on the correct DB session for mutation. - - When sharding is enabled rows live on shard databases; ``get_db`` only - opens the catalog. Returns ``(row_db, row, extra_catalog_to_close)`` - where ``extra_catalog_to_close`` is the catalog session opened by - :func:`locate_call_import_row` (distinct from the route's catalog - session) and must be closed via :func:`close_row_sessions`. - """ - from app.db_sharding.row_ops import close_row_sessions, locate_call_import_row - - try: - row_db, located_catalog, row, _shard_id = locate_call_import_row(row_id) - except LookupError: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import row not found", - ) from None - if ( - row.call_import_id != call_import_id - or row.organization_id != organization_id - ): - close_row_sessions( - row_db, - located_catalog if located_catalog is not row_db else None, - ) - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call import row not found", - ) - extra_catalog = located_catalog if located_catalog is not row_db else None - return row_db, row, extra_catalog - - @router.delete( "/{call_import_id}/rows/{row_id}", status_code=status.HTTP_204_NO_CONTENT, @@ -3049,35 +3009,38 @@ async def delete_call_import_row( detail="Call import not found", ) - from app.db_sharding.row_ops import close_row_sessions - - row_db, row, extra_catalog = _locate_call_import_row_or_404( - db, - call_import_id=call_import.id, - row_id=row_id, - organization_id=organization_id, + row = ( + db.query(CallImportRow) + .filter( + CallImportRow.id == row_id, + CallImportRow.call_import_id == call_import.id, + ) + .first() ) - try: - _revoke_pending_tasks([row]) + if not row: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import row not found", + ) - if row.recording_s3_key and s3_service.is_enabled(): - try: - s3_service.delete_file_by_key(row.recording_s3_key) - except Exception as exc: # noqa: BLE001 — best-effort, DB is source of truth - logger.warning( - "Failed to delete S3 object {} for row {}: {}", - row.recording_s3_key, - row.id, - exc, - ) + _revoke_pending_tasks([row]) - row_db.delete(row) - row_db.commit() + if row.recording_s3_key and s3_service.is_enabled(): + try: + s3_service.delete_file_by_key(row.recording_s3_key) + except Exception as exc: # noqa: BLE001 — best-effort, DB is source of truth + logger.warning( + "Failed to delete S3 object {} for row {}: {}", + row.recording_s3_key, + row.id, + exc, + ) - _recompute_call_import_counters(db, call_import) - db.commit() - finally: - close_row_sessions(row_db, extra_catalog) + db.delete(row) + db.flush() + + _recompute_call_import_counters(db, call_import) + db.commit() logger.info( "Deleted call_import_row {} (call_import={}, org={})", @@ -3566,21 +3529,24 @@ async def cancel_call_import_row_diarisation( detail="Call import not found", ) - from app.db_sharding.row_ops import close_row_sessions - - row_db, row, extra_catalog = _locate_call_import_row_or_404( - db, - call_import_id=call_import_id, - row_id=row_id, - organization_id=organization_id, + row = ( + db.query(CallImportRow) + .filter( + CallImportRow.id == row_id, + CallImportRow.call_import_id == call_import_id, + ) + .first() ) - try: - _apply_diarisation_cancel([row]) - row_db.commit() - row_db.refresh(row) - return CallImportRowResponse.model_validate(row) - finally: - close_row_sessions(row_db, extra_catalog) + if not row: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import row not found", + ) + + _apply_diarisation_cancel([row]) + db.commit() + db.refresh(row) + return CallImportRowResponse.model_validate(row) @router.post( @@ -3736,40 +3702,43 @@ async def toggle_call_import_row_speaker_swap( detail="Call import not found", ) - from app.db_sharding.row_ops import close_row_sessions - - row_db, row, extra_catalog = _locate_call_import_row_or_404( - db, - call_import_id=call_import_id, - row_id=row_id, - organization_id=organization_id, + row = ( + db.query(CallImportRow) + .filter( + CallImportRow.id == row_id, + CallImportRow.call_import_id == call_import_id, + CallImportRow.organization_id == organization_id, + ) + .first() ) - try: - segments = ( - row.diarised_segments if isinstance(row.diarised_segments, list) else None + if not row: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call import row not found", ) - if not segments: - # Without structured turns the swap toggle would have nothing to - # re-render — surface a clear error rather than silently - # flipping a flag the UI never read. - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - "This row has no structured diarised segments to swap. " - "Re-run diarisation to generate per-speaker turns first." - ), - ) - new_swap = not bool(row.diarised_speaker_swap) - row.diarised_speaker_swap = new_swap - row.diarised_transcript = ( - _render_diarised_segments_text(segments, swap=new_swap) or None + segments = row.diarised_segments if isinstance(row.diarised_segments, list) else None + if not segments: + # Without structured turns the swap toggle would have nothing to + # re-render — surface a clear error rather than silently + # flipping a flag the UI never read. + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + "This row has no structured diarised segments to swap. " + "Re-run diarisation to generate per-speaker turns first." + ), ) - row_db.commit() - row_db.refresh(row) - return CallImportRowResponse.model_validate(row) - finally: - close_row_sessions(row_db, extra_catalog) + + new_swap = not bool(row.diarised_speaker_swap) + row.diarised_speaker_swap = new_swap + row.diarised_transcript = ( + _render_diarised_segments_text(segments, swap=new_swap) or None + ) + db.commit() + db.refresh(row) + + return CallImportRowResponse.model_validate(row) # --------------------------------------------------------------------------- diff --git a/app/config/models.json b/app/config/models.json index c3324c3a..d6fb9fb1 100644 --- a/app/config/models.json +++ b/app/config/models.json @@ -305,11 +305,6 @@ "featured_rank": 7, "highlights": ["GA stable", "1M context", "Agentic coding"] }, - "gemini-3.5-flash-lite": { - "provider": "google", - "model_type": "llm", - "description": "Gemini 3.5 Flash Lite \u2014 fastest, lowest-cost 3.5 model for high-throughput extraction, classification, and subagent workflows" - }, "gemini-3.1-flash-lite": { "provider": "google", "model_type": "llm", diff --git a/app/db_sharding/scatter_gather.py b/app/db_sharding/scatter_gather.py index 931aa57a..917acc31 100644 --- a/app/db_sharding/scatter_gather.py +++ b/app/db_sharding/scatter_gather.py @@ -779,61 +779,6 @@ def count_shard(db: Session, _shard_id: str) -> int: return total -def _non_empty_production_transcript_filter(): - """Rows whose CSV ``transcript`` column has non-whitespace content.""" - return ( - CallImportRow.transcript.isnot(None), - func.length(func.trim(CallImportRow.transcript)) > 0, - ) - - -def count_call_import_rows_with_production_transcript( - catalog_db: Session, - call_import_id: UUID, -) -> int: - """Import rows with a non-empty production transcript (shard-aware).""" - transcript_filters = _non_empty_production_transcript_filter() - - if not is_sharding_enabled(): - return int( - catalog_db.query(func.count(CallImportRow.id)) - .filter( - CallImportRow.call_import_id == call_import_id, - *transcript_filters, - ) - .scalar() - or 0 - ) - - total = 0 - shard_ids = shard_ids_for_import(catalog_db, call_import_id) - - def count_shard(db: Session, _shard_id: str) -> int: - return int( - db.query(func.count(CallImportRow.id)) - .filter( - CallImportRow.call_import_id == call_import_id, - *transcript_filters, - ) - .scalar() - or 0 - ) - - for part in scatter_gather_on_shards(shard_ids, count_shard): - total += int(part) - if total == 0: - total = int( - catalog_db.query(func.count(CallImportRow.id)) - .filter( - CallImportRow.call_import_id == call_import_id, - *transcript_filters, - ) - .scalar() - or 0 - ) - return total - - def count_completed_call_import_rows( catalog_db: Session, call_import_id: UUID, diff --git a/app/models/schemas.py b/app/models/schemas.py index 10a4d5b7..59c3ced2 100644 --- a/app/models/schemas.py +++ b/app/models/schemas.py @@ -3000,10 +3000,11 @@ class CallImportEvaluationCreate(BaseModel): min_length=1, max_length=1, description=( - "Which transcript to score against. ``'diarised'`` (default) " - "auto-diarises rows missing a diarised transcript then scores " - "``diarised_transcript``. ``'production'`` scores the CSV " - "``transcript`` column directly and skips diarisation." + "Which transcript to score against. Only ``diarised`` is " + "supported — every evaluation run scores the diarised " + "transcript. Pass ``['diarised']`` explicitly or omit the " + "field to take the default; any other value (including the " + "legacy ``'production'`` source) is rejected with a 400." ), ) @@ -3012,16 +3013,20 @@ class CallImportEvaluationCreate(BaseModel): def _validate_transcript_sources( cls, value: List[str] ) -> List["CallImportEvaluationTranscriptSource"]: - allowed = {"production", "diarised"} - invalid = [src for src in value if src not in allowed] + # The Field min_length/max_length constraints catch empty + + # over-long payloads; this validator's job is to reject any + # non-diarised source value and normalize the result to + # ``['diarised']`` for downstream code. + invalid = [src for src in value if src != "diarised"] if invalid: raise ValueError( - "transcript_sources must be ['production'] or ['diarised'] " + "Only the 'diarised' transcript source is supported " "(received: " + ", ".join(repr(src) for src in invalid) - + ")." + + "). Remove 'production' from transcript_sources or " + "omit the field to use the default." ) - return value # type: ignore[return-value] + return ["diarised"] # --- Run-level LLM config --- llm_provider: Optional[str] = Field( default=None, diff --git a/app/services/call_import_user_insights.py b/app/services/call_import_user_insights.py index bf723e6a..e1947a0b 100644 --- a/app/services/call_import_user_insights.py +++ b/app/services/call_import_user_insights.py @@ -143,12 +143,9 @@ def _pick_transcript( evaluation: CallImportEvaluation, source_row: CallImportRow, ) -> str: - source = (evaluation.transcript_source or "diarised").strip().lower() - if source == "production": - text = (source_row.transcript or "").strip() - else: - text = (source_row.diarised_transcript or "").strip() - return text[:ROW_TRANSCRIPT_CHAR_CAP] + del evaluation + diarised = (source_row.diarised_transcript or "").strip() + return diarised[:ROW_TRANSCRIPT_CHAR_CAP] def _metric_name_map(metrics: Sequence[Metric]) -> Dict[str, str]: diff --git a/app/services/call_imports/bulk_ops.py b/app/services/call_imports/bulk_ops.py index 2d83faf5..cbed6ceb 100644 --- a/app/services/call_imports/bulk_ops.py +++ b/app/services/call_imports/bulk_ops.py @@ -199,16 +199,6 @@ def count_completed_source_rows(db: Session, call_import_id: UUID) -> int: return count_completed_call_import_rows(db, call_import_id) -def count_source_rows_with_production_transcript( - db: Session, call_import_id: UUID -) -> int: - from app.db_sharding.scatter_gather import ( - count_call_import_rows_with_production_transcript, - ) - - return count_call_import_rows_with_production_transcript(db, call_import_id) - - def _completed_source_row_ids(db: Session, call_import_id: UUID) -> List[UUID]: from app.db_sharding.scatter_gather import list_completed_source_row_ids_ordered @@ -680,19 +670,7 @@ def execute_call_import_materialization( return {"total_rows": 0, "status": "failed"} try: - logger.info( - "execute_call_import_materialization downloading staged source " - "(call_import={} key={})", - call_import_id, - call_import.source_s3_key, - ) file_bytes = s3_service.download_file_by_key(call_import.source_s3_key) - logger.info( - "execute_call_import_materialization downloaded {} bytes " - "(call_import={})", - len(file_bytes), - call_import_id, - ) except StorageError as exc: call_import.status = CallImportStatus.FAILED call_import.error_message = f"Could not read staged source file from S3: {exc}" diff --git a/app/services/reporting/call_import_evaluation_pdf_report.py b/app/services/reporting/call_import_evaluation_pdf_report.py index 64c53df7..1a3a3d82 100644 --- a/app/services/reporting/call_import_evaluation_pdf_report.py +++ b/app/services/reporting/call_import_evaluation_pdf_report.py @@ -1373,7 +1373,7 @@ def _failure_diagnostics_section_html( if not metrics_body and not discovered_section: metrics_body = "

No cluster groups in this run.

" return f""" -
+

04 Failure Diagnostics

Per-metric clustering of flagged calls with engineering gap labels (LOGIC_GAP, UNDERSPEC, EXISTS_NO_TRIGGER, MISSING) and Level-2 sub-categories.

@@ -1485,7 +1485,7 @@ def _prompt_improvements_section_html( """ ) return f""" -
+

05 Prompt Improvement Recommendations

Top {_PDF_PROMPT_IMPROVEMENTS_MAX} recommended prompt changes for {html.escape(agent_name)}, aligned with the Visualizations → Prompt / Agent Improvements view.

@@ -2005,7 +2005,7 @@ def _render_html(self, payload: dict[str, Any]) -> str: """ ) quality_panel_markup = f""" -
+

02 Quality Metric Panel

{''.join(quality_groups_markup)}
@@ -2104,7 +2104,7 @@ def _render_html(self, payload: dict[str, Any]) -> str: else "User insights are identified by analyzing patterns across LLM rationales and diarized transcripts from the evaluation run." ) business_section = f""" -
+

03 User Insights

{intro}

@@ -2137,7 +2137,7 @@ def _render_html(self, payload: dict[str, Any]) -> str: ) if notes: design_notes_section = f""" -
+

05 User Experience Design Notes

    {notes}
@@ -2169,11 +2169,6 @@ def _render_html(self, payload: dict[str, Any]) -> str: if internal_logo_uri else "" ) - report_body_class = ( - "report-body internal-report" - if is_internal - else "report-body external-report" - ) return f""" @@ -2208,20 +2203,17 @@ def _render_html(self, payload: dict[str, Any]) -> str: .eyebrow .muted {{ color: #7a756d; }} h1 {{ font-size: 28px; font-weight: 800; margin: 4px 0; letter-spacing: .1px; }} .subtitle {{ font-size: 13px; color: #666; margin-bottom: 10px; }} - h2 {{ font-size: 16px; margin: 8px 0 6px; color: #0b1220; border-bottom: 2px solid #0b1220; padding-bottom: 2px; break-after: avoid; page-break-after: avoid; }} - h3 {{ font-size: 13px; margin: 0; break-after: avoid; page-break-after: avoid; }} - .report-body section {{ margin: 0 0 10px; padding: 0; }} - .report-body section > .method:first-of-type {{ margin-top: 0; margin-bottom: 6px; }} - .metric-group {{ margin-bottom: 8px; break-inside: auto; page-break-inside: auto; }} + h2 {{ font-size: 16px; margin: 16px 0 8px; color: #0b1220; border-bottom: 2px solid #0b1220; padding-bottom: 2px; }} + h3 {{ font-size: 13px; margin: 0; }} .meta {{ display: grid; grid-template-columns: repeat(5, 1fr); gap: 10px; }} .summary {{ display: grid; grid-template-columns: repeat(4, 1fr); gap: 10px; }} - .metric-group-title {{ font-size: 13px; margin: 8px 0 6px; color: #374151; text-transform: uppercase; letter-spacing: .08em; break-after: avoid; page-break-after: avoid; }} + .metric-group-title {{ font-size: 13px; margin: 12px 0 8px; color: #374151; text-transform: uppercase; letter-spacing: .08em; }} .box {{ border: 1px solid #d9dee8; padding: 7px 8px; border-radius: 6px; background: #f7f9fc; }} .box strong {{ display: block; font-size: 13px; color: #111827; }} .box span {{ color: #667085; font-size: 9px; text-transform: uppercase; letter-spacing: .04em; }} - .metric {{ break-inside: avoid; page-break-inside: avoid; border: 1px solid #d9dee8; border-radius: 2px; padding: 10px; margin-bottom: 8px; background: #fff; }} - .metric-compact-grid {{ display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 8px; margin-bottom: 6px; break-inside: auto; page-break-inside: auto; }} - .metric-compact {{ padding: 9px 10px 8px; margin-bottom: 0; min-height: 0; break-inside: avoid; page-break-inside: avoid; }} + .metric {{ break-inside: avoid; border: 1px solid #d9dee8; border-radius: 2px; padding: 10px; margin-bottom: 10px; background: #fff; }} + .metric-compact-grid {{ display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 10px; margin-bottom: 8px; }} + .metric-compact {{ padding: 9px 10px 8px; margin-bottom: 0; min-height: 148px; }} .metric-compact-head {{ display: flex; justify-content: space-between; align-items: flex-start; gap: 8px; margin-bottom: 6px; }} .metric-compact-head .metric-title {{ font-size: 9px; letter-spacing: .08em; color: #667085; font-weight: 800; }} .metric-compact-status {{ width: 7px; height: 7px; border-radius: 50%; background: #16a34a; margin-top: 2px; flex-shrink: 0; }} @@ -2291,16 +2283,16 @@ def _render_html(self, payload: dict[str, Any]) -> str: .dist-fill {{ height: 100%; background: #c7725e; }} .dist-value {{ text-align: right; font-size: 10px; font-weight: 800; color: #111827; }} .empty-bars {{ color: #667085; font-size: 10px; font-style: italic; }} - .insight-block {{ break-inside: auto; page-break-inside: auto; margin: 8px 0 10px; }} - .insight-heading {{ margin-bottom: 6px; break-after: avoid; page-break-after: avoid; }} + .insight-block {{ break-inside: avoid; margin: 14px 0 18px; }} + .insight-heading {{ margin-bottom: 8px; }} .insight-heading h3 {{ margin: 0; }} - .insight-body {{ display: grid; grid-template-columns: 1.15fr 0.85fr; gap: 10px; align-items: start; break-inside: auto; page-break-inside: auto; }} - .insight-table-col {{ min-width: 0; break-inside: auto; page-break-inside: auto; }} - .insight-sidecol {{ display: grid; gap: 8px; min-width: 0; }} - .insight-overview-box {{ break-inside: auto; page-break-inside: auto; margin: 0 0 10px; padding: 10px 12px; border: 1px solid #e7ddd1; border-radius: 8px; background: #faf7f2; }} + .insight-body {{ display: grid; grid-template-columns: 1.15fr 0.85fr; gap: 14px; align-items: start; }} + .insight-table-col {{ min-width: 0; }} + .insight-sidecol {{ display: grid; gap: 10px; min-width: 0; }} + .insight-overview-box {{ break-inside: avoid; margin: 0 0 16px; padding: 12px 14px; border: 1px solid #e7ddd1; border-radius: 8px; background: #faf7f2; }} .insight-overview-label {{ font-size: 10px; text-transform: uppercase; letter-spacing: .08em; color: #7a756d; font-weight: 800; margin-bottom: 6px; }} .insight-overview-box p {{ margin: 0; color: #374151; line-height: 1.5; }} - .insight-callout {{ break-inside: avoid; page-break-inside: avoid; border: 1px solid #eadfce; border-radius: 8px; padding: 10px 12px; }} + .insight-callout {{ border: 1px solid #eadfce; border-radius: 8px; padding: 10px 12px; }} .insight-observation-box {{ background: #fdf3f0; border-color: #f0d4cb; }} .insight-evidence-box {{ background: #faf6ef; border-color: #eadfce; }} .insight-callout-label {{ font-size: 10px; text-transform: uppercase; letter-spacing: .08em; color: #b85f4b; font-weight: 800; margin-bottom: 6px; }} @@ -2310,7 +2302,7 @@ def _render_html(self, payload: dict[str, Any]) -> str: .insight-table td:nth-child(2), .insight-table td:nth-child(4) {{ white-space: nowrap; font-weight: 800; }} .insight-track {{ height: 9px; background: #e8edff; border: 1px solid #cfd8ff; min-width: 120px; }} .insight-fill {{ height: 100%; background: #4f46e5; }} - .failure-diagnostics {{ margin-top: 0; max-width: 100%; break-inside: auto; page-break-inside: auto; }} + .failure-diagnostics {{ margin-top: 4px; max-width: 100%; }} .failure-diagnostics-intro {{ break-inside: auto; margin-bottom: 6px; max-width: 100%; }} .failure-diagnostics-intro h2 {{ margin-bottom: 4px; }} .failure-diagnostics .method {{ margin: 0 0 6px; }} @@ -2484,29 +2476,27 @@ def _render_html(self, payload: dict[str, Any]) -> str: .design-notes {{ margin: 8px 0 0; padding-left: 18px; }} .design-notes li {{ margin-bottom: 8px; line-height: 1.5; }} .gap-badge {{ display: inline-block; font-size: 9px; font-weight: 800; letter-spacing: .04em; color: #b42318; text-transform: uppercase; }} - table {{ width: 100%; border-collapse: collapse; margin-top: 6px; break-inside: auto; page-break-inside: auto; }} - thead {{ display: table-header-group; }} - tr {{ break-inside: avoid; page-break-inside: avoid; }} + table {{ width: 100%; border-collapse: collapse; margin-top: 8px; }} th, td {{ text-align: left; border-bottom: 1px solid #e5e7eb; padding: 7px; vertical-align: top; }} th {{ background: #111827; color: #fff; font-size: 10px; text-transform: uppercase; }} .method {{ color: #475467; line-height: 1.5; }} .audit-summary p {{ margin: 0 0 7px; }} .audit-summary ul {{ margin: 7px 0 10px; padding-left: 16px; }} .audit-summary li {{ margin-bottom: 5px; }} - .audit-summary-section {{ margin-bottom: 0; break-after: auto; page-break-after: auto; }} + .audit-summary-section {{ margin-bottom: 0; break-after: avoid; page-break-after: avoid; }} .audit-summary-section + section {{ margin-top: 0; }} - .audit-summary-section + section h2 {{ margin-top: 8px; }} - .audit-stat-strip {{ display: flex; gap: 0; margin: 10px 0 0; border-top: 1px solid #111827; break-after: auto; page-break-after: auto; }} - .audit-stat-strip:last-child {{ margin-bottom: 10px; }} + .audit-summary-section + section h2 {{ margin-top: 12px; }} + .audit-stat-strip {{ display: flex; gap: 0; margin: 14px 0 0; border-top: 1px solid #111827; break-after: avoid; page-break-after: avoid; }} + .audit-stat-strip:last-child {{ margin-bottom: 18px; }} .audit-stat-card {{ flex: 1; min-width: 0; padding: 10px 12px 12px 0; }} .audit-stat-card + .audit-stat-card {{ border-left: 1px solid #d1d5db; padding-left: 12px; }} .audit-stat-rule {{ display: none; }} .audit-stat-label {{ font-size: 8px; font-weight: 800; letter-spacing: .08em; text-transform: uppercase; color: #d16532; margin-bottom: 6px; line-height: 1.3; }} .audit-stat-value {{ font-family: Georgia, 'Times New Roman', serif; font-size: 28px; font-weight: 700; color: #111827; line-height: 1; }} - .audit-delta-panel {{ break-inside: auto; page-break-inside: auto; margin: 6px 0 8px; padding-top: 6px; border-top: 1px solid #e5e7eb; }} - .audit-delta-title {{ font-size: 13px; font-weight: 900; letter-spacing: .04em; text-transform: uppercase; color: #111827; margin-bottom: 6px; break-after: avoid; page-break-after: avoid; }} - .audit-delta-grid {{ display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 8px; break-inside: auto; page-break-inside: auto; }} - .audit-delta-card {{ break-inside: avoid; page-break-inside: avoid; border: 1px solid #d9dee8; border-radius: 6px; padding: 8px 9px; background: #f8fafc; min-height: 0; }} + .audit-delta-panel {{ break-inside: auto; page-break-inside: auto; margin: 8px 0 12px; padding-top: 8px; border-top: 1px solid #e5e7eb; }} + .audit-delta-title {{ font-size: 13px; font-weight: 900; letter-spacing: .04em; text-transform: uppercase; color: #111827; margin-bottom: 8px; }} + .audit-delta-grid {{ display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 8px; }} + .audit-delta-card {{ break-inside: auto; page-break-inside: auto; border: 1px solid #d9dee8; border-radius: 6px; padding: 8px 9px; background: #f8fafc; min-height: 96px; }} .audit-delta-head {{ display: flex; justify-content: space-between; align-items: flex-start; gap: 8px; margin-bottom: 4px; }} .audit-delta-head-main {{ min-width: 0; }} .audit-delta-head span {{ display: block; color: #111827; font-size: 9px; font-weight: 900; letter-spacing: .06em; line-height: 1.25; text-transform: uppercase; overflow-wrap: anywhere; }} @@ -2519,17 +2509,12 @@ def _render_html(self, payload: dict[str, Any]) -> str: .audit-delta-chart .metric-sparkline {{ width: 100%; max-width: 100%; }} .audit-delta-empty {{ height: 50px; display: flex; align-items: center; justify-content: center; border: 1px dashed #cbd5e1; background: #fff; color: #667085; font-size: 9px; }} .audit-delta-reason {{ margin: 6px 0 0; padding-top: 5px; border-top: 1px solid #e5e7eb; color: #374151; font-size: 10px; font-weight: 600; line-height: 1.45; }} - .report-footer {{ margin-top: 14px; border-top: 1px solid #1f2937; padding-top: 10px; display: flex; justify-content: space-between; align-items: flex-start; gap: 16px; color: #4b5563; font-size: 10px; break-inside: avoid; page-break-inside: avoid; }} + .report-footer {{ margin-top: 24px; border-top: 1px solid #1f2937; padding-top: 10px; display: flex; justify-content: space-between; align-items: flex-start; gap: 16px; color: #4b5563; font-size: 10px; }} .footer-left, .footer-right {{ display: flex; flex-direction: column; gap: 3px; }} .footer-right {{ text-align: right; }} .brand-title {{ color: #111827; font-weight: 800; font-size: 11px; letter-spacing: .2px; }} .brand-link {{ color: #d16532; font-weight: 700; text-decoration: none; }} - .internal-report .insight-block {{ margin: 6px 0 8px; }} - .internal-report .failure-diagnostics-intro h2, - .internal-report .prompt-improvements > h2 {{ - margin-top: 8px; - }} - .prompt-improvements {{ margin-top: 0; max-width: 100%; break-inside: auto; page-break-inside: auto; }} + .prompt-improvements {{ margin-top: 4px; max-width: 100%; }} .prompt-improvement-card {{ margin: 0 0 10px; padding: 8px 10px; @@ -2642,7 +2627,7 @@ def _render_html(self, payload: dict[str, Any]) -> str: }} - + {repeat_header_markup}
{brand_header_markup} @@ -2659,7 +2644,7 @@ def _render_html(self, payload: dict[str, Any]) -> str:
{f''' -
+

01 Audit Summary

{audit_summary_markup}
{top_metric_strip_markup} @@ -2672,7 +2657,8 @@ def _render_html(self, payload: dict[str, Any]) -> str: {prompt_improvements_section} {design_notes_section} {f''' -
+
+

Methodology

This report is generated from completed Call Import evaluation results. Metrics are derived from the saved evaluation outputs and the transcript source configured for the run.{html.escape(weekly_methodology)}

''' if show_methodology else ''} diff --git a/app/workers/concurrency/eval_dispatch.py b/app/workers/concurrency/eval_dispatch.py index 89debff8..7afbd735 100644 --- a/app/workers/concurrency/eval_dispatch.py +++ b/app/workers/concurrency/eval_dispatch.py @@ -64,13 +64,6 @@ def _needs_transcribe_for_eval( transcribe_overwrite: bool, auto_transcribe: bool = True, ) -> bool: - if ( - (getattr(evaluation, "transcript_source", None) or "") - .strip() - .lower() - == "production" - ): - return False if not auto_transcribe: return False transcribe_mode = ( @@ -219,13 +212,6 @@ def enqueue_eval_chain_transcribe_after_import( transcribe_overwrite: bool = False, ) -> bool: """Directly chain diarisation after a successful eval-chain recording fetch.""" - if not _needs_transcribe_for_eval( - evaluation, - source_row, - transcribe_overwrite=transcribe_overwrite, - ): - return False - from app.db_sharding.row_ops import shard_row_write_context recover_eval_row_for_eval_chain(eval_row) diff --git a/app/workers/concurrency/fair_dispatch.py b/app/workers/concurrency/fair_dispatch.py index 55e4bbc7..913734ea 100644 --- a/app/workers/concurrency/fair_dispatch.py +++ b/app/workers/concurrency/fair_dispatch.py @@ -352,12 +352,6 @@ def _dispatch_batch_for_workspace( for eval_row, source_row, evaluation in pending: restricted_metric_ids = get_row_restricted_metrics(eval_row.id) transcribe_overwrite = evaluation_transcribe_overwrite(evaluation.id) - eval_auto_transcribe = ( - (getattr(evaluation, "transcript_source", None) or "diarised") - .strip() - .lower() - != "production" - ) outcome = _try_dispatch_single_row( db=db, evaluation=evaluation, @@ -365,7 +359,7 @@ def _dispatch_batch_for_workspace( source_row=source_row, restricted_metric_ids=restricted_metric_ids, transcribe_overwrite=transcribe_overwrite, - auto_transcribe=eval_auto_transcribe, + auto_transcribe=True, call_import=call_import, shard_cache=shard_cache, ) diff --git a/app/workers/tasks/call_import_bulk_ops.py b/app/workers/tasks/call_import_bulk_ops.py index 866cb215..d36cccde 100644 --- a/app/workers/tasks/call_import_bulk_ops.py +++ b/app/workers/tasks/call_import_bulk_ops.py @@ -114,12 +114,6 @@ def materialize_mapped_call_import_evaluation_task( block the Run Evaluation request. """ del self - logger.info( - "materialize_mapped_call_import_evaluation starting " - "(call_import={} evaluation={})", - call_import_id, - evaluation_id, - ) db = SessionLocal() eval_uuid = UUID(evaluation_id) try: @@ -130,14 +124,6 @@ def materialize_mapped_call_import_evaluation_task( UUID(workspace_id), schedule_import_dispatch=False, ) - logger.info( - "materialize_mapped_call_import_evaluation materialization finished " - "(call_import={} evaluation={} status={} total_rows={})", - call_import_id, - evaluation_id, - mat_result.get("status"), - mat_result.get("total_rows"), - ) status = mat_result.get("status") if status == "failed": call_import = ( diff --git a/app/workers/tasks/evaluate_call_import_row.py b/app/workers/tasks/evaluate_call_import_row.py index 29247d24..9ccb9dbb 100644 --- a/app/workers/tasks/evaluate_call_import_row.py +++ b/app/workers/tasks/evaluate_call_import_row.py @@ -377,15 +377,8 @@ def evaluate_call_import_row_task( production_transcript = (source_row.transcript or "").strip() diarised_transcript = (source_row.diarised_transcript or "").strip() - eval_source = ( - (evaluation.transcript_source or "diarised").strip().lower() - ) - if eval_source == "production": - transcript = production_transcript - missing_label = "production" - else: - transcript = diarised_transcript - missing_label = "diarised" + transcript = diarised_transcript + missing_label = "diarised" recording_s3_key = (source_row.recording_s3_key or "").strip() or None has_audio = recording_s3_key is not None diff --git a/app/workers/tasks/process_call_import_row.py b/app/workers/tasks/process_call_import_row.py index 09f0feae..5138cc3a 100644 --- a/app/workers/tasks/process_call_import_row.py +++ b/app/workers/tasks/process_call_import_row.py @@ -548,7 +548,6 @@ def process_call_import_row_task( if run_eval_row_id: from app.models.database import CallImportEvaluation, CallImportEvaluationRow from app.workers.concurrency.eval_dispatch import ( - _needs_transcribe_for_eval, enqueue_eval_chain_transcribe_after_import, ) @@ -569,11 +568,7 @@ def process_call_import_row_task( .filter(CallImportEvaluation.id == eval_row.evaluation_id) .first() ) - if evaluation is not None and _needs_transcribe_for_eval( - evaluation, - row, - transcribe_overwrite=False, - ): + if evaluation is not None: enqueue_eval_chain_transcribe_after_import( db, evaluation=evaluation, diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 5ab03863..4f66da62 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -1629,9 +1629,9 @@ class ApiClient { metric_ids: string[] name?: string | null /** - * Which transcript to score against. ``'diarised'`` (default) - * auto-diarizes then scores; ``'production'`` scores the CSV - * transcript directly. + * Which transcript(s) to score against. Passing both values triggers + * two evaluation runs (one per source). Defaults server-side to + * `['production']` for backwards compatibility. */ transcript_sources?: Array<'production' | 'diarised'> /** Run-level LLM provider key. Leave undefined for legacy default. */ diff --git a/frontend/src/lib/clipboard.ts b/frontend/src/lib/clipboard.ts deleted file mode 100644 index f5ca753b..00000000 --- a/frontend/src/lib/clipboard.ts +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Copy text to the system clipboard with a secure-context fallback. - * - * ``navigator.clipboard`` is async and requires a secure context. Falls - * back to ``document.execCommand('copy')`` so LAN dev over HTTP still - * works. - */ -export function copyTextToClipboard( - text: string, - onSuccess?: () => void, -): void { - if (!text) return - - const finalize = () => { - onSuccess?.() - } - - const fallbackCopy = () => { - try { - const ta = document.createElement('textarea') - ta.value = text - ta.style.position = 'fixed' - ta.style.opacity = '0' - document.body.appendChild(ta) - ta.select() - document.execCommand('copy') - document.body.removeChild(ta) - finalize() - } catch { - // Swallow — user can still drag-select visible text. - } - } - - if (navigator.clipboard?.writeText) { - navigator.clipboard.writeText(text).then(finalize).catch(fallbackCopy) - } else { - fallbackCopy() - } -} - -/** - * Read text from the clipboard. Requires a secure context and permission. - */ -export async function readTextFromClipboard(): Promise { - if (!navigator.clipboard?.readText) { - throw new Error('Clipboard read is not supported in this browser.') - } - return navigator.clipboard.readText() -} diff --git a/frontend/src/pages/callImports/CallImportDetail.tsx b/frontend/src/pages/callImports/CallImportDetail.tsx index bc8687b8..86a27dcb 100644 --- a/frontend/src/pages/callImports/CallImportDetail.tsx +++ b/frontend/src/pages/callImports/CallImportDetail.tsx @@ -40,7 +40,6 @@ import { Loader2, } from 'lucide-react' import { apiClient } from '../../lib/api' -import { copyTextToClipboard } from '../../lib/clipboard' import { getApiErrorMessage } from '../../lib/apiErrors' import { useToast } from '../../hooks/useToast' import { formatDiarisationError } from '../../lib/diarisationErrors' @@ -313,47 +312,62 @@ export default function CallImportDetail() { // exact button that was clicked without affecting the rest of the // table. Auto-clears after 1.5s via a setTimeout inside the handler. const [copiedRowId, setCopiedRowId] = useState(null) - const [copiedTranscriptField, setCopiedTranscriptField] = useState<{ - rowId: string - field: 'production' | 'diarised' - } | null>(null) - const handleCopyConversationId = ( row: CallImportRow, event: React.MouseEvent, ) => { + // Stop the synthetic click from bubbling up to the expand button + // that wraps the row header — otherwise copying would also toggle + // the row open/closed. event.preventDefault() event.stopPropagation() const text = row.conversation_id || '' if (!text) return - copyTextToClipboard(text, () => { + const finalize = () => { setCopiedRowId(row.id) window.setTimeout(() => { - setCopiedRowId((current) => (current === row.id ? null : current)) + // Only clear if THIS row is still the active one — a quick + // double-click on two different rows shouldn't prematurely + // wipe the badge on the second. + setCopiedRowId((prev) => (prev === row.id ? null : prev)) }, 1500) - }) - } - - const handleCopyTranscript = ( - row: CallImportRow, - field: 'production' | 'diarised', - event: React.MouseEvent, - ) => { - event.preventDefault() - event.stopPropagation() - const text = - field === 'production' - ? row.transcript || '' - : row.diarised_transcript || '' - if (!text) return - copyTextToClipboard(text, () => { - setCopiedTranscriptField({ rowId: row.id, field }) - window.setTimeout(() => { - setCopiedTranscriptField((current) => - current?.rowId === row.id && current?.field === field ? null : current, - ) - }, 1500) - }) + } + // ``navigator.clipboard`` is async + requires a secure context. + // Fall back to the legacy ``execCommand`` path so localhost-over- + // http (e.g. ``http://10.x.x.x:5173`` LAN dev) still works. + if (navigator.clipboard?.writeText) { + navigator.clipboard.writeText(text).then(finalize).catch(() => { + // Best-effort fallback if the async API rejects (permission / + // not focused / unsupported MIME, …). + try { + const ta = document.createElement('textarea') + ta.value = text + ta.style.position = 'fixed' + ta.style.opacity = '0' + document.body.appendChild(ta) + ta.select() + document.execCommand('copy') + document.body.removeChild(ta) + finalize() + } catch { + // Swallow — user can still drag-select the visible text. + } + }) + } else { + try { + const ta = document.createElement('textarea') + ta.value = text + ta.style.position = 'fixed' + ta.style.opacity = '0' + document.body.appendChild(ta) + ta.select() + document.execCommand('copy') + document.body.removeChild(ta) + finalize() + } catch { + // Swallow — drag-select still works. + } + } } // Row search: debounce keystrokes so we don't refire the rows fetch on // every character — the backend filters by conversation_id ILIKE %q% @@ -456,9 +470,6 @@ export default function CallImportDetail() { const [evalTranscribeMode, setEvalTranscribeMode] = useState< 'stt_llm' | 'llm_only' >('llm_only') - const [evalTranscriptSource, setEvalTranscriptSource] = useState< - 'production' | 'diarised' - >('diarised') const [evalSTT, setEvalSTT] = useState({ provider: null, model: null, @@ -2613,29 +2624,6 @@ export default function CallImportDetail() { )}
- {row.transcript && ( - - )}
{row.transcript ? ( @@ -2698,29 +2686,6 @@ export default function CallImportDetail() { )}
- {row.diarised_transcript && ( - - )} {Array.isArray(row.diarised_segments) && row.diarised_segments.length > 0 && ( - -
- {evalTranscriptSource === 'production' ? ( -

- Skips diarization and scores the transcript - imported from your CSV. Recordings are still - fetched from Exotel/Plivo when a recording URL - is mapped (needed for audio metrics). -

- ) : null} - - - {evalTranscriptSource === 'diarised' && - (() => { + {/* Diarisation is now mandatory for every eval + run — the checkbox is shown as always-on + (matching the spec) so the user knows the + STT picker below is required. The legacy + "Production vs Diarised" transcript-source + selector has been removed: runs always + score the diarised transcript. */} + {(() => { // In ``llm_only`` mode there is no STT — // the diariser LLM consumes the audio // directly. We adapt the "is the auto- @@ -4586,8 +4502,8 @@ export default function CallImportDetail() { {evalTranscribeMode === 'stt_llm' - ? "Rows that don't already have a diarised transcript are diarised first via the STT provider you pick below." - : "Rows that don't already have a diarised transcript are diarised by feeding the audio directly to the multimodal LLM you pick below. Calls longer than about 8 minutes need enough output headroom—if results look cut off, use STT + LLM diariser instead."} + ? "Every evaluation scores the diarised transcript. Rows that don't already have one are diarised first via the STT provider you pick below." + : "Every evaluation scores the diarised transcript. Rows that don't already have one are diarised by feeding the audio directly to the multimodal LLM you pick below. Calls longer than about 8 minutes need enough output headroom—if results look cut off, use STT + LLM diariser instead."} @@ -4816,27 +4732,28 @@ export default function CallImportDetail() { } else if (selectedMetricIds.length === 0) { disabledReasons.push('Select at least one metric to score.') } - // STT and diariser are only required when diarising. - if (evalTranscriptSource === 'diarised') { - if (evalTranscribeMode === 'stt_llm') { - if (!evalSTT.provider) { - disabledReasons.push( - 'Pick an STT provider (auto-diarisation is required).', - ) - } else if (!evalSTT.model) { - disabledReasons.push( - 'Pick an STT model for the selected provider.', - ) - } - } - if (!isLLMSelectionComplete(evalDiariserLLM, aiProviders)) { + // STT is only required in the legacy two-stage + // mode. ``llm_only`` mode feeds the audio + // straight to the diariser LLM, so the STT + // checks are gated on the active mode. + if (evalTranscribeMode === 'stt_llm') { + if (!evalSTT.provider) { + disabledReasons.push( + 'Pick an STT provider (auto-diarisation is required).', + ) + } else if (!evalSTT.model) { disabledReasons.push( - evalTranscribeMode === 'llm_only' - ? 'Pick a multimodal LLM provider — the recording is fed to it directly in LLM-only mode.' - : 'Pick a diariser LLM provider and model (or a Custom gateway credential).', + 'Pick an STT model for the selected provider.', ) } } + if (!isLLMSelectionComplete(evalDiariserLLM, aiProviders)) { + disabledReasons.push( + evalTranscribeMode === 'llm_only' + ? 'Pick a multimodal LLM provider — the recording is fed to it directly in LLM-only mode.' + : 'Pick a diariser LLM provider and model (or a Custom gateway credential).', + ) + } if ( aiProviders.length > 0 && isLLMSelectionPartial(runLLM, aiProviders) @@ -4924,7 +4841,9 @@ export default function CallImportDetail() { runEvaluationMutation.mutate({ metric_ids: selectedMetricIds, name: runDraftName.trim() || null, - transcript_sources: [evalTranscriptSource], + // Diarised is the only supported source + // now; the backend rejects anything else. + transcript_sources: ['diarised'], llm_provider: runLLMComplete ? runLLM.provider || null : null, @@ -4942,40 +4861,41 @@ export default function CallImportDetail() { metric_llm_overrides: Object.keys(overrides).length ? overrides : null, - auto_transcribe: evalTranscriptSource === 'diarised', - ...(evalTranscriptSource === 'diarised' - ? { - transcribe_overwrite: transcribeOverwrite, - transcribe_mode: evalTranscribeMode, - stt_provider: - evalTranscribeMode === 'stt_llm' - ? evalSTT.provider - : null, - stt_model: - evalTranscribeMode === 'stt_llm' - ? evalSTT.model - : null, - stt_credential_id: - evalTranscribeMode === 'stt_llm' - ? evalSTT.credential_id || null - : null, - stt_language: - evalTranscribeMode === 'stt_llm' - ? evalSTTLanguage.trim() || null - : null, - diarization_llm_provider: - evalDiariserLLM.provider, - diarization_llm_model: - resolveLLMModelForSubmit( - evalDiariserLLM, - aiProviders, - ) ?? evalDiariserLLM.model, - diarization_llm_credential_id: - evalDiariserLLM.credential_id || null, - diarization_prompt: - evalDiarisationPrompt.trim() || null, - } - : {}), + // Auto-diarise is always on; ``transcribe_mode`` + // decides whether the STT step actually runs + // (and therefore whether the STT fields are + // sent or nulled out for the backend's + // validator). + auto_transcribe: true, + transcribe_overwrite: transcribeOverwrite, + transcribe_mode: evalTranscribeMode, + stt_provider: + evalTranscribeMode === 'stt_llm' + ? evalSTT.provider + : null, + stt_model: + evalTranscribeMode === 'stt_llm' + ? evalSTT.model + : null, + stt_credential_id: + evalTranscribeMode === 'stt_llm' + ? evalSTT.credential_id || null + : null, + stt_language: + evalTranscribeMode === 'stt_llm' + ? evalSTTLanguage.trim() || null + : null, + diarization_llm_provider: + evalDiariserLLM.provider, + diarization_llm_model: + resolveLLMModelForSubmit( + evalDiariserLLM, + aiProviders, + ) ?? evalDiariserLLM.model, + diarization_llm_credential_id: + evalDiariserLLM.credential_id || null, + diarization_prompt: + evalDiarisationPrompt.trim() || null, discover_new_metrics: false, ...(data.status === 'mapped' ? (() => { diff --git a/frontend/src/pages/metrics/MetricsManagement.tsx b/frontend/src/pages/metrics/MetricsManagement.tsx index c359679c..5eefdbe8 100644 --- a/frontend/src/pages/metrics/MetricsManagement.tsx +++ b/frontend/src/pages/metrics/MetricsManagement.tsx @@ -8,8 +8,6 @@ import type { LLMGenerationConfig } from '../../config/llmGenerationParams' import { useToast } from '../../hooks/useToast' import { useWorkspaceStore } from '../../store/workspaceStore' import { - Copy, - ClipboardPaste, Edit, Trash2, X, @@ -26,13 +24,6 @@ import { Layers, AlertTriangle, } from 'lucide-react' -import { copyTextToClipboard, readTextFromClipboard } from '../../lib/clipboard' -import { - categoryFormFromMetricClipboard, - parseMetricClipboardPayload, - serializeMetricToClipboard, - singleFormFromMetricClipboard, -} from './metricClipboardUtils' import { categoryChildrenFromPartial, createCategoryChildrenFromPartial, @@ -205,7 +196,6 @@ export default function MetricsManagement() { description: string surfaces: MetricSurface[] capture_rationale: boolean - selection_mode: 'single_choice' | 'multi_label' // CREATE-time visibility scope (same semantics as ``formData.scope``). // Inherited by every child of the new category. scope: 'workspace' | 'organization' @@ -220,7 +210,6 @@ export default function MetricsManagement() { description: '', surfaces: ['agent'], capture_rationale: false, - selection_mode: 'single_choice', scope: 'workspace', children: [ { local_id: 'c1', name: '', description: '', example: '' }, @@ -255,7 +244,6 @@ export default function MetricsManagement() { const [pendingDeleteMetric, setPendingDeleteMetric] = useState( null, ) - const [pasteMetricError, setPasteMetricError] = useState(null) // --------------------------------------------------------------------------- // Edit-category form — only used when editing an existing PARENT // (``selection_mode`` set, no ``parent_metric_id``). Reuses the @@ -901,7 +889,6 @@ export default function MetricsManagement() { description: '', surfaces: ['agent'], capture_rationale: false, - selection_mode: 'single_choice', scope: 'workspace', children: [ { local_id: 'c1', name: '', description: '', example: '' }, @@ -1078,45 +1065,6 @@ export default function MetricsManagement() { setPendingDeleteMetric(null) } - const handleCopyMetric = (metric: Metric) => { - if (metric.parent_metric_id) { - showToast('Copy the parent category metric to include all labels.', 'error') - return - } - const payload = serializeMetricToClipboard(metric) - copyTextToClipboard(JSON.stringify(payload, null, 2), () => { - showToast('Metric copied to clipboard', 'success') - }) - } - - const handlePasteMetric = async () => { - setPasteMetricError(null) - try { - const text = await readTextFromClipboard() - const payload = parseMetricClipboardPayload(text) - const targetScope = 'workspace' as const - setEditingMetric(null) - setIsEditingCategory(false) - setIsCustomMetricMode(true) - if (payload.kind === 'category') { - setCreateMode('category') - setCategoryForm(categoryFormFromMetricClipboard(payload, targetScope)) - resetForm() - } else { - setCreateMode('single') - setFormData(singleFormFromMetricClipboard(payload, targetScope)) - resetCategoryForm() - } - setShowCreateModal(true) - showToast('Metric pasted — review scope and save', 'success') - } catch (err) { - const message = - err instanceof Error ? err.message : 'Could not paste metric from clipboard' - setPasteMetricError(message) - showToast(message, 'error') - } - } - const handleEdit = (metric: Metric) => { // A parent category metric: switch to the dedicated category // editor (children appear as editable rows inside the form). @@ -1309,7 +1257,6 @@ export default function MetricsManagement() { setEditingMetric(null) setIsEditingCategory(false) setCreateMode('single') - setPasteMetricError(null) resetForm() resetAIForm() resetCategoryForm() @@ -1856,33 +1803,14 @@ export default function MetricsManagement() { ? 'Create Custom Metric' : 'Create Metric'} -
- {!editingMetric && ( - - )} - -
+ - {pasteMetricError && !editingMetric && ( -
- {pasteMetricError} -
- )} - {/* Mode switcher: lets the user pick between the single metric and the parent category flow without leaving this modal. Hidden during edit because edit always @@ -2789,9 +2717,8 @@ export default function MetricsManagement() { name: categoryForm.name.trim(), description: categoryForm.description.trim() || null, - // Preserve pasted selection_mode; default flow - // remains single_choice. - selection_mode: categoryForm.selection_mode, + // Always single_choice for the new flow. + selection_mode: 'single_choice', allow_discovery: false, capture_rationale: categoryForm.capture_rationale, supported_surfaces: categoryForm.surfaces, @@ -3244,15 +3171,6 @@ export default function MetricsManagement() { > {m.enabled ? 'Disable' : 'Enable'} - {!isChild && ( - - )} {canDelete ? (