diff --git a/EfficientAI-Docs/.github/workflows/main.yml b/EfficientAI-Docs/.github/workflows/main.yml deleted file mode 100644 index 530d5932..00000000 --- a/EfficientAI-Docs/.github/workflows/main.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: Deploy to S3 - -on: - push: - branches: - - main # Change to the branch you want to deploy from - -jobs: - deploy: - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Install dependencies & Build - run: | - npm install # Modify for your project (e.g., yarn install, pip install) - npm run build # Modify if necessary - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v2 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} - - - name: Deploy to S3 - run: | - aws s3 sync ./build s3://${{ secrets.AWS_S3_BUCKET }} --delete - # Replace ./dist with your actual build folder (e.g., build, public, etc.) - - - name: Invalidating CloudFront cache - run: | - aws cloudfront create-invalidation --distribution-id ${{ secrets.AWS_CLOUDFRONT_DISTRIBUTION_ID }} --paths "/*" \ No newline at end of file diff --git a/EfficientAI-Docs/docs/getting-started/installation.md b/EfficientAI-Docs/docs/getting-started/installation.md index 213086bf..d91dd8be 100644 --- a/EfficientAI-Docs/docs/getting-started/installation.md +++ b/EfficientAI-Docs/docs/getting-started/installation.md @@ -77,16 +77,71 @@ pip install -e . eai init-config ``` -Edit `config.yml` with your database and Redis connection strings: +Edit `config.yml` with your settings: ```yaml +# EfficientAI Configuration File + +# Application Settings +app: + name: "Voice AI Evaluation Platform" + version: "0.1.0" + debug: true # Set to false in production + secret_key: "your-secret-key-here-change-in-production" + +# Server Settings +server: + host: "0.0.0.0" + port: 8000 + +# Database Configuration (Required) database: url: "postgresql://efficientai:password@localhost:5432/efficientai" +# Redis Configuration (Required) redis: url: "redis://localhost:6379/0" + +# Celery Configuration +celery: + broker_url: "redis://localhost:6379/0" + result_backend: "redis://localhost:6379/0" + +# File Storage +storage: + upload_dir: "./uploads" + max_file_size_mb: 500 + allowed_audio_formats: + - "wav" + - "mp3" + - "flac" + - "m4a" + +# S3 Configuration (Optional - for cloud audio storage) +s3: + enabled: false + bucket_name: "your-s3-bucket-name" + region: "us-east-1" + access_key_id: "your-access-key-id" + secret_access_key: "your-secret-access-key" + endpoint_url: null # For S3-compatible services (MinIO, DigitalOcean Spaces) + prefix: "audio/" + +# CORS Settings +cors: + origins: + - "http://localhost:3000" + - "http://localhost:8000" + +# API Settings +api: + prefix: "/api/v1" + key_header: "X-API-Key" + rate_limit_per_minute: 60 ``` +> **Important**: Make sure to change `secret_key` to a secure random value in production! + ### Start the application and worker **Option A: Start both together (Recommended)** diff --git a/EfficientAI-Docs/docs/getting-started/s3-storage.md b/EfficientAI-Docs/docs/getting-started/s3-storage.md new file mode 100644 index 00000000..0e83ff12 --- /dev/null +++ b/EfficientAI-Docs/docs/getting-started/s3-storage.md @@ -0,0 +1,152 @@ +--- +id: s3-storage +title: S3 Storage +sidebar_position: 2 +--- + +# S3 Cloud Storage + +## Overview + +EfficientAI can store audio files and recordings in Amazon S3 or any S3-compatible storage service (MinIO, DigitalOcean Spaces, etc.). + +This is useful for: +- Storing large audio files in the cloud +- Scaling storage independently from your server +- Integrating with existing cloud infrastructure + +--- + +## Configuration + +### 1. Update config.yml + +Add the S3 configuration section to your `config.yml`: + +```yaml +s3: + enabled: true + bucket_name: "your-bucket-name" + region: "us-east-1" + access_key_id: "AKIAIOSFODNN7EXAMPLE" + secret_access_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + endpoint_url: null # Optional: for S3-compatible services + prefix: "audio/" # Optional: folder prefix for files +``` + +### 2. Configuration Options + +| Option | Required | Description | +|--------|----------|-------------| +| `enabled` | Yes | Set to `true` to enable S3 storage | +| `bucket_name` | Yes | Name of your S3 bucket | +| `region` | Yes | AWS region (e.g., `us-east-1`, `eu-west-1`) | +| `access_key_id` | Yes | AWS Access Key ID | +| `secret_access_key` | Yes | AWS Secret Access Key | +| `endpoint_url` | No | Custom endpoint for S3-compatible services | +| `prefix` | No | Folder prefix for uploaded files (default: `audio/`) | + +--- + +## AWS Setup + +### 1. Create an S3 Bucket + +```bash +aws s3 mb s3://your-bucket-name --region us-east-1 +``` + +### 2. Create an IAM User + +Create a user with programmatic access and attach this policy: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:DeleteObject", + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::your-bucket-name", + "arn:aws:s3:::your-bucket-name/*" + ] + } + ] +} +``` + +### 3. Get Credentials + +After creating the IAM user, note the **Access Key ID** and **Secret Access Key**. + +--- + +## S3-Compatible Services + +### MinIO + +```yaml +s3: + enabled: true + bucket_name: "efficientai" + region: "us-east-1" + access_key_id: "minioadmin" + secret_access_key: "minioadmin" + endpoint_url: "http://localhost:9000" + prefix: "audio/" +``` + +### DigitalOcean Spaces + +```yaml +s3: + enabled: true + bucket_name: "your-space-name" + region: "nyc3" + access_key_id: "your-spaces-key" + secret_access_key: "your-spaces-secret" + endpoint_url: "https://nyc3.digitaloceanspaces.com" + prefix: "audio/" +``` + +### Cloudflare R2 + +```yaml +s3: + enabled: true + bucket_name: "your-bucket" + region: "auto" + access_key_id: "your-r2-access-key" + secret_access_key: "your-r2-secret-key" + endpoint_url: "https://.r2.cloudflarestorage.com" + prefix: "audio/" +``` + +--- + +## Verifying Connection + +After configuring S3, restart the application: + +```bash +eai start-all --config config.yml +``` + +Upload a test file through the UI or API. Check your S3 bucket to confirm the file appears under the configured prefix. + +--- + +## Troubleshooting + +| Issue | Solution | +|-------|----------| +| `Access Denied` | Check IAM permissions and bucket policy | +| `NoSuchBucket` | Verify bucket name and region | +| `Connection refused` | Check endpoint_url for S3-compatible services | +| `Invalid credentials` | Verify access_key_id and secret_access_key | diff --git a/EfficientAI-Docs/docs/monitoring/_category_.json b/EfficientAI-Docs/docs/monitoring/_category_.json new file mode 100644 index 00000000..35e270cc --- /dev/null +++ b/EfficientAI-Docs/docs/monitoring/_category_.json @@ -0,0 +1,8 @@ +{ + "label": "Monitoring", + "position": 4, + "link": { + "type": "generated-index", + "description": "Monitor your Voice AI in production with observability, alerting, and scheduled tests." + } +} \ No newline at end of file diff --git a/EfficientAI-Docs/docs/monitoring/alerting.md b/EfficientAI-Docs/docs/monitoring/alerting.md new file mode 100644 index 00000000..d9fe1c1a --- /dev/null +++ b/EfficientAI-Docs/docs/monitoring/alerting.md @@ -0,0 +1,126 @@ +--- +id: alerting +title: Alerting +sidebar_position: 2 +--- + +# Alerting + +## What is Alerting? + +**Alerting** lets you set up automated notifications when your Voice AI metrics cross certain thresholds. + +Instead of manually checking dashboards, you can configure alerts to notify you via email or webhook (Slack, etc.) when something needs attention. + +--- + +## Creating an Alert + +An alert is defined by: + +1. **Metric**: What are you measuring? +2. **Condition**: When should it trigger? +3. **Notification**: How should you be notified? + +--- + +## Available Metrics + +| Metric | Description | +|--------|-------------| +| **Number of Calls** | Total call count | +| **Call Duration** | Average or total call length | +| **Error Rate** | Percentage of failed calls | +| **Success Rate** | Percentage of successful calls | +| **Latency** | Response time | +| **Custom** | Your own defined metrics | + +--- + +## Aggregation Types + +How the metric is calculated over the time window: + +| Aggregation | Description | +|-------------|-------------| +| **Sum** | Total value | +| **Average** | Mean value | +| **Count** | Number of occurrences | +| **Min** | Minimum value | +| **Max** | Maximum value | + +--- + +## Operators + +| Operator | Meaning | +|----------|---------| +| `>` | Greater than | +| `<` | Less than | +| `>=` | Greater than or equal | +| `<=` | Less than or equal | +| `=` | Equal to | +| `!=` | Not equal to | + +--- + +## Example Alert + +> "Alert me when the **average latency** is **greater than 500ms** in the last **60 minutes**" + +Configuration: +- **Metric**: Latency +- **Aggregation**: Average +- **Operator**: > +- **Threshold**: 500 +- **Time Window**: 60 minutes + +--- + +## Notification Options + +### Email +Add one or more email addresses to receive alert notifications. + +### Webhooks +Send alerts to Slack, Discord, or any webhook-compatible service. + +Example Slack webhook: +``` +https://hooks.slack.com/services/T00000000/B00000000/XXXXXXXX +``` + +--- + +## Notification Frequency + +Control how often you receive notifications: + +| Frequency | Description | +|-----------|-------------| +| **Immediate** | Send as soon as triggered | +| **Hourly** | Batch notifications hourly | +| **Daily** | Daily digest | +| **Weekly** | Weekly summary | + +--- + +## Alert Status + +| Status | Description | +|--------|-------------| +| **Active** | Alert is monitoring and will trigger | +| **Paused** | Alert is temporarily disabled | +| **Disabled** | Alert is turned off | + +--- + +## Alert History + +When an alert is triggered, a history record is created with: +- Trigger timestamp +- Metric value at trigger +- Acknowledgement status +- Resolution notes + +View alert history in the Alerts dashboard to track incidents over time. diff --git a/EfficientAI-Docs/docs/monitoring/cron-jobs.md b/EfficientAI-Docs/docs/monitoring/cron-jobs.md new file mode 100644 index 00000000..39c1b74c --- /dev/null +++ b/EfficientAI-Docs/docs/monitoring/cron-jobs.md @@ -0,0 +1,110 @@ +--- +id: cron-jobs +title: Scheduled Tests (Cron Jobs) +sidebar_position: 3 +--- + +# Scheduled Tests (Cron Jobs) + +## What are Cron Jobs? + +**Cron Jobs** let you schedule automated test calls to run on a recurring basis. + +Instead of manually triggering evaluations, you can set up a schedule like "Run this test every day at 9 AM" and EfficientAI will automatically execute it. + +--- + +## Use Cases + +- **Daily Health Checks**: Run a test every morning to ensure your AI is working +- **Regression Testing**: Automatically test after deployments +- **Performance Monitoring**: Continuous testing throughout the day +- **Off-Hours Testing**: Run tests when traffic is low + +--- + +## Creating a Cron Job + +A cron job requires: + +1. **Name**: A descriptive name for the schedule +2. **Cron Expression**: When to run (standard cron format) +3. **Timezone**: Which timezone to use +4. **Evaluators**: Which tests to run +5. **Max Runs** (optional): Limit total executions + +--- + +## Cron Expression Format + +Standard 5-field cron format: + +``` +┌───────────── minute (0-59) +│ ┌───────────── hour (0-23) +│ │ ┌───────────── day of month (1-31) +│ │ │ ┌───────────── month (1-12) +│ │ │ │ ┌───────────── day of week (0-6, Sun=0) +│ │ │ │ │ +* * * * * +``` + +### Examples + +| Expression | Description | +|------------|-------------| +| `0 9 * * *` | Every day at 9:00 AM | +| `0 */2 * * *` | Every 2 hours | +| `30 8 * * 1-5` | 8:30 AM on weekdays | +| `0 0 1 * *` | First day of each month at midnight | +| `*/15 * * * *` | Every 15 minutes | + +--- + +## Timezone Support + +All standard timezones are supported (pytz format): + +- `America/New_York` +- `Europe/London` +- `Asia/Kolkata` +- `UTC` + +The cron expression is evaluated in your selected timezone. + +--- + +## Status + +| Status | Description | +|--------|-------------| +| **Active** | Job is scheduled and will run | +| **Paused** | Job is temporarily stopped | +| **Completed** | Job hit max runs and stopped | + +--- + +## Example Setup + +**"Daily Morning Test"** + +``` +Name: Daily Morning Test +Cron: 0 9 * * * +Timezone: America/New_York +Evaluators: [Support Bot Test, Sales Bot Test] +Max Runs: (empty = unlimited) +``` + +This runs the selected evaluators every day at 9 AM Eastern time. + +--- + +## Managing Cron Jobs + +From the Cron Jobs dashboard you can: +- **Create** new scheduled tests +- **Pause/Resume** existing jobs +- **Edit** schedule or evaluators +- **Delete** jobs you no longer need +- **View** next scheduled run time diff --git a/EfficientAI-Docs/docs/monitoring/observability.md b/EfficientAI-Docs/docs/monitoring/observability.md new file mode 100644 index 00000000..d7392986 --- /dev/null +++ b/EfficientAI-Docs/docs/monitoring/observability.md @@ -0,0 +1,112 @@ +--- +id: observability +title: Observability +sidebar_position: 1 +--- + +# Observability + +## What is Observability? + +**Observability** lets you track and monitor all calls made by your Voice AI in production. + +Instead of only testing with simulated calls, you can connect your live Voice AI to EfficientAI and see real call data flowing in. This gives you visibility into: +- What calls are happening +- How your AI is performing in the real world +- Issues that occur in production + +--- + +## How It Works + +EfficientAI provides webhook endpoints that your Voice AI provider can send call events to: + +``` +POST /api/v1/observability/calls +``` + +When a call starts, ends, or has an event, your provider sends the data to EfficientAI, and we store and analyze it. + +--- + +## Supported Providers + +EfficientAI can ingest calls from any voice AI platform: + +| Provider | Integration Type | +|----------|-----------------| +| **Retell** | Dedicated webhook (no API key needed) | +| **Vapi** | Generic webhook | +| **Custom** | Generic webhook | + +--- + +## Setting Up Observability + +### For Retell + +Use the dedicated Retell webhook endpoint (no EfficientAI API key required): + +``` +POST https://your-domain.com/api/v1/observability/calls/retell/webhook +``` + +Configure this URL in your Retell dashboard under webhook settings. + +### For Other Providers + +Use the generic webhook endpoint with your API key: + +```bash +curl -X POST https://your-domain.com/api/v1/observability/calls \ + -H "X-EFFICIENTAI-API-KEY: your-api-key" \ + -H "Content-Type: application/json" \ + -d '{ + "provider_platform": "vapi", + "provider_call_id": "call_abc123", + "agent_id": "your-agent-uuid", + "call_data": { + "event": "call_ended", + "duration": 120, + "transcript": "..." + } + }' +``` + +--- + +## Viewing Call Data + +Once calls are ingested, you can: + +1. **List all calls**: `GET /api/v1/observability/calls` +2. **View call details**: `GET /api/v1/observability/calls/{call_short_id}` +3. **Delete a call**: `DELETE /api/v1/observability/calls/{call_short_id}` + +All calls appear in the Observability dashboard in the frontend. + +--- + +## Webhook Payload Format + +The webhook accepts flexible payloads to support different providers: + +```json +{ + "provider_platform": "retell", + "provider_call_id": "call_123", + "agent_id": "agent-uuid-or-external-id", + "call_data": { + "event": "call_ended", + "duration": 120, + "transcript": "Hello, how can I help you today?..." + } +} +``` + +| Field | Required | Description | +|-------|----------|-------------| +| `provider_platform` | Yes | Name of the voice AI provider | +| `provider_call_id` | Yes | The call ID from the provider | +| `agent_id` | No | Links to your EfficientAI Agent | +| `call_data` | Yes | Full call payload from provider | diff --git a/EfficientAI-Docs/docs/products/metrics.md b/EfficientAI-Docs/docs/products/metrics.md index 3d41d26c..6bd606e1 100644 --- a/EfficientAI-Docs/docs/products/metrics.md +++ b/EfficientAI-Docs/docs/products/metrics.md @@ -13,30 +13,69 @@ sidebar_position: 5 After a test call finishes, you want to know how well it went. EfficientAI answers this with data. We give you two types of grades: -1. **Hard Numbers**: Things like "How fast did it answer?" (Latency) or "Did it mishear any words?" (Accuracy). -2. **Quality Scores**: Did the AI sound friendly? Did it solve the customer's problem? +1. **Quantitative Metrics**: Hard numbers like latency, speaking rate, and voice quality measurements. +2. **Qualitative Metrics**: Subjective scores like how human the AI sounds or emotional accuracy. -You check these metrics to see if your AI is ready for the real world. +--- + +## Quantitative Metrics + +These are calculated automatically from the audio and conversation data: + +| Metric | Description | +|--------|-------------| +| **E2E Latency** | End-to-end response time — how fast the AI responds | +| **Pitch Variance** | Voice pitch variation analysis | +| **Jitter & Shimmer** | Voice quality fluctuations — stability of the voice | +| **Speaking Rate (WPM)** | Words per minute — conversation pacing | +| **Interruption Gap** | Time between speaker turns | +| **HNR** | Harmonics-to-Noise Ratio — voice clarity measurement | +| **Turn Taking** | Speaker transition patterns and timing | +| **Barge-In Interruption** | Detection of when speakers talk over each other | +| **Silence Duration** | Length and frequency of pauses | + +### How They Work + +Quantitative metrics are computed by analyzing: +- Audio waveforms (for pitch, jitter, shimmer, HNR) +- Timestamps (for latency, turn taking, gaps) +- Transcriptions (for speaking rate) --- -## Technical Details +## Qualitative Metrics -Metrics are the quantitative and qualitative measurements used to assess performance. +These are evaluated using LLM analysis of the conversation: -### Technical Metrics (Quantitative) +| Metric | Description | +|--------|-------------| +| **Human Likeness (HPDR-5)** | How natural and human the AI sounds | +| **MOS** | Mean Opinion Score — overall quality rating | +| **Emotional Match Accuracy** | How well the AI matches appropriate emotions | +| **Valence/Arousal** | Emotional intensity and positivity analysis | +| **Prosody Expressiveness** | Speech rhythm, intonation, and expression quality | +| **Speaker Consistency** | Voice consistency across the conversation | -These are calculated automatically by the `MetricsService`: +### How They Work -* **Word Error Rate (WER)**: Accuracy of the Agent's speech recognition. calculated by comparing what was said vs. what was transcribed. 0.0 is perfect; higher is worse. -* **Character Error Rate (CER)**: Similar to WER but at the character level. -* **Latency**: How long the system takes to process inputs (in milliseconds). -* **Real-Time Factor (RTF)**: `Processing Time / Audio Duration`. Needs to be `< 1.0` for real-time performance. +Qualitative metrics use an LLM to analyze the conversation transcript and audio characteristics, providing subjective assessments that would normally require human evaluation. -### Custom Metrics (Qualitative) +--- + +## Custom Metrics + +You can define your own custom metrics for specific use cases: -You can define custom metrics (like "Empathy" or "Resolution Success") which can be rated on a scale (Type: `rating`) or as a pass/fail (Type: `boolean`). +- **Rating Type**: Score on a scale (e.g., 1-5) +- **Boolean Type**: Pass/fail assessment + +### Examples +- "Resolution Success" — Did the AI solve the customer's problem? +- "Empathy Score" — Was the AI appropriately empathetic? +- "Upsell Attempted" — Did the AI try to upsell? + +--- -### Storage +## Storage -Metric results are stored in the JSON `metric_scores` column of the `EvaluatorResult` database table. +Metric results are stored in the `metric_scores` JSON column of the `EvaluatorResult` database table and can be viewed in the Results Dashboard. diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 910e4c6b..8276ed0b 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -28,15 +28,16 @@ import Settings from './pages/Settings' import Alerts from './pages/Alerts' import AlertHistory from './pages/AlertHistory' import CronJobs from './pages/CronJobs' +import VoicePlayground from './pages/VoicePlayground' function PrivateRoute({ children }: { children: React.ReactNode }) { const { apiKey } = useAuthStore() - + if (!apiKey) { return } - + return <>{children} } @@ -66,6 +67,7 @@ function App() { } /> } /> } /> + } /> } /> } /> } /> diff --git a/frontend/src/components/Layout.tsx b/frontend/src/components/Layout.tsx index cd4def18..3292a8ba 100644 --- a/frontend/src/components/Layout.tsx +++ b/frontend/src/components/Layout.tsx @@ -59,6 +59,7 @@ const navigationSections: NavSection[] = [ icon: FileCheck, items: [ { name: 'Playground', href: '/playground', icon: Play }, + { name: 'Voice Playground', href: '/voice-playground', icon: Mic }, { name: 'Evaluators', href: '/evaluate-test-agents', icon: Mic }, { name: 'Evaluation Results', href: '/results', icon: BarChart3 }, ], @@ -264,7 +265,7 @@ export default function Layout() { )} - + {/* Profile Link */} @@ -309,11 +310,10 @@ function ProfileAvatar() {
diff --git a/frontend/src/pages/VoicePlayground.tsx b/frontend/src/pages/VoicePlayground.tsx new file mode 100644 index 00000000..e6145289 --- /dev/null +++ b/frontend/src/pages/VoicePlayground.tsx @@ -0,0 +1,731 @@ +import { useState, useMemo, useRef } from 'react' +import { useQuery } from '@tanstack/react-query' +import { apiClient } from '../lib/api' +import Button from '../components/Button' +import { + Play, + Pause, + Volume2, + Trophy, + BarChart3, + DollarSign, + Clock, + Mic, + Share2, + RotateCcw, + Plus, + X, + Loader2, + FileText +} from 'lucide-react' + +// ============ MOCK DATA ============ + +const TTS_PROVIDERS = [ + { id: 'murf-falcon', name: 'Murf Falcon (2024)', costPer1M: 1.60, costPer100kMin: 1200 }, + { id: 'elevenlabs-turbo', name: 'ElevenLabs Turbo v2.5', costPer1M: 4.20, costPer100kMin: 3100 }, + { id: 'cartesia-sonic', name: 'Cartesia Sonic', costPer1M: 2.40, costPer100kMin: 1800 }, + { id: 'azure-neural', name: 'Azure Neural TTS', costPer1M: 1.00, costPer100kMin: 750 }, + { id: 'google-wavenet', name: 'Google WaveNet', costPer1M: 1.60, costPer100kMin: 1200 }, + { id: 'amazon-polly', name: 'Amazon Polly Neural', costPer1M: 0.80, costPer100kMin: 600 }, +] + +const SAMPLE_TRANSCRIPTS = [ + "Hello! Thank you for calling customer support. How may I assist you today?", + "Your order number is 1-2-3-4-5-6-7-8-9. It will be delivered on January 15th, 2025.", + "I understand your concern. Let me look into this for you right away.", + "The total amount due is $1,234.56. Would you like to proceed with the payment?", + "Is there anything else I can help you with today? We appreciate your business!" +] + +const PROVIDER_VOICES: Record = { + 'murf-falcon': [ + { id: 'murf-aisha', name: 'Aisha', gender: 'Female', accent: 'Indian', language: 'English' }, + { id: 'murf-james', name: 'James', gender: 'Male', accent: 'American', language: 'English' }, + { id: 'murf-sofia', name: 'Sofia', gender: 'Female', accent: 'British', language: 'English' }, + ], + 'elevenlabs-turbo': [ + { id: 'el-rachel', name: 'Rachel', gender: 'Female', accent: 'American', language: 'English' }, + { id: 'el-adam', name: 'Adam', gender: 'Male', accent: 'American', language: 'English' }, + { id: 'el-bella', name: 'Bella', gender: 'Female', accent: 'American', language: 'English' }, + ], + 'cartesia-sonic': [ + { id: 'cs-nova', name: 'Nova', gender: 'Female', accent: 'American', language: 'English' }, + { id: 'cs-echo', name: 'Echo', gender: 'Male', accent: 'British', language: 'English' }, + ], + 'azure-neural': [ + { id: 'az-jenny', name: 'Jenny', gender: 'Female', accent: 'American', language: 'English' }, + { id: 'az-guy', name: 'Guy', gender: 'Male', accent: 'American', language: 'English' }, + ], + 'google-wavenet': [ + { id: 'gw-wavenet-a', name: 'WaveNet A', gender: 'Female', accent: 'American', language: 'English' }, + { id: 'gw-wavenet-b', name: 'WaveNet B', gender: 'Male', accent: 'American', language: 'English' }, + ], + 'amazon-polly': [ + { id: 'ap-joanna', name: 'Joanna', gender: 'Female', accent: 'American', language: 'English' }, + { id: 'ap-matthew', name: 'Matthew', gender: 'Male', accent: 'American', language: 'English' }, + ], +} + +const AGENT_PLATFORMS = [ + { id: 'vapi', name: 'Vapi' }, + { id: 'retell', name: 'Retell' }, + { id: 'bland', name: 'Bland AI' }, + { id: 'vocode', name: 'Vocode' }, +] + +const MOCK_CALL_TRANSCRIPTS = [ + { + agentLine: "Hello! Thank you for calling customer support. My name is Sarah, how may I assist you today?", + customerLine: "Hi Sarah, I'm calling about my recent order. I haven't received it yet.", + }, + { + agentLine: "I'd be happy to help you with that. Could you please provide me with your order number?", + customerLine: "Yes, it's 1-2-3-4-5-6-7-8-9.", + }, + { + agentLine: "Thank you. I can see your order was shipped on January 10th. It's scheduled to arrive by January 15th, 2025.", + customerLine: "Okay, that's reassuring. Can you confirm the delivery address?", + }, + { + agentLine: "Of course! The delivery address is 123 Main Street, Apartment 4B, New York, NY 10001.", + customerLine: "That's correct. Thank you for checking.", + }, +] + +// Generate mock results with per-call metrics +const generateMockResults = (providerA: string, providerB: string, voicesA: string[], voicesB: string[], numCalls: number, mode: 'agent' | 'tts' = 'agent', ttsText: string = '') => { + const provA = TTS_PROVIDERS.find(p => p.id === providerA) + const provB = TTS_PROVIDERS.find(p => p.id === providerB) + + const callRecordings = Array.from({ length: numCalls }, (_, i) => ({ + id: `call-${i + 1}`, + callNumber: i + 1, + duration: Math.floor(Math.random() * 120 + 30), + transcripts: mode === 'tts' + ? [{ agentLine: ttsText, customerLine: null }] + : MOCK_CALL_TRANSCRIPTS.slice(0, Math.floor(Math.random() * 3 + 2)), + voiceSamples: { + A: voicesA.map(vId => { + const v = PROVIDER_VOICES[providerA]?.find(p => p.id === vId) + return { id: vId, name: v?.name || vId } + }), + B: voicesB.map(vId => { + const v = PROVIDER_VOICES[providerB]?.find(p => p.id === vId) + return { id: vId, name: v?.name || vId } + }) + }, + metrics: { + ttfb: { A: Math.floor(Math.random() * 200 + 300), B: Math.floor(Math.random() * 300 + 400) }, + gibberishRate: { A: (Math.random() * 8).toFixed(1), B: (Math.random() * 12).toFixed(1) }, + pronunciationAccuracy: { A: (Math.random() * 8 + 92).toFixed(1), B: (Math.random() * 12 + 85).toFixed(1) }, + emotionalMatch: { A: Math.floor(Math.random() * 15 + 80), B: Math.floor(Math.random() * 20 + 70) }, + mos: { A: (Math.random() * 0.8 + 4.0).toFixed(2), B: (Math.random() * 0.8 + 3.8).toFixed(2) }, + } + })) + + return { + providerA: provA?.name || providerA, + providerB: provB?.name || providerB, + winner: Math.random() > 0.5 ? 'A' : 'B', + winnerPreference: Math.floor(Math.random() * 30 + 55), + callRecordings, + metrics: { + blindPreference: { A: Math.floor(Math.random() * 30 + 50), B: Math.floor(Math.random() * 30 + 30) }, + gibberishRate: { A: (Math.random() * 5 + 0.5).toFixed(1), B: (Math.random() * 8 + 1).toFixed(1) }, + pronunciationAccuracy: { A: (Math.random() * 5 + 94).toFixed(1), B: (Math.random() * 8 + 88).toFixed(1) }, + emotionalMatch: { A: Math.floor(Math.random() * 10 + 85), B: Math.floor(Math.random() * 15 + 75) }, + mos: { A: (Math.random() * 0.5 + 4.3).toFixed(2), B: (Math.random() * 0.5 + 4.1).toFixed(2) }, + avgTTFB: { A: Math.floor(Math.random() * 200 + 350), B: Math.floor(Math.random() * 300 + 500) }, + costPer1M: { A: provA?.costPer1M || 1.60, B: provB?.costPer1M || 4.20 }, + costPer100kMin: { A: provA?.costPer100kMin || 1200, B: provB?.costPer100kMin || 3100 }, + } + } +} + +interface CustomVoice { + id: string + name: string + gender: string + accent: string + language: string +} + +function VoicePreview({ voice, isPlaying, onPlay, colorScheme = 'blue' }: { + voice: CustomVoice + isPlaying: boolean + onPlay: () => void + colorScheme?: 'blue' | 'purple' +}) { + const bgColor = colorScheme === 'blue' ? 'bg-blue-100' : 'bg-purple-100' + const textColor = colorScheme === 'blue' ? 'text-blue-700' : 'text-purple-700' + const buttonColor = colorScheme === 'blue' ? 'bg-blue-600 hover:bg-blue-700' : 'bg-purple-600 hover:bg-purple-700' + + return ( +
+
+ +
+
+ {voice.name} + • {voice.gender} • {voice.accent} +
+

"{SAMPLE_TRANSCRIPTS[0]}"

+
+
+
+ ) +} + +export default function VoicePlayground() { + const [providerA, setProviderA] = useState('') + const [providerB, setProviderB] = useState('') + const [selectedVoicesA, setSelectedVoicesA] = useState([]) + const [selectedVoicesB, setSelectedVoicesB] = useState([]) + + const [selectedScenario, setSelectedScenario] = useState('') + const [customScenario, setCustomScenario] = useState('') + const [useCustomScenario, setUseCustomScenario] = useState(false) + const [selectedAgent, setSelectedAgent] = useState('') + const [numberOfCalls, setNumberOfCalls] = useState(5) + + const [playingVoice, setPlayingVoice] = useState(null) + const audioRef = useRef(null) + + const [isRunning, setIsRunning] = useState(false) + const [results, setResults] = useState | null>(null) + + const [selectedTranscript, setSelectedTranscript] = useState(0) + + // Evaluation mode state + const [evaluationMode, setEvaluationMode] = useState<'agent' | 'tts'>('agent') + const [ttsText, setTtsText] = useState(SAMPLE_TRANSCRIPTS[0]) + + // Cost calculator state + const [costMinutes, setCostMinutes] = useState(100000) + const [costCharacters, setCostCharacters] = useState(1000000) + + const { data: scenarios = [] } = useQuery({ + queryKey: ['scenarios'], + queryFn: () => apiClient.listScenarios(), + }) + + const voicesA = useMemo(() => providerA ? (PROVIDER_VOICES[providerA] || []) : [], [providerA]) + const voicesB = useMemo(() => providerB ? (PROVIDER_VOICES[providerB] || []) : [], [providerB]) + + const activeVoicesA = useMemo(() => voicesA.filter(v => selectedVoicesA.includes(v.id)), [voicesA, selectedVoicesA]) + const activeVoicesB = useMemo(() => voicesB.filter(v => selectedVoicesB.includes(v.id)), [voicesB, selectedVoicesB]) + + const canRunTest = () => { + const basicCheck = providerA && providerB && providerA !== providerB && selectedVoicesA.length > 0 && selectedVoicesB.length > 0 && numberOfCalls >= 1 + if (evaluationMode === 'agent') { + return basicCheck && (useCustomScenario ? customScenario.trim() : selectedScenario) && selectedAgent + } + return basicCheck && ttsText.trim() + } + + const handlePlayVoice = (voiceId: string) => { + if (playingVoice === voiceId) { + setPlayingVoice(null) + } else { + setPlayingVoice(voiceId) + setTimeout(() => setPlayingVoice(null), 3000) + } + } + + const handleRunTest = () => { + if (!canRunTest()) return + setIsRunning(true) + setResults(null) + setTimeout(() => { + setResults(generateMockResults(providerA, providerB, selectedVoicesA, selectedVoicesB, numberOfCalls, evaluationMode, ttsText)) + setIsRunning(false) + }, 2000) + } + + const resetPlayground = () => { + setProviderA('') + setProviderB('') + setSelectedVoicesA([]) + setSelectedVoicesB([]) + setSelectedScenario('') + setCustomScenario('') + setSelectedAgent('') + setNumberOfCalls(5) + setResults(null) + setPlayingVoice(null) + setEvaluationMode('agent') + setTtsText(SAMPLE_TRANSCRIPTS[0]) + } + + return ( +
+ {/* Header */} +
+
+

+ + Voice Playground +

+

A/B test TTS providers — Compare voice quality, latency, and cost

+
+ {(providerA || providerB || results) && ( + + )} +
+ + {/* Mode Switcher */} +
+
+ + +
+
+ + {/* Sample Transcripts */} +
+
+ +

Sample Transcript

+
+
+ {SAMPLE_TRANSCRIPTS.map((_, idx) => ( + + ))} +
+

"{SAMPLE_TRANSCRIPTS[selectedTranscript]}"

+
+ + {/* TTS Input */} + {evaluationMode === 'tts' && ( +
+
+ + Text to Speak +
+