The Conversation Evaluation RPC API provides advanced evaluation operations including on-demand evaluation runs, comparative analysis, and automated quality benchmarking. These endpoints enable sophisticated evaluation workflows and continuous quality improvement.
Base URL: https://your-api-name.mercury.ratiomachina.com Authentication: x-mercury-api-key: YOUR_API_KEY
🚀 Primary RPC Endpoint: Trigger on-demand persona evaluation runs with custom parameters, sample conversations, and analysis depth. Perfect for testing persona changes, A/B testing, and quality audits.
{
"sampling": {
"method": "stratified_random",
"sampleSize": 100,
"criteria": {
"dateRange": {
"from": "2024-01-08T00:00:00Z",
"to": "2024-01-15T23:59:59Z"
},
"channels": ["email", "chat", "sms"],
"conversationTypes": ["support", "sales"],
"minMessageCount": 3,
"excludeResolved": false
},
"stratification": {
"byChannel": true,
"byCustomerTier": true,
"ensureRepresentation": true
}
},
"evaluation": {
"dimensions": [
{
"name": "accuracy",
"weight": 0.35,
"customCriteria": [
"solution_effectiveness",
"technical_correctness"
]
},
{
"name": "customer_satisfaction",
"weight": 0.30,
"customCriteria": [
"satisfaction_prediction",
"resolution_quality"
]
}
],
"scoringOverride": {
"scale": "1_to_10",
"passingThreshold": 7.5,
"includeConfidence": true
}
},
"analysis": {
"includeInsights": true,
"generateRecommendations": true,
"comparisons": {
"toBaseline": true,
"toLastRun": true,
"toPreviousPeriod": "7d"
}
},
"reporting": {
"format": "comprehensive",
"includeConversationSamples": true,
"maxSamples": 10,
"includeVisualizations": false
}
}{
"personaEvalRun": {
"id": "uuid-run-789",
"personaEvalJobId": "uuid-job-456",
"status": "running",
"progress": {
"phase": "sampling_conversations",
"completed": 25,
"total": 100,
"estimatedCompletion": "2024-01-15T10:33:00Z"
},
"sampling": {
"method": "stratified_random",
"sampleSize": 100,
"conversationsFound": 1247,
"stratificationResults": {
"email": 35,
"chat": 40,
"sms": 25
}
},
"createdAt": "2024-01-15T10:30:00Z"
},
"job": {
"id": "uuid-job-456",
"name": "Customer Support Quality Assessment",
"personaId": "uuid-persona-123"
},
"execution": {
"priority": "normal",
"estimatedDuration": "3-5 minutes",
"resourceAllocation": "standard"
}
}Run evaluations with built-in comparisons to baselines, previous periods, or different persona configurations. Ideal for measuring the impact of changes and identifying trends.
{
"analysis": {
"type": "before_after_comparison",
"beforePeriod": {
"from": "2024-01-01T00:00:00Z",
"to": "2024-01-07T23:59:59Z",
"description": "Pre-training baseline"
},
"afterPeriod": {
"from": "2024-01-08T00:00:00Z",
"to": "2024-01-14T23:59:59Z",
"description": "Post-training performance"
},
"sampleSize": 150,
"matchedPairing": true
},
"evaluation": {
"focusDimensions": [
"accuracy",
"efficiency",
"customer_satisfaction"
],
"statisticalTests": {
"tTest": true,
"confidenceLevel": 0.95,
"effectSizeCalculation": true
}
},
"reporting": {
"includeStatistics": true,
"visualizeComparison": true,
"highlightSignificantChanges": true
}
}{
"analysis": {
"type": "multi_persona_comparison",
"personas": [
{
"personaId": "uuid-persona-sales-v1",
"name": "Sales Assistant V1",
"sampleSize": 75
},
{
"personaId": "uuid-persona-sales-v2",
"name": "Sales Assistant V2",
"sampleSize": 75
}
],
"controlledVariables": {
"sameTimeRange": true,
"sameChannels": true,
"similarCustomerProfiles": true
}
},
"evaluation": {
"dimensions": [
"conversion_effectiveness",
"customer_engagement",
"objection_handling"
],
"pairwiseComparisons": true
}
}{
"personaEvalRun": {
"id": "uuid-run-comparative-123",
"type": "comparative_analysis",
"status": "completed",
"overallScore": 8.2,
"comparison": {
"baselineScore": 7.6,
"improvement": "+0.6",
"statisticalSignificance": {
"pValue": 0.003,
"significant": true,
"confidenceLevel": 0.95,
"effectSize": "medium"
}
},
"dimensionComparisons": [
{
"dimension": "accuracy",
"currentScore": 8.5,
"baselineScore": 8.1,
"change": "+0.4",
"significant": false
},
{
"dimension": "efficiency",
"currentScore": 8.8,
"baselineScore": 7.3,
"change": "+1.5",
"significant": true
}
],
"insights": {
"keyFindings": [
"Significant improvement in response efficiency",
"Maintained high accuracy standards",
"Customer satisfaction scores increased"
],
"recommendations": [
"Deploy updated persona to production",
"Monitor efficiency metrics closely",
"Consider similar improvements for other personas"
]
}
}
}Define custom evaluation criteria and scoring models for specific use cases or business requirements.
{
"evaluation": {
"customDimensions": [
{
"name": "medical_accuracy",
"weight": 0.4,
"criteria": [
{
"name": "symptom_recognition",
"description": "Ability to identify symptoms",
"weight": 0.3,
"evaluationMethod": "expert_review"
},
{
"name": "treatment_recommendations",
"description": "Appropriateness of suggestions",
"weight": 0.4,
"evaluationMethod": "clinical_guidelines"
},
{
"name": "safety_protocols",
"description": "Adherence to safety guidelines",
"weight": 0.3,
"evaluationMethod": "compliance_check"
}
]
}
],
"scoringModel": {
"type": "custom_weighted",
"safeguards": {
"criticalFailures": ["safety_protocol_violation"],
"automaticFailThreshold": 0.0
}
}
},
"sampling": {
"criteria": {
"conversationTags": ["medical_inquiry"],
"requiresExpertValidation": true
}
}
}{
"evaluation": {
"businessAlignment": {
"kpis": [
{
"name": "customer_retention_prediction",
"weight": 0.3,
"calculation": "predictive_model",
"targetValue": 0.85
},
{
"name": "upsell_opportunity_identification",
"weight": 0.25,
"calculation": "opportunity_scoring",
"targetValue": 0.70
},
{
"name": "resolution_time_optimization",
"weight": 0.25,
"calculation": "efficiency_metric",
"targetValue": 120
},
{
"name": "brand_sentiment_impact",
"weight": 0.2,
"calculation": "sentiment_analysis",
"targetValue": 0.75
}
]
},
"scoringModel": {
"type": "kpi_weighted",
"normalization": "target_based",
"bonusPenalty": {
"exceedsTarget": 1.1,
"belowThreshold": 0.8
}
}
}
}🛠️ Skill Evaluation Runs: Execute comprehensive testing of individual skills with custom test cases, performance benchmarks, and functional validation. Perfect for skill development, debugging, and performance optimization.
{
"testSuite": {
"name": "Sentiment Analysis Comprehensive Test",
"testCases": [
{
"id": "positive_sentiment_test",
"input": {
"text": "I absolutely love this product! It exceeded all my expectations and the customer service was fantastic.",
"analysisType": "detailed"
},
"expected": {
"sentiment": "positive",
"confidence": {"min": 0.8},
"emotions": ["joy", "satisfaction"],
"intensity": {"min": 0.7}
},
"weight": 1.0,
"critical": true
},
{
"id": "mixed_sentiment_test",
"input": {
"text": "The product is okay, but the shipping was delayed and that was frustrating."
},
"expected": {
"sentiment": "mixed",
"positiveAspects": ["product_quality"],
"negativeAspects": ["shipping_experience"]
},
"weight": 1.5
}
]
},
"performance": {
"benchmarks": {
"maxExecutionTime": 2.0,
"memoryUsage": "< 100MB",
"concurrency": 10
},
"loadTesting": {
"enabled": true,
"requests": 100,
"duration": "30s"
}
},
"validation": {
"crossValidation": true,
"folds": 5,
"testDatasetSize": 1000
}
}{
"skillEvalRun": {
"id": "uuid-skill-run-123",
"skillEvalJobId": "uuid-skill-job-456",
"status": "completed",
"overallScore": 94.5,
"results": {
"functionalAccuracy": 96.0,
"performanceBenchmarks": 92.0,
"reliability": 95.0
},
"testResults": [
{
"testCaseId": "positive_sentiment_test",
"status": "passed",
"actualOutput": {
"sentiment": "positive",
"confidence": 0.92,
"emotions": ["joy", "satisfaction"],
"intensity": 0.85
},
"score": 100.0,
"executionTime": 1.2
}
],
"performance": {
"avgExecutionTime": 1.4,
"maxExecutionTime": 2.1,
"memoryUsage": "67MB",
"throughput": "45 req/s"
},
"insights": {
"strengths": [
"Excellent accuracy on positive sentiment",
"Fast execution time",
"Low memory footprint"
],
"improvements": [
"Could improve mixed sentiment detection",
"Consider optimizing edge case handling"
]
}
}
}{
"testSuite": {
"name": "Knowledge Base Retrieval Test",
"testCases": [
{
"id": "password_reset_query",
"input": {
"query": "How do I reset my password?",
"maxResults": 5,
"categories": ["account_management"]
},
"expected": {
"topResult": {
"articleId": "kb_001_password_reset",
"relevanceScore": {"min": 0.9}
},
"totalResults": {"min": 3},
"categories": ["account_management", "security"]
}
},
{
"id": "billing_inquiry",
"input": {
"query": "Why was I charged twice?",
"includeRelated": true
},
"expected": {
"containsArticles": [
"billing_double_charge",
"payment_troubleshooting"
],
"avgRelevance": {"min": 0.7}
}
}
]
},
"metrics": [
"precision_at_k",
"recall_at_k",
"mean_reciprocal_rank",
"normalized_dcg"
]
}{
"testSuite": {
"name": "JSON Parser Validation Suite",
"testCases": [
{
"id": "order_parsing_test",
"input": {
"jsonData": "{\"order\": {\"id\": 12345, \"items\": [{\"name\": \"Widget\", \"price\": 19.99}], \"total\": 19.99}}",
"extractFields": [
"order.id",
"order.items[].name",
"order.total"
],
"validateSchema": true
},
"expected": {
"parsedFields": {
"order.id": 12345,
"order.items[0].name": "Widget",
"order.total": 19.99
},
"schemaValid": true,
"errors": []
}
},
{
"id": "malformed_json_test",
"input": {
"jsonData": "{\"incomplete\": \"data\",",
"errorHandling": "graceful"
},
"expected": {
"parsedFields": null,
"errors": [
{"type": "syntax_error", "severity": "error"}
],
"gracefulFailure": true
}
}
]
}
}Execute multiple evaluation jobs simultaneously with coordinated sampling, shared resources, and aggregated reporting. Ideal for comprehensive quality assessments and system-wide evaluations.
{
"suiteId": "uuid-suite-comprehensive-qa",
"execution": {
"mode": "parallel",
"maxConcurrency": 3,
"resourceSharing": true,
"failureHandling": "continue_on_failure"
},
"customization": {
"overrideSchedule": true,
"customSampling": {
"stratifiedAcrossJobs": true,
"ensureNoOverlap": true,
"totalSampleBudget": 500
}
},
"reporting": {
"aggregatedMetrics": true,
"crossJobComparisons": true,
"executiveSummary": true,
"detailLevel": "comprehensive"
},
"notifications": {
"progressUpdates": true,
"updateInterval": "5m",
"onCompletion": {
"email": ["qa-team@company.com"],
"webhook": "https://internal.company.com/evaluation-complete",
"generateDashboard": true
}
}
}{
"evaluation": {
"scope": "cross_system",
"systems": [
{
"name": "production_personas",
"personas": ["uuid-persona-1", "uuid-persona-2"],
"environment": "production"
},
{
"name": "staging_personas",
"personas": ["uuid-persona-1-v2", "uuid-persona-2-v2"],
"environment": "staging"
}
],
"comparison": {
"method": "side_by_side",
"sameDataset": true,
"blindEvaluation": true
}
},
"metrics": {
"focus": "deployment_readiness",
"dimensions": [
"performance_regression",
"quality_improvement",
"consistency_maintenance"
],
"decisionThresholds": {
"deploymentApproval": 8.0,
"requiresReview": 7.0,
"rollbackRecommendation": 6.0
}
}
}Compare your personas and skills against industry standards, competitor baselines, and best-practice benchmarks using standardized test suites and evaluation criteria.
{
"benchmark": {
"name": "customer_service_industry_standard",
"version": "2024.1",
"domains": [
"retail_support",
"technical_support",
"billing_inquiries",
"general_customer_service"
]
},
"evaluation": {
"testSuites": [
{
"name": "response_quality",
"scenarios": 50,
"dimensions": [
"accuracy", "helpfulness", "professionalism"
]
},
{
"name": "efficiency_benchmarks",
"scenarios": 30,
"metrics": [
"response_time", "resolution_rate", "escalation_rate"
]
}
],
"scoring": {
"industryPercentiles": true,
"competitorComparison": true,
"bestPracticeAlignment": true
}
},
"reporting": {
"benchmarkPosition": true,
"improvementAreas": true,
"competitiveAnalysis": true,
"certificationEligibility": true
}
}{
"benchmark": {
"name": "internal_excellence_standard",
"type": "custom",
"basedOn": {
"topPerformingPersonas": [
"uuid-persona-champion-1",
"uuid-persona-champion-2"
],
"performancePeriod": "best_90_days",
"exclusions": ["outlier_conversations"]
}
},
"criteria": {
"dimensions": [
{
"name": "brand_alignment",
"weight": 0.3,
"description": "Adherence to brand voice and values"
},
{
"name": "business_impact",
"weight": 0.4,
"description": "Measurable business outcomes"
},
{
"name": "innovation",
"weight": 0.3,
"description": "Creative problem-solving approach"
}
]
},
"validation": {
"expertReview": true,
"statisticalValidation": true,
"crossValidation": {
"folds": 10,
"stratified": true
}
}
}{
"benchmarkResults": {
"overall": {
"score": 8.3,
"industryPercentile": 78,
"rank": "Above Average",
"certification": "Silver Level"
},
"dimensionBreakdown": [
{
"dimension": "accuracy",
"score": 8.7,
"industryAverage": 7.9,
"percentile": 85,
"status": "excellent"
},
{
"dimension": "efficiency",
"score": 7.8,
"industryAverage": 8.1,
"percentile": 45,
"status": "needs_improvement"
}
],
"competitorComparison": [
{
"competitor": "Industry Leader A",
"theirScore": 9.1,
"gap": -0.8,
"keyDifferences": [
"Superior automation integration",
"More personalized responses"
]
}
],
"recommendations": [
{
"priority": "high",
"area": "efficiency_optimization",
"action": "Implement response template optimization",
"expectedImprovement": "+0.5 points",
"timeline": "4-6 weeks"
},
{
"priority": "medium",
"area": "personalization_enhancement",
"action": "Integrate customer history analysis",
"expectedImprovement": "+0.3 points",
"timeline": "8-10 weeks"
}
]
}
}Deep performance analysis of personas and skills including latency profiling, resource usage optimization, and scalability testing under various load conditions.
{
"loadTesting": {
"profiles": [
{
"name": "normal_load",
"concurrency": 10,
"duration": "5m",
"requestPattern": "constant"
},
{
"name": "peak_load",
"concurrency": 50,
"duration": "2m",
"requestPattern": "spike"
},
{
"name": "stress_test",
"concurrency": 100,
"duration": "1m",
"requestPattern": "sustained"
}
],
"scenarios": [
{
"type": "conversation_simulation",
"messageExchanges": 5,
"thinkTime": "1-3s",
"complexity": "mixed"
}
]
},
"monitoring": {
"metrics": [
"response_time_percentiles",
"error_rate",
"memory_usage",
"cpu_utilization",
"throughput"
],
"alerts": {
"responseTimeP95": "> 5s",
"errorRate": "> 1%",
"memoryUsage": "> 80%"
}
}
}{
"performanceResults": {
"summary": {
"overallGrade": "B+",
"bottlenecks": [
"skill_execution_latency",
"knowledge_base_lookup"
],
"strengths": [
"consistent_response_quality",
"good_error_handling"
]
},
"loadTestResults": [
{
"profile": "normal_load",
"metrics": {
"avgResponseTime": "1.2s",
"p95ResponseTime": "2.1s",
"errorRate": "0.1%",
"throughput": "8.3 req/s"
},
"status": "passed"
},
{
"profile": "peak_load",
"metrics": {
"avgResponseTime": "3.4s",
"p95ResponseTime": "7.8s",
"errorRate": "2.1%",
"throughput": "14.2 req/s"
},
"status": "degraded"
}
],
"optimizations": [
{
"area": "skill_caching",
"impact": "High",
"effort": "Medium",
"expectedImprovement": "30% response time reduction"
}
]
}
}