From 1ad8cb1edff3ea90fc780c8e5c0c81cba437940a Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 27 Aug 2026 10:24:15 +0000 Subject: [PATCH] chore(semantics): trim generated research artifacts Co-authored-by: Rocky --- .gitignore | 2 + .../EvidenceGateFineTune/final-report.json | 8422 --------- .../EvidenceGateFineTune/rounds.json | 14831 ---------------- .../PerIntentFineTune/final-report.json | 8422 --------- .../PerIntentFineTune/rounds.json | 14831 ---------------- .../IterativeResearch/final-report.json | 8411 --------- .../IterativeResearch/rounds.json | 14831 ---------------- ModelTraining/ClipboardSemantics/README.md | 7 +- 8 files changed, 6 insertions(+), 69751 deletions(-) delete mode 100644 ModelTraining/ClipboardSemantics/IterativeResearch/EvidenceGateFineTune/final-report.json delete mode 100644 ModelTraining/ClipboardSemantics/IterativeResearch/EvidenceGateFineTune/rounds.json delete mode 100644 ModelTraining/ClipboardSemantics/IterativeResearch/PerIntentFineTune/final-report.json delete mode 100644 ModelTraining/ClipboardSemantics/IterativeResearch/PerIntentFineTune/rounds.json delete mode 100644 ModelTraining/ClipboardSemantics/IterativeResearch/final-report.json delete mode 100644 ModelTraining/ClipboardSemantics/IterativeResearch/rounds.json diff --git a/.gitignore b/.gitignore index fac2570..275c9bd 100644 --- a/.gitignore +++ b/.gitignore @@ -70,6 +70,8 @@ ModelTraining/ClipboardSemantics/open-training-corpus.jsonl ModelTraining/ClipboardSemantics/combined-training-corpus.jsonl ModelTraining/ClipboardSemantics/Consensus/*.jsonl ModelTraining/ClipboardSemantics/VerifierCandidates/ +ModelTraining/ClipboardSemantics/IterativeResearch/**/rounds.json +ModelTraining/ClipboardSemantics/IterativeResearch/**/final-report.json # Python bytecode __pycache__/ diff --git a/ModelTraining/ClipboardSemantics/IterativeResearch/EvidenceGateFineTune/final-report.json b/ModelTraining/ClipboardSemantics/IterativeResearch/EvidenceGateFineTune/final-report.json deleted file mode 100644 index 7dfd2a9..0000000 --- a/ModelTraining/ClipboardSemantics/IterativeResearch/EvidenceGateFineTune/final-report.json +++ /dev/null @@ -1,8422 +0,0 @@ -{ - "baseTrainingRecords": 11500, - "deploymentDecision": "retain-current-deployed-models", - "elapsedSeconds": 188.088, - "finalEvaluations": { - "golden": { - "byLanguage": { - "en": { - "intentsMeeting95Precision": 1, - "macroF1": 0.654, - "macroPrecision": 0.985, - "macroRecall": 0.5396, - "minimumPrecision": 0.8649, - "perIntent": { - "blessing": { - "f1": 0.8235, - "falseNegative": 3, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 7, - "recall": 0.7, - "total": 126, - "trueNegative": 116, - "truePositive": 7, - "wilsonPrecisionLower95": 0.6457 - }, - "complaint": { - "f1": 0.6667, - "falseNegative": 3, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 3, - "recall": 0.5, - "total": 126, - "trueNegative": 120, - "truePositive": 3, - "wilsonPrecisionLower95": 0.4385 - }, - "confirmationDecision": { - "f1": 0.8889, - "falseNegative": 2, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 8, - "recall": 0.8, - "total": 126, - "trueNegative": 116, - "truePositive": 8, - "wilsonPrecisionLower95": 0.6756 - }, - "followUpReminder": { - "f1": 0.4706, - "falseNegative": 9, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 4, - "recall": 0.3077, - "total": 126, - "trueNegative": 113, - "truePositive": 4, - "wilsonPrecisionLower95": 0.5101 - }, - "invitation": { - "f1": 0.5455, - "falseNegative": 5, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 3, - "recall": 0.375, - "total": 126, - "trueNegative": 118, - "truePositive": 3, - "wilsonPrecisionLower95": 0.4385 - }, - "question": { - "f1": 0.9048, - "falseNegative": 4, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 19, - "recall": 0.8261, - "total": 126, - "trueNegative": 103, - "truePositive": 19, - "wilsonPrecisionLower95": 0.8318 - }, - "replyableMessage": { - "f1": 0.8889, - "falseNegative": 6, - "falsePositive": 10, - "precision": 0.8649, - "predictedPositive": 74, - "recall": 0.9143, - "total": 126, - "trueNegative": 46, - "truePositive": 64, - "wilsonPrecisionLower95": 0.7688 - }, - "scheduleNegotiation": { - "f1": 0.4615, - "falseNegative": 7, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 3, - "recall": 0.3, - "total": 126, - "trueNegative": 116, - "truePositive": 3, - "wilsonPrecisionLower95": 0.4385 - }, - "task": { - "f1": 0.2353, - "falseNegative": 13, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 2, - "recall": 0.1333, - "total": 126, - "trueNegative": 111, - "truePositive": 2, - "wilsonPrecisionLower95": 0.3424 - } - }, - "recordCount": 126 - }, - "zh-Hans": { - "intentsMeeting95Precision": 1, - "macroF1": 0.4022, - "macroPrecision": 0.7683, - "macroRecall": 0.3312, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.3333, - "falseNegative": 8, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 2, - "recall": 0.2, - "total": 126, - "trueNegative": 116, - "truePositive": 2, - "wilsonPrecisionLower95": 0.3424 - }, - "complaint": { - "f1": 0.2857, - "falseNegative": 5, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 1, - "recall": 0.1667, - "total": 126, - "trueNegative": 120, - "truePositive": 1, - "wilsonPrecisionLower95": 0.2065 - }, - "confirmationDecision": { - "f1": 0.5714, - "falseNegative": 6, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 4, - "recall": 0.4, - "total": 126, - "trueNegative": 116, - "truePositive": 4, - "wilsonPrecisionLower95": 0.5101 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 13, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 126, - "trueNegative": 113, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 8, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 126, - "trueNegative": 118, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 23, - "recall": 1.0, - "total": 126, - "trueNegative": 103, - "truePositive": 23, - "wilsonPrecisionLower95": 0.8569 - }, - "replyableMessage": { - "f1": 0.9143, - "falseNegative": 6, - "falsePositive": 6, - "precision": 0.9143, - "predictedPositive": 70, - "recall": 0.9143, - "total": 126, - "trueNegative": 50, - "truePositive": 64, - "wilsonPrecisionLower95": 0.8253 - }, - "scheduleNegotiation": { - "f1": 0.1818, - "falseNegative": 9, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 1, - "recall": 0.1, - "total": 126, - "trueNegative": 116, - "truePositive": 1, - "wilsonPrecisionLower95": 0.2065 - }, - "task": { - "f1": 0.3333, - "falseNegative": 12, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 3, - "recall": 0.2, - "total": 126, - "trueNegative": 111, - "truePositive": 3, - "wilsonPrecisionLower95": 0.4385 - } - }, - "recordCount": 126 - } - }, - "bySource": { - "acknowledgment": { - "intentsMeeting95Precision": 0, - "macroF1": 0.0, - "macroPrecision": 0.0, - "macroRecall": 0.0, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 8 - }, - "blessing_boundary": { - "intentsMeeting95Precision": 0, - "macroF1": 0.1482, - "macroPrecision": 0.1667, - "macroRecall": 0.1667, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.6667, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 1, - "recall": 0.5, - "total": 20, - "trueNegative": 18, - "truePositive": 1, - "wilsonPrecisionLower95": 0.2065 - }, - "replyableMessage": { - "f1": 0.6667, - "falseNegative": 0, - "falsePositive": 4, - "precision": 0.5, - "predictedPositive": 8, - "recall": 1.0, - "total": 20, - "trueNegative": 12, - "truePositive": 4, - "wilsonPrecisionLower95": 0.2152 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 2, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 18, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 20 - }, - "blessing_golden": { - "intentsMeeting95Precision": 1, - "macroF1": 0.1772, - "macroPrecision": 0.2222, - "macroRecall": 0.1556, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.6207, - "falseNegative": 11, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 9, - "recall": 0.45, - "total": 20, - "trueNegative": 0, - "truePositive": 9, - "wilsonPrecisionLower95": 0.7009 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.9744, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 19, - "recall": 0.95, - "total": 20, - "trueNegative": 0, - "truePositive": 19, - "wilsonPrecisionLower95": 0.8318 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 20 - }, - "complaint_request": { - "intentsMeeting95Precision": 0, - "macroF1": 0.1962, - "macroPrecision": 0.2222, - "macroRecall": 0.1759, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 6, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.8571, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 3, - "recall": 0.75, - "total": 6, - "trueNegative": 2, - "truePositive": 3, - "wilsonPrecisionLower95": 0.4385 - }, - "replyableMessage": { - "f1": 0.9091, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 5, - "recall": 0.8333, - "total": 6, - "trueNegative": 0, - "truePositive": 5, - "wilsonPrecisionLower95": 0.5655 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 6 - }, - "complaint_statement": { - "intentsMeeting95Precision": 0, - "macroF1": 0.2, - "macroPrecision": 0.2222, - "macroRecall": 0.1852, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.8, - "falseNegative": 2, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 4, - "recall": 0.6667, - "total": 6, - "trueNegative": 0, - "truePositive": 4, - "wilsonPrecisionLower95": 0.5101 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 6, - "recall": 1.0, - "total": 6, - "trueNegative": 0, - "truePositive": 6, - "wilsonPrecisionLower95": 0.6097 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 6 - }, - "confirmation_decision_boundary": { - "intentsMeeting95Precision": 0, - "macroF1": 0.0, - "macroPrecision": 0.0, - "macroRecall": 0.0, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 3, - "precision": 0.0, - "predictedPositive": 3, - "recall": 0.0, - "total": 20, - "trueNegative": 17, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 20 - }, - "confirmation_decision_golden": { - "intentsMeeting95Precision": 2, - "macroF1": 0.1944, - "macroPrecision": 0.2222, - "macroRecall": 0.1778, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.75, - "falseNegative": 8, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 12, - "recall": 0.6, - "total": 20, - "trueNegative": 0, - "truePositive": 12, - "wilsonPrecisionLower95": 0.7575 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 20, - "recall": 1.0, - "total": 20, - "trueNegative": 0, - "truePositive": 20, - "wilsonPrecisionLower95": 0.8389 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 20 - }, - "conversational_message": { - "intentsMeeting95Precision": 1, - "macroF1": 0.1111, - "macroPrecision": 0.1111, - "macroRecall": 0.1111, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 10, - "trueNegative": 10, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 10, - "trueNegative": 10, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 10, - "trueNegative": 10, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 10, - "trueNegative": 10, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 10, - "trueNegative": 10, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 10, - "trueNegative": 10, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 10, - "recall": 1.0, - "total": 10, - "trueNegative": 0, - "truePositive": 10, - "wilsonPrecisionLower95": 0.7225 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 10, - "trueNegative": 10, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 10, - "trueNegative": 10, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 10 - }, - "event_statement": { - "intentsMeeting95Precision": 0, - "macroF1": 0.0, - "macroPrecision": 0.0, - "macroRecall": 0.0, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 4 - }, - "follow_up_reminder_boundary": { - "intentsMeeting95Precision": 0, - "macroF1": 0.0, - "macroPrecision": 0.0, - "macroRecall": 0.0, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 3, - "precision": 0.0, - "predictedPositive": 3, - "recall": 0.0, - "total": 20, - "trueNegative": 17, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 20 - }, - "follow_up_reminder_golden": { - "intentsMeeting95Precision": 0, - "macroF1": 0.1179, - "macroPrecision": 0.3333, - "macroRecall": 0.0802, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.2609, - "falseNegative": 17, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 3, - "recall": 0.15, - "total": 20, - "trueNegative": 0, - "truePositive": 3, - "wilsonPrecisionLower95": 0.4385 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.6667, - "falseNegative": 7, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 7, - "recall": 0.5, - "total": 20, - "trueNegative": 6, - "truePositive": 7, - "wilsonPrecisionLower95": 0.6457 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.1333, - "falseNegative": 13, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 1, - "recall": 0.0714, - "total": 20, - "trueNegative": 6, - "truePositive": 1, - "wilsonPrecisionLower95": 0.2065 - } - }, - "recordCount": 20 - }, - "information_question": { - "intentsMeeting95Precision": 0, - "macroF1": 0.1693, - "macroPrecision": 0.2222, - "macroRecall": 0.1389, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.6667, - "falseNegative": 2, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 2, - "recall": 0.5, - "total": 4, - "trueNegative": 0, - "truePositive": 2, - "wilsonPrecisionLower95": 0.3424 - }, - "replyableMessage": { - "f1": 0.8571, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 3, - "recall": 0.75, - "total": 4, - "trueNegative": 0, - "truePositive": 3, - "wilsonPrecisionLower95": 0.4385 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 4 - }, - "invitation_question": { - "intentsMeeting95Precision": 0, - "macroF1": 0.2469, - "macroPrecision": 0.3333, - "macroRecall": 0.2361, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.2222, - "falseNegative": 7, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 1, - "recall": 0.125, - "total": 8, - "trueNegative": 0, - "truePositive": 1, - "wilsonPrecisionLower95": 0.2065 - }, - "question": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 8, - "recall": 1.0, - "total": 8, - "trueNegative": 0, - "truePositive": 8, - "wilsonPrecisionLower95": 0.6756 - }, - "replyableMessage": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 8, - "recall": 1.0, - "total": 8, - "trueNegative": 0, - "truePositive": 8, - "wilsonPrecisionLower95": 0.6756 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 8 - }, - "negative_news": { - "intentsMeeting95Precision": 0, - "macroF1": 0.0, - "macroPrecision": 0.0, - "macroRecall": 0.0, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 1, - "precision": 0.0, - "predictedPositive": 1, - "recall": 0.0, - "total": 6, - "trueNegative": 5, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 6 - }, - "neutral_fact": { - "intentsMeeting95Precision": 0, - "macroF1": 0.0, - "macroPrecision": 0.0, - "macroRecall": 0.0, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 10, - "trueNegative": 10, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 10, - "trueNegative": 10, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 10, - "trueNegative": 10, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 10, - "trueNegative": 10, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 10, - "trueNegative": 10, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 10, - "trueNegative": 10, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 2, - "precision": 0.0, - "predictedPositive": 2, - "recall": 0.0, - "total": 10, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 10, - "trueNegative": 10, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 10, - "trueNegative": 10, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 10 - }, - "positive_feedback": { - "intentsMeeting95Precision": 0, - "macroF1": 0.1111, - "macroPrecision": 0.1111, - "macroRecall": 0.1111, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 6, - "recall": 1.0, - "total": 6, - "trueNegative": 0, - "truePositive": 6, - "wilsonPrecisionLower95": 0.6097 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 6 - }, - "quoted_question": { - "intentsMeeting95Precision": 0, - "macroF1": 0.0, - "macroPrecision": 0.0, - "macroRecall": 0.0, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 4 - }, - "schedule_negotiation_boundary": { - "intentsMeeting95Precision": 0, - "macroF1": 0.2601, - "macroPrecision": 0.321, - "macroRecall": 0.25, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.4, - "falseNegative": 6, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 2, - "recall": 0.25, - "total": 20, - "trueNegative": 12, - "truePositive": 2, - "wilsonPrecisionLower95": 0.3424 - }, - "question": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 4, - "recall": 1.0, - "total": 20, - "trueNegative": 16, - "truePositive": 4, - "wilsonPrecisionLower95": 0.5101 - }, - "replyableMessage": { - "f1": 0.9412, - "falseNegative": 0, - "falsePositive": 1, - "precision": 0.8889, - "predictedPositive": 9, - "recall": 1.0, - "total": 20, - "trueNegative": 11, - "truePositive": 8, - "wilsonPrecisionLower95": 0.565 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 20 - }, - "schedule_negotiation_golden": { - "intentsMeeting95Precision": 2, - "macroF1": 0.2564, - "macroPrecision": 0.3333, - "macroRecall": 0.2389, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 18, - "recall": 1.0, - "total": 20, - "trueNegative": 2, - "truePositive": 18, - "wilsonPrecisionLower95": 0.8241 - }, - "replyableMessage": { - "f1": 0.9744, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 19, - "recall": 0.95, - "total": 20, - "trueNegative": 0, - "truePositive": 19, - "wilsonPrecisionLower95": 0.8318 - }, - "scheduleNegotiation": { - "f1": 0.3333, - "falseNegative": 16, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 4, - "recall": 0.2, - "total": 20, - "trueNegative": 0, - "truePositive": 4, - "wilsonPrecisionLower95": 0.5101 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 20 - }, - "self_plan": { - "intentsMeeting95Precision": 0, - "macroF1": 0.0741, - "macroPrecision": 0.1111, - "macroRecall": 0.0556, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.6667, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 1, - "recall": 0.5, - "total": 6, - "trueNegative": 4, - "truePositive": 1, - "wilsonPrecisionLower95": 0.2065 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 2, - "precision": 0.0, - "predictedPositive": 2, - "recall": 0.0, - "total": 6, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 6 - }, - "task_question": { - "intentsMeeting95Precision": 0, - "macroF1": 0.254, - "macroPrecision": 0.3333, - "macroRecall": 0.2407, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 6, - "recall": 1.0, - "total": 6, - "trueNegative": 0, - "truePositive": 6, - "wilsonPrecisionLower95": 0.6097 - }, - "replyableMessage": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 6, - "recall": 1.0, - "total": 6, - "trueNegative": 0, - "truePositive": 6, - "wilsonPrecisionLower95": 0.6097 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.2857, - "falseNegative": 5, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 1, - "recall": 0.1667, - "total": 6, - "trueNegative": 0, - "truePositive": 1, - "wilsonPrecisionLower95": 0.2065 - } - }, - "recordCount": 6 - }, - "task_statement": { - "intentsMeeting95Precision": 0, - "macroF1": 0.1643, - "macroPrecision": 0.2222, - "macroRecall": 0.1389, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 4, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.9333, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 7, - "recall": 0.875, - "total": 8, - "trueNegative": 0, - "truePositive": 7, - "wilsonPrecisionLower95": 0.6457 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.5455, - "falseNegative": 5, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 3, - "recall": 0.375, - "total": 8, - "trueNegative": 0, - "truePositive": 3, - "wilsonPrecisionLower95": 0.4385 - } - }, - "recordCount": 8 - } - }, - "intentsMeeting95Precision": 2, - "macroF1": 0.5476, - "macroPrecision": 0.9877, - "macroRecall": 0.4354, - "minimumPrecision": 0.8889, - "perIntent": { - "blessing": { - "f1": 0.6207, - "falseNegative": 11, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 9, - "recall": 0.45, - "total": 252, - "trueNegative": 232, - "truePositive": 9, - "wilsonPrecisionLower95": 0.7009 - }, - "complaint": { - "f1": 0.5, - "falseNegative": 8, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 4, - "recall": 0.3333, - "total": 252, - "trueNegative": 240, - "truePositive": 4, - "wilsonPrecisionLower95": 0.5101 - }, - "confirmationDecision": { - "f1": 0.75, - "falseNegative": 8, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 12, - "recall": 0.6, - "total": 252, - "trueNegative": 232, - "truePositive": 12, - "wilsonPrecisionLower95": 0.7575 - }, - "followUpReminder": { - "f1": 0.2667, - "falseNegative": 22, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 4, - "recall": 0.1538, - "total": 252, - "trueNegative": 226, - "truePositive": 4, - "wilsonPrecisionLower95": 0.5101 - }, - "invitation": { - "f1": 0.3158, - "falseNegative": 13, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 3, - "recall": 0.1875, - "total": 252, - "trueNegative": 236, - "truePositive": 3, - "wilsonPrecisionLower95": 0.4385 - }, - "question": { - "f1": 0.9545, - "falseNegative": 4, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 42, - "recall": 0.913, - "total": 252, - "trueNegative": 206, - "truePositive": 42, - "wilsonPrecisionLower95": 0.9162 - }, - "replyableMessage": { - "f1": 0.9014, - "falseNegative": 12, - "falsePositive": 16, - "precision": 0.8889, - "predictedPositive": 144, - "recall": 0.9143, - "total": 252, - "trueNegative": 96, - "truePositive": 128, - "wilsonPrecisionLower95": 0.8271 - }, - "scheduleNegotiation": { - "f1": 0.3333, - "falseNegative": 16, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 4, - "recall": 0.2, - "total": 252, - "trueNegative": 232, - "truePositive": 4, - "wilsonPrecisionLower95": 0.5101 - }, - "task": { - "f1": 0.2857, - "falseNegative": 25, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 5, - "recall": 0.1667, - "total": 252, - "trueNegative": 222, - "truePositive": 5, - "wilsonPrecisionLower95": 0.5655 - } - }, - "recordCount": 252 - }, - "randomHoldout": { - "byLanguage": { - "en": { - "intentsMeeting95Precision": 5, - "macroF1": 0.4512, - "macroPrecision": 0.7745, - "macroRecall": 0.3651, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 340, - "trueNegative": 340, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0952, - "falseNegative": 19, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 1, - "recall": 0.05, - "total": 340, - "trueNegative": 320, - "truePositive": 1, - "wilsonPrecisionLower95": 0.2065 - }, - "confirmationDecision": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 20, - "recall": 1.0, - "total": 340, - "trueNegative": 320, - "truePositive": 20, - "wilsonPrecisionLower95": 0.8389 - }, - "followUpReminder": { - "f1": 0.5, - "falseNegative": 40, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 20, - "recall": 0.3333, - "total": 340, - "trueNegative": 280, - "truePositive": 20, - "wilsonPrecisionLower95": 0.8389 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 20, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 340, - "trueNegative": 320, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.6087, - "falseNegative": 45, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 35, - "recall": 0.4375, - "total": 340, - "trueNegative": 260, - "truePositive": 35, - "wilsonPrecisionLower95": 0.9011 - }, - "replyableMessage": { - "f1": 0.8859, - "falseNegative": 37, - "falsePositive": 5, - "precision": 0.9702, - "predictedPositive": 168, - "recall": 0.815, - "total": 340, - "trueNegative": 135, - "truePositive": 163, - "wilsonPrecisionLower95": 0.9322 - }, - "scheduleNegotiation": { - "f1": 0.5714, - "falseNegative": 12, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 8, - "recall": 0.4, - "total": 340, - "trueNegative": 320, - "truePositive": 8, - "wilsonPrecisionLower95": 0.6756 - }, - "task": { - "f1": 0.4, - "falseNegative": 45, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 15, - "recall": 0.25, - "total": 340, - "trueNegative": 280, - "truePositive": 15, - "wilsonPrecisionLower95": 0.7961 - } - }, - "recordCount": 340 - }, - "zh-Hans": { - "intentsMeeting95Precision": 2, - "macroF1": 0.3012, - "macroPrecision": 0.6608, - "macroRecall": 0.2406, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 340, - "trueNegative": 340, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0952, - "falseNegative": 19, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 1, - "recall": 0.05, - "total": 340, - "trueNegative": 320, - "truePositive": 1, - "wilsonPrecisionLower95": 0.2065 - }, - "confirmationDecision": { - "f1": 0.5185, - "falseNegative": 13, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 7, - "recall": 0.35, - "total": 340, - "trueNegative": 320, - "truePositive": 7, - "wilsonPrecisionLower95": 0.6457 - }, - "followUpReminder": { - "f1": 0.0645, - "falseNegative": 58, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 2, - "recall": 0.0333, - "total": 340, - "trueNegative": 280, - "truePositive": 2, - "wilsonPrecisionLower95": 0.3424 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 20, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 340, - "trueNegative": 320, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.7786, - "falseNegative": 29, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 51, - "recall": 0.6375, - "total": 340, - "trueNegative": 260, - "truePositive": 51, - "wilsonPrecisionLower95": 0.93 - }, - "replyableMessage": { - "f1": 0.9203, - "falseNegative": 21, - "falsePositive": 10, - "precision": 0.9471, - "predictedPositive": 189, - "recall": 0.895, - "total": 340, - "trueNegative": 130, - "truePositive": 179, - "wilsonPrecisionLower95": 0.9054 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 20, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 340, - "trueNegative": 320, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.3333, - "falseNegative": 48, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 12, - "recall": 0.2, - "total": 340, - "trueNegative": 280, - "truePositive": 12, - "wilsonPrecisionLower95": 0.7575 - } - }, - "recordCount": 340 - } - }, - "bySource": { - "acknowledgment_boundary": { - "intentsMeeting95Precision": 0, - "macroF1": 0.0, - "macroPrecision": 0.0, - "macroRecall": 0.0, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 40 - }, - "complaint_support": { - "intentsMeeting95Precision": 1, - "macroF1": 0.1518, - "macroPrecision": 0.3333, - "macroRecall": 0.1222, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0952, - "falseNegative": 38, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 2, - "recall": 0.05, - "total": 40, - "trueNegative": 0, - "truePositive": 2, - "wilsonPrecisionLower95": 0.3424 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.3673, - "falseNegative": 31, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 9, - "recall": 0.225, - "total": 40, - "trueNegative": 0, - "truePositive": 9, - "wilsonPrecisionLower95": 0.7009 - }, - "replyableMessage": { - "f1": 0.9041, - "falseNegative": 7, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 33, - "recall": 0.825, - "total": 40, - "trueNegative": 0, - "truePositive": 33, - "wilsonPrecisionLower95": 0.8957 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 40 - }, - "confirmation_decision": { - "intentsMeeting95Precision": 2, - "macroF1": 0.19, - "macroPrecision": 0.2222, - "macroRecall": 0.1667, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.806, - "falseNegative": 13, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 27, - "recall": 0.675, - "total": 40, - "trueNegative": 0, - "truePositive": 27, - "wilsonPrecisionLower95": 0.8754 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.9041, - "falseNegative": 7, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 33, - "recall": 0.825, - "total": 40, - "trueNegative": 0, - "truePositive": 33, - "wilsonPrecisionLower95": 0.8957 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 40 - }, - "conversation": { - "intentsMeeting95Precision": 1, - "macroF1": 0.1053, - "macroPrecision": 0.1111, - "macroRecall": 0.1, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.9474, - "falseNegative": 4, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 36, - "recall": 0.9, - "total": 40, - "trueNegative": 0, - "truePositive": 36, - "wilsonPrecisionLower95": 0.9036 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 40 - }, - "event_boundary": { - "intentsMeeting95Precision": 0, - "macroF1": 0.0, - "macroPrecision": 0.0, - "macroRecall": 0.0, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 2, - "precision": 0.0, - "predictedPositive": 2, - "recall": 0.0, - "total": 40, - "trueNegative": 38, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 40 - }, - "fixed_invitation": { - "intentsMeeting95Precision": 2, - "macroF1": 0.2134, - "macroPrecision": 0.2222, - "macroRecall": 0.2056, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 40, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.9333, - "falseNegative": 5, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 35, - "recall": 0.875, - "total": 40, - "trueNegative": 0, - "truePositive": 35, - "wilsonPrecisionLower95": 0.9011 - }, - "replyableMessage": { - "f1": 0.9873, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 39, - "recall": 0.975, - "total": 40, - "trueNegative": 0, - "truePositive": 39, - "wilsonPrecisionLower95": 0.9103 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 40 - }, - "follow_up_instruction": { - "intentsMeeting95Precision": 1, - "macroF1": 0.0934, - "macroPrecision": 0.1111, - "macroRecall": 0.0806, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 40, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.8406, - "falseNegative": 11, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 29, - "recall": 0.725, - "total": 40, - "trueNegative": 0, - "truePositive": 29, - "wilsonPrecisionLower95": 0.883 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 40, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 40 - }, - "follow_up_personal_reminder": { - "intentsMeeting95Precision": 1, - "macroF1": 0.0789, - "macroPrecision": 0.1111, - "macroRecall": 0.0611, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.7097, - "falseNegative": 18, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 22, - "recall": 0.55, - "total": 40, - "trueNegative": 0, - "truePositive": 22, - "wilsonPrecisionLower95": 0.8513 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 40 - }, - "information_question": { - "intentsMeeting95Precision": 2, - "macroF1": 0.1421, - "macroPrecision": 0.2222, - "macroRecall": 0.1083, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.4906, - "falseNegative": 27, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 13, - "recall": 0.325, - "total": 40, - "trueNegative": 0, - "truePositive": 13, - "wilsonPrecisionLower95": 0.7719 - }, - "replyableMessage": { - "f1": 0.7879, - "falseNegative": 14, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 26, - "recall": 0.65, - "total": 40, - "trueNegative": 0, - "truePositive": 26, - "wilsonPrecisionLower95": 0.8713 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 40 - }, - "negative_news": { - "intentsMeeting95Precision": 0, - "macroF1": 0.0, - "macroPrecision": 0.0, - "macroRecall": 0.0, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 3, - "precision": 0.0, - "predictedPositive": 3, - "recall": 0.0, - "total": 40, - "trueNegative": 37, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 40 - }, - "neutral_fact": { - "intentsMeeting95Precision": 0, - "macroF1": 0.0, - "macroPrecision": 0.0, - "macroRecall": 0.0, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 5, - "precision": 0.0, - "predictedPositive": 5, - "recall": 0.0, - "total": 40, - "trueNegative": 35, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 40 - }, - "positive_feedback": { - "intentsMeeting95Precision": 1, - "macroF1": 0.1068, - "macroPrecision": 0.1111, - "macroRecall": 0.1028, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.961, - "falseNegative": 3, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 37, - "recall": 0.925, - "total": 40, - "trueNegative": 0, - "truePositive": 37, - "wilsonPrecisionLower95": 0.9059 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 40 - }, - "quoted_question_boundary": { - "intentsMeeting95Precision": 0, - "macroF1": 0.0, - "macroPrecision": 0.0, - "macroRecall": 0.0, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 1, - "precision": 0.0, - "predictedPositive": 1, - "recall": 0.0, - "total": 40, - "trueNegative": 39, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 40 - }, - "schedule_negotiation": { - "intentsMeeting95Precision": 2, - "macroF1": 0.2415, - "macroPrecision": 0.3333, - "macroRecall": 0.2139, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.8406, - "falseNegative": 11, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 29, - "recall": 0.725, - "total": 40, - "trueNegative": 0, - "truePositive": 29, - "wilsonPrecisionLower95": 0.883 - }, - "replyableMessage": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 40, - "recall": 1.0, - "total": 40, - "trueNegative": 0, - "truePositive": 40, - "wilsonPrecisionLower95": 0.9124 - }, - "scheduleNegotiation": { - "f1": 0.3333, - "falseNegative": 32, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 8, - "recall": 0.2, - "total": 40, - "trueNegative": 0, - "truePositive": 8, - "wilsonPrecisionLower95": 0.6756 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 40 - }, - "task_next_action": { - "intentsMeeting95Precision": 2, - "macroF1": 0.1517, - "macroPrecision": 0.2222, - "macroRecall": 0.125, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 40, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.9041, - "falseNegative": 7, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 33, - "recall": 0.825, - "total": 40, - "trueNegative": 0, - "truePositive": 33, - "wilsonPrecisionLower95": 0.8957 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.4615, - "falseNegative": 28, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 12, - "recall": 0.3, - "total": 40, - "trueNegative": 0, - "truePositive": 12, - "wilsonPrecisionLower95": 0.7575 - } - }, - "recordCount": 40 - }, - "task_request": { - "intentsMeeting95Precision": 2, - "macroF1": 0.1659, - "macroPrecision": 0.2222, - "macroRecall": 0.1417, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.9474, - "falseNegative": 4, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 36, - "recall": 0.9, - "total": 40, - "trueNegative": 0, - "truePositive": 36, - "wilsonPrecisionLower95": 0.9036 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.5455, - "falseNegative": 25, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 15, - "recall": 0.375, - "total": 40, - "trueNegative": 0, - "truePositive": 15, - "wilsonPrecisionLower95": 0.7961 - } - }, - "recordCount": 40 - }, - "vague_future_boundary": { - "intentsMeeting95Precision": 0, - "macroF1": 0.0, - "macroPrecision": 0.0, - "macroRecall": 0.0, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 4, - "precision": 0.0, - "predictedPositive": 4, - "recall": 0.0, - "total": 40, - "trueNegative": 36, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 40 - } - }, - "intentsMeeting95Precision": 5, - "macroF1": 0.3905, - "macroPrecision": 0.7731, - "macroRecall": 0.3029, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 680, - "trueNegative": 680, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0952, - "falseNegative": 38, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 2, - "recall": 0.05, - "total": 680, - "trueNegative": 640, - "truePositive": 2, - "wilsonPrecisionLower95": 0.3424 - }, - "confirmationDecision": { - "f1": 0.806, - "falseNegative": 13, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 27, - "recall": 0.675, - "total": 680, - "trueNegative": 640, - "truePositive": 27, - "wilsonPrecisionLower95": 0.8754 - }, - "followUpReminder": { - "f1": 0.3099, - "falseNegative": 98, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 22, - "recall": 0.1833, - "total": 680, - "trueNegative": 560, - "truePositive": 22, - "wilsonPrecisionLower95": 0.8513 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 40, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 680, - "trueNegative": 640, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.6992, - "falseNegative": 74, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 86, - "recall": 0.5375, - "total": 680, - "trueNegative": 520, - "truePositive": 86, - "wilsonPrecisionLower95": 0.9572 - }, - "replyableMessage": { - "f1": 0.9036, - "falseNegative": 58, - "falsePositive": 15, - "precision": 0.958, - "predictedPositive": 357, - "recall": 0.855, - "total": 680, - "trueNegative": 265, - "truePositive": 342, - "wilsonPrecisionLower95": 0.9318 - }, - "scheduleNegotiation": { - "f1": 0.3333, - "falseNegative": 32, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 8, - "recall": 0.2, - "total": 680, - "trueNegative": 640, - "truePositive": 8, - "wilsonPrecisionLower95": 0.6756 - }, - "task": { - "f1": 0.3673, - "falseNegative": 93, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 27, - "recall": 0.225, - "total": 680, - "trueNegative": 560, - "truePositive": 27, - "wilsonPrecisionLower95": 0.8754 - } - }, - "recordCount": 680 - }, - "researchOnlyComprehensive": { - "byLanguage": { - "en": { - "intentsMeeting95Precision": 2, - "macroF1": 0.299, - "macroPrecision": 0.5506, - "macroRecall": 0.2272, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.4667, - "falseNegative": 48, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 21, - "recall": 0.3043, - "total": 13176, - "trueNegative": 13107, - "truePositive": 21, - "wilsonPrecisionLower95": 0.8454 - }, - "complaint": { - "f1": 0.009, - "falseNegative": 432, - "falsePositive": 6, - "precision": 0.25, - "predictedPositive": 8, - "recall": 0.0046, - "total": 13176, - "trueNegative": 12736, - "truePositive": 2, - "wilsonPrecisionLower95": 0.0715 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 12, - "falsePositive": 11, - "precision": 0.0, - "predictedPositive": 11, - "recall": 0.0, - "total": 13176, - "trueNegative": 13153, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "followUpReminder": { - "f1": 0.297, - "falseNegative": 95, - "falsePositive": 47, - "precision": 0.3896, - "predictedPositive": 77, - "recall": 0.24, - "total": 13176, - "trueNegative": 13004, - "truePositive": 30, - "wilsonPrecisionLower95": 0.2884 - }, - "invitation": { - "f1": 0.3014, - "falseNegative": 50, - "falsePositive": 1, - "precision": 0.9167, - "predictedPositive": 12, - "recall": 0.1803, - "total": 13176, - "trueNegative": 13114, - "truePositive": 11, - "wilsonPrecisionLower95": 0.6461 - }, - "question": { - "f1": 0.4565, - "falseNegative": 1862, - "falsePositive": 29, - "precision": 0.9648, - "predictedPositive": 823, - "recall": 0.2989, - "total": 13176, - "trueNegative": 10491, - "truePositive": 794, - "wilsonPrecisionLower95": 0.9499 - }, - "replyableMessage": { - "f1": 0.7642, - "falseNegative": 2967, - "falsePositive": 1917, - "precision": 0.805, - "predictedPositive": 9831, - "recall": 0.7273, - "total": 13176, - "trueNegative": 378, - "truePositive": 7914, - "wilsonPrecisionLower95": 0.7971 - }, - "scheduleNegotiation": { - "f1": 0.14, - "falseNegative": 60, - "falsePositive": 26, - "precision": 0.2121, - "predictedPositive": 33, - "recall": 0.1045, - "total": 13176, - "trueNegative": 13083, - "truePositive": 7, - "wilsonPrecisionLower95": 0.1068 - }, - "task": { - "f1": 0.2563, - "falseNegative": 912, - "falsePositive": 289, - "precision": 0.4173, - "predictedPositive": 496, - "recall": 0.185, - "total": 13176, - "trueNegative": 11768, - "truePositive": 207, - "wilsonPrecisionLower95": 0.3747 - } - }, - "recordCount": 13176 - }, - "zh-Hans": { - "intentsMeeting95Precision": 0, - "macroF1": 0.1436, - "macroPrecision": 0.5377, - "macroRecall": 0.1039, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.2105, - "falseNegative": 15, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 2, - "recall": 0.1176, - "total": 28019, - "trueNegative": 28002, - "truePositive": 2, - "wilsonPrecisionLower95": 0.3424 - }, - "complaint": { - "f1": 0.1895, - "falseNegative": 241, - "falsePositive": 7, - "precision": 0.8056, - "predictedPositive": 36, - "recall": 0.1074, - "total": 28019, - "trueNegative": 27742, - "truePositive": 29, - "wilsonPrecisionLower95": 0.6497 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 57, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 28019, - "trueNegative": 27962, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0075, - "falseNegative": 264, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 1, - "recall": 0.0038, - "total": 28019, - "trueNegative": 27754, - "truePositive": 1, - "wilsonPrecisionLower95": 0.2065 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 73, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 28019, - "trueNegative": 27946, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.1195, - "falseNegative": 6725, - "falsePositive": 36, - "precision": 0.9273, - "predictedPositive": 495, - "recall": 0.0639, - "total": 28019, - "trueNegative": 20799, - "truePositive": 459, - "wilsonPrecisionLower95": 0.901 - }, - "replyableMessage": { - "f1": 0.7622, - "falseNegative": 9480, - "falsePositive": 1096, - "precision": 0.9393, - "predictedPositive": 18042, - "recall": 0.6413, - "total": 28019, - "trueNegative": 497, - "truePositive": 16946, - "wilsonPrecisionLower95": 0.9357 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 4, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 28019, - "trueNegative": 28015, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.003, - "falseNegative": 658, - "falsePositive": 5, - "precision": 0.1667, - "predictedPositive": 6, - "recall": 0.0015, - "total": 28019, - "trueNegative": 27355, - "truePositive": 1, - "wilsonPrecisionLower95": 0.0301 - } - }, - "recordCount": 28019 - } - }, - "bySource": { - "AmazonScience/MASSIVE en-US test": { - "intentsMeeting95Precision": 2, - "macroF1": 0.1742, - "macroPrecision": 0.2929, - "macroRecall": 0.1343, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 2967, - "trueNegative": 2967, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 1, - "precision": 0.0, - "predictedPositive": 1, - "recall": 0.0, - "total": 2967, - "trueNegative": 2966, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 1, - "precision": 0.0, - "predictedPositive": 1, - "recall": 0.0, - "total": 2967, - "trueNegative": 2966, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "followUpReminder": { - "f1": 0.321, - "falseNegative": 71, - "falsePositive": 39, - "precision": 0.4, - "predictedPositive": 65, - "recall": 0.268, - "total": 2967, - "trueNegative": 2831, - "truePositive": 26, - "wilsonPrecisionLower95": 0.2897 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 2967, - "trueNegative": 2967, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.3271, - "falseNegative": 1300, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 316, - "recall": 0.1955, - "total": 2967, - "trueNegative": 1351, - "truePositive": 316, - "wilsonPrecisionLower95": 0.988 - }, - "replyableMessage": { - "f1": 0.7951, - "falseNegative": 1009, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 1958, - "recall": 0.6599, - "total": 2967, - "trueNegative": 0, - "truePositive": 1958, - "wilsonPrecisionLower95": 0.998 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 5, - "precision": 0.0, - "predictedPositive": 5, - "recall": 0.0, - "total": 2967, - "trueNegative": 2962, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "task": { - "f1": 0.125, - "falseNegative": 495, - "falsePositive": 149, - "precision": 0.2359, - "predictedPositive": 195, - "recall": 0.085, - "total": 2967, - "trueNegative": 2277, - "truePositive": 46, - "wilsonPrecisionLower95": 0.1818 - } - }, - "recordCount": 2967 - }, - "AmazonScience/MASSIVE zh-CN test": { - "intentsMeeting95Precision": 2, - "macroF1": 0.1042, - "macroPrecision": 0.3687, - "macroRecall": 0.0795, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 2850, - "trueNegative": 2850, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 2850, - "trueNegative": 2850, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 2850, - "trueNegative": 2850, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.012, - "falseNegative": 165, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 1, - "recall": 0.006, - "total": 2850, - "trueNegative": 2684, - "truePositive": 1, - "wilsonPrecisionLower95": 0.2065 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 2850, - "trueNegative": 2850, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.1535, - "falseNegative": 1432, - "falsePositive": 2, - "precision": 0.9848, - "predictedPositive": 132, - "recall": 0.0832, - "total": 2850, - "trueNegative": 1286, - "truePositive": 130, - "wilsonPrecisionLower95": 0.9464 - }, - "replyableMessage": { - "f1": 0.7684, - "falseNegative": 1072, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 1778, - "recall": 0.6239, - "total": 2850, - "trueNegative": 0, - "truePositive": 1778, - "wilsonPrecisionLower95": 0.9978 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 2850, - "trueNegative": 2850, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0042, - "falseNegative": 477, - "falsePositive": 2, - "precision": 0.3333, - "predictedPositive": 3, - "recall": 0.0021, - "total": 2850, - "trueNegative": 2370, - "truePositive": 1, - "wilsonPrecisionLower95": 0.0615 - } - }, - "recordCount": 2850 - }, - "CFPB via hpe-ai/customer-complaints": { - "intentsMeeting95Precision": 1, - "macroF1": 0.082, - "macroPrecision": 0.2222, - "macroRecall": 0.0621, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 250, - "trueNegative": 250, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 250, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 250, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 250, - "trueNegative": 250, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 250, - "trueNegative": 250, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 250, - "trueNegative": 250, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 17, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 250, - "trueNegative": 233, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.7013, - "falseNegative": 115, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 135, - "recall": 0.54, - "total": 250, - "trueNegative": 0, - "truePositive": 135, - "wilsonPrecisionLower95": 0.9723 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 250, - "trueNegative": 250, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.037, - "falseNegative": 52, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 1, - "recall": 0.0189, - "total": 250, - "trueNegative": 197, - "truePositive": 1, - "wilsonPrecisionLower95": 0.2065 - } - }, - "recordCount": 250 - }, - "CPED test split": { - "intentsMeeting95Precision": 0, - "macroF1": 0.1202, - "macroPrecision": 0.3154, - "macroRecall": 0.091, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.2105, - "falseNegative": 15, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 2, - "recall": 0.1176, - "total": 25049, - "trueNegative": 25032, - "truePositive": 2, - "wilsonPrecisionLower95": 0.3424 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 150, - "falsePositive": 7, - "precision": 0.0, - "predictedPositive": 7, - "recall": 0.0, - "total": 25049, - "trueNegative": 24892, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 57, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 25049, - "trueNegative": 24992, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 99, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 25049, - "trueNegative": 24950, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 73, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 25049, - "trueNegative": 24976, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.1106, - "falseNegative": 5257, - "falsePositive": 34, - "precision": 0.9063, - "predictedPositive": 363, - "recall": 0.0589, - "total": 25049, - "trueNegative": 19429, - "truePositive": 329, - "wilsonPrecisionLower95": 0.872 - }, - "replyableMessage": { - "f1": 0.7609, - "falseNegative": 8378, - "falsePositive": 1096, - "precision": 0.9322, - "predictedPositive": 16174, - "recall": 0.6428, - "total": 25049, - "trueNegative": 497, - "truePositive": 15078, - "wilsonPrecisionLower95": 0.9283 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 4, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 25049, - "trueNegative": 25045, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 168, - "falsePositive": 3, - "precision": 0.0, - "predictedPositive": 3, - "recall": 0.0, - "total": 25049, - "trueNegative": 24878, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - } - }, - "recordCount": 25049 - }, - "Google Research GoEmotions test": { - "intentsMeeting95Precision": 2, - "macroF1": 0.2362, - "macroPrecision": 0.4125, - "macroRecall": 0.175, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.4651, - "falseNegative": 46, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 20, - "recall": 0.303, - "total": 5418, - "trueNegative": 5352, - "truePositive": 20, - "wilsonPrecisionLower95": 0.8389 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 20, - "falsePositive": 4, - "precision": 0.0, - "predictedPositive": 4, - "recall": 0.0, - "total": 5418, - "trueNegative": 5394, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 5, - "precision": 0.0, - "predictedPositive": 5, - "recall": 0.0, - "total": 5418, - "trueNegative": 5413, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 3, - "precision": 0.0, - "predictedPositive": 3, - "recall": 0.0, - "total": 5418, - "trueNegative": 5415, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 5418, - "trueNegative": 5418, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.4649, - "falseNegative": 363, - "falsePositive": 26, - "precision": 0.8667, - "predictedPositive": 195, - "recall": 0.3177, - "total": 5418, - "trueNegative": 4860, - "truePositive": 169, - "wilsonPrecisionLower95": 0.8118 - }, - "replyableMessage": { - "f1": 0.8467, - "falseNegative": 1440, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 3978, - "recall": 0.7342, - "total": 5418, - "trueNegative": 0, - "truePositive": 3978, - "wilsonPrecisionLower95": 0.999 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 1, - "precision": 0.0, - "predictedPositive": 1, - "recall": 0.0, - "total": 5418, - "trueNegative": 5417, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "task": { - "f1": 0.3492, - "falseNegative": 39, - "falsePositive": 2, - "precision": 0.8462, - "predictedPositive": 13, - "recall": 0.22, - "total": 5418, - "trueNegative": 5366, - "truePositive": 11, - "wilsonPrecisionLower95": 0.5777 - } - }, - "recordCount": 5418 - }, - "Meituan-Dianping/ASAP test": { - "intentsMeeting95Precision": 2, - "macroF1": 0.1385, - "macroPrecision": 0.2222, - "macroRecall": 0.1102, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 120, - "trueNegative": 120, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.3893, - "falseNegative": 91, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 29, - "recall": 0.2417, - "total": 120, - "trueNegative": 0, - "truePositive": 29, - "wilsonPrecisionLower95": 0.883 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 120, - "trueNegative": 120, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 120, - "trueNegative": 120, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 120, - "trueNegative": 120, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 36, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 120, - "trueNegative": 84, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.8571, - "falseNegative": 30, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 90, - "recall": 0.75, - "total": 120, - "trueNegative": 0, - "truePositive": 90, - "wilsonPrecisionLower95": 0.9591 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 120, - "trueNegative": 120, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 13, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 120, - "trueNegative": 107, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 120 - }, - "Sulak2020 GitHub issues multi-repo dataset": { - "intentsMeeting95Precision": 2, - "macroF1": 0.1469, - "macroPrecision": 0.2963, - "macroRecall": 0.1143, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 2, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 1004, - "trueNegative": 1002, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.024, - "falseNegative": 162, - "falsePositive": 1, - "precision": 0.6667, - "predictedPositive": 3, - "recall": 0.0122, - "total": 1004, - "trueNegative": 839, - "truePositive": 2, - "wilsonPrecisionLower95": 0.2077 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 2, - "falsePositive": 1, - "precision": 0.0, - "predictedPositive": 1, - "recall": 0.0, - "total": 1004, - "trueNegative": 1001, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 2, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 1004, - "trueNegative": 1002, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 1004, - "trueNegative": 1004, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.4658, - "falseNegative": 39, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 17, - "recall": 0.3036, - "total": 1004, - "trueNegative": 948, - "truePositive": 17, - "wilsonPrecisionLower95": 0.8157 - }, - "replyableMessage": { - "f1": 0.8326, - "falseNegative": 288, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 716, - "recall": 0.7131, - "total": 1004, - "trueNegative": 0, - "truePositive": 716, - "wilsonPrecisionLower95": 0.9947 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 25, - "falsePositive": 2, - "precision": 0.0, - "predictedPositive": 2, - "recall": 0.0, - "total": 1004, - "trueNegative": 977, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "task": { - "f1": 0.0, - "falseNegative": 90, - "falsePositive": 1, - "precision": 0.0, - "predictedPositive": 1, - "recall": 0.0, - "total": 1004, - "trueNegative": 913, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - } - }, - "recordCount": 1004 - }, - "Verified Enron Intent Dataset": { - "intentsMeeting95Precision": 1, - "macroF1": 0.3899, - "macroPrecision": 0.5025, - "macroRecall": 0.3852, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 1, - "recall": 1.0, - "total": 3537, - "trueNegative": 3536, - "truePositive": 1, - "wilsonPrecisionLower95": 0.2065 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 3537, - "trueNegative": 3537, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 10, - "falsePositive": 4, - "precision": 0.0, - "predictedPositive": 4, - "recall": 0.0, - "total": 3537, - "trueNegative": 3523, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "followUpReminder": { - "f1": 0.2286, - "falseNegative": 22, - "falsePositive": 5, - "precision": 0.4444, - "predictedPositive": 9, - "recall": 0.1538, - "total": 3537, - "trueNegative": 3506, - "truePositive": 4, - "wilsonPrecisionLower95": 0.1888 - }, - "invitation": { - "f1": 0.3014, - "falseNegative": 50, - "falsePositive": 1, - "precision": 0.9167, - "predictedPositive": 12, - "recall": 0.1803, - "total": 3537, - "trueNegative": 3475, - "truePositive": 11, - "wilsonPrecisionLower95": 0.6461 - }, - "question": { - "f1": 0.8, - "falseNegative": 143, - "falsePositive": 3, - "precision": 0.9898, - "predictedPositive": 295, - "recall": 0.6713, - "total": 3537, - "trueNegative": 3099, - "truePositive": 292, - "wilsonPrecisionLower95": 0.9705 - }, - "replyableMessage": { - "f1": 0.5259, - "falseNegative": 115, - "falsePositive": 1917, - "precision": 0.3702, - "predictedPositive": 3044, - "recall": 0.9074, - "total": 3537, - "trueNegative": 378, - "truePositive": 1127, - "wilsonPrecisionLower95": 0.3533 - }, - "scheduleNegotiation": { - "f1": 0.209, - "falseNegative": 35, - "falsePositive": 18, - "precision": 0.28, - "predictedPositive": 25, - "recall": 0.1667, - "total": 3537, - "trueNegative": 3477, - "truePositive": 7, - "wilsonPrecisionLower95": 0.1428 - }, - "task": { - "f1": 0.4441, - "falseNegative": 236, - "falsePositive": 137, - "precision": 0.521, - "predictedPositive": 286, - "recall": 0.387, - "total": 3537, - "trueNegative": 3015, - "truePositive": 149, - "wilsonPrecisionLower95": 0.4632 - } - }, - "recordCount": 3537 - } - }, - "intentsMeeting95Precision": 2, - "macroF1": 0.2325, - "macroPrecision": 0.6097, - "macroRecall": 0.1647, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.422, - "falseNegative": 63, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 23, - "recall": 0.2674, - "total": 41195, - "trueNegative": 41109, - "truePositive": 23, - "wilsonPrecisionLower95": 0.8569 - }, - "complaint": { - "f1": 0.0829, - "falseNegative": 673, - "falsePositive": 13, - "precision": 0.7045, - "predictedPositive": 44, - "recall": 0.044, - "total": 41195, - "trueNegative": 40478, - "truePositive": 31, - "wilsonPrecisionLower95": 0.5578 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 69, - "falsePositive": 11, - "precision": 0.0, - "predictedPositive": 11, - "recall": 0.0, - "total": 41195, - "trueNegative": 41115, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "followUpReminder": { - "f1": 0.1325, - "falseNegative": 359, - "falsePositive": 47, - "precision": 0.3974, - "predictedPositive": 78, - "recall": 0.0795, - "total": 41195, - "trueNegative": 40758, - "truePositive": 31, - "wilsonPrecisionLower95": 0.2961 - }, - "invitation": { - "f1": 0.1507, - "falseNegative": 123, - "falsePositive": 1, - "precision": 0.9167, - "predictedPositive": 12, - "recall": 0.0821, - "total": 41195, - "trueNegative": 41060, - "truePositive": 11, - "wilsonPrecisionLower95": 0.6461 - }, - "question": { - "f1": 0.2246, - "falseNegative": 8587, - "falsePositive": 65, - "precision": 0.9507, - "predictedPositive": 1318, - "recall": 0.1273, - "total": 41195, - "trueNegative": 31290, - "truePositive": 1253, - "wilsonPrecisionLower95": 0.9376 - }, - "replyableMessage": { - "f1": 0.7628, - "falseNegative": 12447, - "falsePositive": 3013, - "precision": 0.8919, - "predictedPositive": 27873, - "recall": 0.6664, - "total": 41195, - "trueNegative": 875, - "truePositive": 24860, - "wilsonPrecisionLower95": 0.8882 - }, - "scheduleNegotiation": { - "f1": 0.1346, - "falseNegative": 64, - "falsePositive": 26, - "precision": 0.2121, - "predictedPositive": 33, - "recall": 0.0986, - "total": 41195, - "trueNegative": 41098, - "truePositive": 7, - "wilsonPrecisionLower95": 0.1068 - }, - "task": { - "f1": 0.1825, - "falseNegative": 1570, - "falsePositive": 294, - "precision": 0.4143, - "predictedPositive": 502, - "recall": 0.117, - "total": 41195, - "trueNegative": 39123, - "truePositive": 208, - "wilsonPrecisionLower95": 0.3721 - } - }, - "recordCount": 41195 - }, - "targetedRelease": { - "byLanguage": { - "en": { - "intentsMeeting95Precision": 4, - "macroF1": 0.3481, - "macroPrecision": 0.5476, - "macroRecall": 0.2799, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 320, - "trueNegative": 320, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.3562, - "falseNegative": 94, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 26, - "recall": 0.2167, - "total": 320, - "trueNegative": 200, - "truePositive": 26, - "wilsonPrecisionLower95": 0.8713 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 320, - "trueNegative": 320, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.8732, - "falseNegative": 9, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 31, - "recall": 0.775, - "total": 320, - "trueNegative": 280, - "truePositive": 31, - "wilsonPrecisionLower95": 0.8897 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 320, - "trueNegative": 320, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.4762, - "falseNegative": 55, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 25, - "recall": 0.3125, - "total": 320, - "trueNegative": 240, - "truePositive": 25, - "wilsonPrecisionLower95": 0.8668 - }, - "replyableMessage": { - "f1": 0.8819, - "falseNegative": 32, - "falsePositive": 13, - "precision": 0.9282, - "predictedPositive": 181, - "recall": 0.84, - "total": 320, - "trueNegative": 107, - "truePositive": 168, - "wilsonPrecisionLower95": 0.881 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 320, - "trueNegative": 320, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.5455, - "falseNegative": 50, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 30, - "recall": 0.375, - "total": 320, - "trueNegative": 240, - "truePositive": 30, - "wilsonPrecisionLower95": 0.8865 - } - }, - "recordCount": 320 - }, - "zh-Hans": { - "intentsMeeting95Precision": 3, - "macroF1": 0.2553, - "macroPrecision": 0.3278, - "macroRecall": 0.2275, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 320, - "trueNegative": 320, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 120, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 320, - "trueNegative": 200, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 320, - "trueNegative": 320, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 40, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 320, - "trueNegative": 280, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 320, - "trueNegative": 320, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.8811, - "falseNegative": 17, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 63, - "recall": 0.7875, - "total": 320, - "trueNegative": 240, - "truePositive": 63, - "wilsonPrecisionLower95": 0.9425 - }, - "replyableMessage": { - "f1": 0.9552, - "falseNegative": 8, - "falsePositive": 10, - "precision": 0.9505, - "predictedPositive": 202, - "recall": 0.96, - "total": 320, - "trueNegative": 110, - "truePositive": 192, - "wilsonPrecisionLower95": 0.9113 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 320, - "trueNegative": 320, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.4615, - "falseNegative": 56, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 24, - "recall": 0.3, - "total": 320, - "trueNegative": 240, - "truePositive": 24, - "wilsonPrecisionLower95": 0.862 - } - }, - "recordCount": 320 - } - }, - "bySource": { - "release_complaint_brief": { - "intentsMeeting95Precision": 1, - "macroF1": 0.1329, - "macroPrecision": 0.2222, - "macroRecall": 0.1222, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.2022, - "falseNegative": 71, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 9, - "recall": 0.1125, - "total": 80, - "trueNegative": 0, - "truePositive": 9, - "wilsonPrecisionLower95": 0.7009 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.9937, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 79, - "recall": 0.9875, - "total": 80, - "trueNegative": 0, - "truePositive": 79, - "wilsonPrecisionLower95": 0.9536 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 80 - }, - "release_complaint_question": { - "intentsMeeting95Precision": 3, - "macroF1": 0.1778, - "macroPrecision": 0.3333, - "macroRecall": 0.1375, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.2222, - "falseNegative": 70, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 10, - "recall": 0.125, - "total": 80, - "trueNegative": 0, - "truePositive": 10, - "wilsonPrecisionLower95": 0.7225 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.5047, - "falseNegative": 53, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 27, - "recall": 0.3375, - "total": 80, - "trueNegative": 0, - "truePositive": 27, - "wilsonPrecisionLower95": 0.8754 - }, - "replyableMessage": { - "f1": 0.8732, - "falseNegative": 18, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 62, - "recall": 0.775, - "total": 80, - "trueNegative": 0, - "truePositive": 62, - "wilsonPrecisionLower95": 0.9417 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 80 - }, - "release_complaint_unspoken": { - "intentsMeeting95Precision": 1, - "macroF1": 0.1231, - "macroPrecision": 0.2222, - "macroRecall": 0.1097, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.1609, - "falseNegative": 73, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 7, - "recall": 0.0875, - "total": 80, - "trueNegative": 0, - "truePositive": 7, - "wilsonPrecisionLower95": 0.6457 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.9474, - "falseNegative": 8, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 72, - "recall": 0.9, - "total": 80, - "trueNegative": 0, - "truePositive": 72, - "wilsonPrecisionLower95": 0.9493 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 80 - }, - "release_negative_boundary": { - "intentsMeeting95Precision": 0, - "macroF1": 0.0, - "macroPrecision": 0.0, - "macroRecall": 0.0, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 80 - }, - "release_personal_boundary": { - "intentsMeeting95Precision": 1, - "macroF1": 0.0621, - "macroPrecision": 0.1111, - "macroRecall": 0.0431, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.5586, - "falseNegative": 49, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 31, - "recall": 0.3875, - "total": 80, - "trueNegative": 0, - "truePositive": 31, - "wilsonPrecisionLower95": 0.8897 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 4, - "precision": 0.0, - "predictedPositive": 4, - "recall": 0.0, - "total": 80, - "trueNegative": 76, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 80 - }, - "release_resolved_boundary": { - "intentsMeeting95Precision": 0, - "macroF1": 0.0, - "macroPrecision": 0.0, - "macroRecall": 0.0, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 19, - "precision": 0.0, - "predictedPositive": 19, - "recall": 0.0, - "total": 80, - "trueNegative": 61, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 80 - }, - "release_task_owner": { - "intentsMeeting95Precision": 2, - "macroF1": 0.1474, - "macroPrecision": 0.2222, - "macroRecall": 0.1319, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.9937, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 79, - "recall": 0.9875, - "total": 80, - "trueNegative": 0, - "truePositive": 79, - "wilsonPrecisionLower95": 0.9536 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.3333, - "falseNegative": 64, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 16, - "recall": 0.2, - "total": 80, - "trueNegative": 0, - "truePositive": 16, - "wilsonPrecisionLower95": 0.8064 - } - }, - "recordCount": 80 - }, - "release_task_polite_request": { - "intentsMeeting95Precision": 3, - "macroF1": 0.2698, - "macroPrecision": 0.3333, - "macroRecall": 0.2319, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.8652, - "falseNegative": 19, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 61, - "recall": 0.7625, - "total": 80, - "trueNegative": 0, - "truePositive": 61, - "wilsonPrecisionLower95": 0.9408 - }, - "replyableMessage": { - "f1": 0.9189, - "falseNegative": 12, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 68, - "recall": 0.85, - "total": 80, - "trueNegative": 0, - "truePositive": 68, - "wilsonPrecisionLower95": 0.9465 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.6441, - "falseNegative": 42, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 38, - "recall": 0.475, - "total": 80, - "trueNegative": 0, - "truePositive": 38, - "wilsonPrecisionLower95": 0.9082 - } - }, - "recordCount": 80 - } - }, - "intentsMeeting95Precision": 4, - "macroF1": 0.3209, - "macroPrecision": 0.5489, - "macroRecall": 0.2537, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 640, - "trueNegative": 640, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.1955, - "falseNegative": 214, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 26, - "recall": 0.1083, - "total": 640, - "trueNegative": 400, - "truePositive": 26, - "wilsonPrecisionLower95": 0.8713 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 640, - "trueNegative": 640, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.5586, - "falseNegative": 49, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 31, - "recall": 0.3875, - "total": 640, - "trueNegative": 560, - "truePositive": 31, - "wilsonPrecisionLower95": 0.8897 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 640, - "trueNegative": 640, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.7097, - "falseNegative": 72, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 88, - "recall": 0.55, - "total": 640, - "trueNegative": 480, - "truePositive": 88, - "wilsonPrecisionLower95": 0.9582 - }, - "replyableMessage": { - "f1": 0.9195, - "falseNegative": 40, - "falsePositive": 23, - "precision": 0.9399, - "predictedPositive": 383, - "recall": 0.9, - "total": 640, - "trueNegative": 217, - "truePositive": 360, - "wilsonPrecisionLower95": 0.9115 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 640, - "trueNegative": 640, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.5047, - "falseNegative": 106, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 54, - "recall": 0.3375, - "total": 640, - "trueNegative": 480, - "truePositive": 54, - "wilsonPrecisionLower95": 0.9336 - } - }, - "recordCount": 640 - } - }, - "generatedAtUnix": 1787826081, - "openTrainingRecords": 3845, - "overlapChecks": { - "ModelTraining/ClipboardSemantics/comprehensive-online-holdout-corpus.jsonl": 0, - "ModelTraining/ClipboardSemantics/random-holdout-corpus.jsonl": 0, - "ModelTraining/ClipboardSemantics/targeted-release-holdout-corpus.jsonl": 0 - }, - "platformLimitation": "Linux surrogate research only. Apple Create ML/Core ML artifacts must be retrained and tested on macOS before deployment.", - "releaseGatePassed": false, - "roundCount": 20, - "schemaVersion": 1, - "seed": 20260827, - "selectedConfiguration": { - "alpha": 2e-05, - "augmentation": "none", - "char_max": 5, - "char_min": 2, - "external_weight": 0.5700000000000001, - "hard_example_weight": 1.0, - "l1_ratio": 0.0, - "max_char_features": 45000, - "max_word_features": 22000, - "min_df": 2, - "pseudo_weight": 0.16499999999999998, - "round": 1, - "word_max": 2 - }, - "selectedRound": 1, - "selectedRoundsByIntent": { - "blessing": 19, - "complaint": 1, - "confirmationDecision": 11, - "followUpReminder": 14, - "invitation": 1, - "question": 1, - "replyableMessage": 1, - "scheduleNegotiation": 7, - "task": 1 - }, - "selectedThresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 180, - "recall": 1.0, - "total": 1485, - "trueNegative": 1305, - "truePositive": 180, - "wilsonPrecisionLower95": 0.9791 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9526, - "falseNegative": 9, - "falsePositive": 8, - "precision": 0.9553, - "predictedPositive": 179, - "recall": 0.95, - "total": 1485, - "trueNegative": 1297, - "truePositive": 171, - "wilsonPrecisionLower95": 0.9143 - }, - "threshold": 0.54 - } - }, - "metrics": { - "f1": 0.9764, - "falseNegative": 9, - "falsePositive": 8, - "precision": 0.9777, - "predictedPositive": 359, - "recall": 0.975, - "total": 2970, - "trueNegative": 2602, - "truePositive": 351, - "wilsonPrecisionLower95": 0.9567 - }, - "threshold": 0.54 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.931, - "falseNegative": 9, - "falsePositive": 3, - "precision": 0.9643, - "predictedPositive": 84, - "recall": 0.9, - "total": 1485, - "trueNegative": 1392, - "truePositive": 81, - "wilsonPrecisionLower95": 0.9002 - }, - "threshold": 0.97 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7222, - "falseNegative": 38, - "falsePositive": 2, - "precision": 0.963, - "predictedPositive": 54, - "recall": 0.5778, - "total": 1485, - "trueNegative": 1393, - "truePositive": 52, - "wilsonPrecisionLower95": 0.8746 - }, - "threshold": 0.86 - } - }, - "metrics": { - "f1": 0.8077, - "falseNegative": 54, - "falsePositive": 6, - "precision": 0.9545, - "predictedPositive": 132, - "recall": 0.7, - "total": 2970, - "trueNegative": 2784, - "truePositive": 126, - "wilsonPrecisionLower95": 0.9044 - }, - "threshold": 0.95 - }, - "followUpReminder": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8788, - "falseNegative": 35, - "falsePositive": 5, - "precision": 0.9667, - "predictedPositive": 150, - "recall": 0.8056, - "total": 1485, - "trueNegative": 1300, - "truePositive": 145, - "wilsonPrecisionLower95": 0.9243 - }, - "threshold": 0.845 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6113, - "falseNegative": 99, - "falsePositive": 4, - "precision": 0.9529, - "predictedPositive": 85, - "recall": 0.45, - "total": 1485, - "trueNegative": 1301, - "truePositive": 81, - "wilsonPrecisionLower95": 0.8852 - }, - "threshold": 0.97 - } - }, - "metrics": { - "f1": 0.7682, - "falseNegative": 128, - "falsePositive": 12, - "precision": 0.9508, - "predictedPositive": 244, - "recall": 0.6444, - "total": 2970, - "trueNegative": 2598, - "truePositive": 232, - "wilsonPrecisionLower95": 0.916 - }, - "threshold": 0.91 - }, - "invitation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8302, - "falseNegative": 24, - "falsePositive": 3, - "precision": 0.9565, - "predictedPositive": 69, - "recall": 0.7333, - "total": 1485, - "trueNegative": 1392, - "truePositive": 66, - "wilsonPrecisionLower95": 0.8798 - }, - "threshold": 0.935 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6765, - "falseNegative": 44, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 46, - "recall": 0.5111, - "total": 1485, - "trueNegative": 1395, - "truePositive": 46, - "wilsonPrecisionLower95": 0.9229 - }, - "threshold": 0.505 - } - }, - "metrics": { - "f1": 0.5476, - "falseNegative": 111, - "falsePositive": 3, - "precision": 0.9583, - "predictedPositive": 72, - "recall": 0.3833, - "total": 2970, - "trueNegative": 2787, - "truePositive": 69, - "wilsonPrecisionLower95": 0.8845 - }, - "threshold": 0.935 - }, - "question": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8445, - "falseNegative": 76, - "falsePositive": 12, - "precision": 0.9522, - "predictedPositive": 251, - "recall": 0.7587, - "total": 1485, - "trueNegative": 1158, - "truePositive": 239, - "wilsonPrecisionLower95": 0.9183 - }, - "threshold": 0.725 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8536, - "falseNegative": 76, - "falsePositive": 6, - "precision": 0.9755, - "predictedPositive": 245, - "recall": 0.7587, - "total": 1485, - "trueNegative": 1164, - "truePositive": 239, - "wilsonPrecisionLower95": 0.9476 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.8729, - "falseNegative": 122, - "falsePositive": 26, - "precision": 0.9513, - "predictedPositive": 534, - "recall": 0.8063, - "total": 2970, - "trueNegative": 2314, - "truePositive": 508, - "wilsonPrecisionLower95": 0.9296 - }, - "threshold": 0.515 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.948, - "falseNegative": 52, - "falsePositive": 46, - "precision": 0.951, - "predictedPositive": 939, - "recall": 0.945, - "total": 1485, - "trueNegative": 494, - "truePositive": 893, - "wilsonPrecisionLower95": 0.9353 - }, - "threshold": 0.7 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9424, - "falseNegative": 62, - "falsePositive": 46, - "precision": 0.9505, - "predictedPositive": 929, - "recall": 0.9344, - "total": 1485, - "trueNegative": 494, - "truePositive": 883, - "wilsonPrecisionLower95": 0.9346 - }, - "threshold": 0.695 - } - }, - "metrics": { - "f1": 0.9454, - "falseNegative": 115, - "falsePositive": 90, - "precision": 0.9517, - "predictedPositive": 1865, - "recall": 0.9392, - "total": 2970, - "trueNegative": 990, - "truePositive": 1775, - "wilsonPrecisionLower95": 0.9411 - }, - "threshold": 0.7 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 45, - "recall": 1.0, - "total": 1485, - "trueNegative": 1440, - "truePositive": 45, - "wilsonPrecisionLower95": 0.9213 - }, - "threshold": 0.66 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 45, - "recall": 1.0, - "total": 1485, - "trueNegative": 1440, - "truePositive": 45, - "wilsonPrecisionLower95": 0.9213 - }, - "threshold": 0.55 - } - }, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 90, - "recall": 1.0, - "total": 2970, - "trueNegative": 2880, - "truePositive": 90, - "wilsonPrecisionLower95": 0.9591 - }, - "threshold": 0.66 - }, - "task": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8403, - "falseNegative": 89, - "falsePositive": 14, - "precision": 0.9509, - "predictedPositive": 285, - "recall": 0.7528, - "total": 1485, - "trueNegative": 1111, - "truePositive": 271, - "wilsonPrecisionLower95": 0.9192 - }, - "threshold": 0.835 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8278, - "falseNegative": 98, - "falsePositive": 11, - "precision": 0.9597, - "predictedPositive": 273, - "recall": 0.7278, - "total": 1485, - "trueNegative": 1114, - "truePositive": 262, - "wilsonPrecisionLower95": 0.9293 - }, - "threshold": 0.745 - } - }, - "metrics": { - "f1": 0.8637, - "falseNegative": 153, - "falsePositive": 26, - "precision": 0.9562, - "predictedPositive": 593, - "recall": 0.7875, - "total": 2970, - "trueNegative": 2224, - "truePositive": 567, - "wilsonPrecisionLower95": 0.9365 - }, - "threshold": 0.755 - } - }, - "selectionObjective": 0.6928, - "trainingPolicy": "No user clipboard data. Official partial labels plus deterministic label-preserving augmentation and three-round high-confidence self-training consensus. Frozen holdouts never enter training or threshold calibration." -} diff --git a/ModelTraining/ClipboardSemantics/IterativeResearch/EvidenceGateFineTune/rounds.json b/ModelTraining/ClipboardSemantics/IterativeResearch/EvidenceGateFineTune/rounds.json deleted file mode 100644 index f0b25a6..0000000 --- a/ModelTraining/ClipboardSemantics/IterativeResearch/EvidenceGateFineTune/rounds.json +++ /dev/null @@ -1,14831 +0,0 @@ -[ - { - "configuration": { - "alpha": 2e-05, - "augmentation": "none", - "char_max": 5, - "char_min": 2, - "external_weight": 0.5700000000000001, - "hard_example_weight": 1.0, - "l1_ratio": 0.0, - "max_char_features": 45000, - "max_word_features": 22000, - "min_df": 2, - "pseudo_weight": 0.16499999999999998, - "round": 1, - "word_max": 2 - }, - "featureCount": 56451, - "pseudoLabelCounts": {}, - "round": 1, - "selectionObjective": 0.6928, - "syntheticRegression": { - "intentsMeeting95Precision": 7, - "macroF1": 0.5923, - "macroPrecision": 0.9767, - "macroRecall": 0.4748, - "minimumPrecision": 0.813, - "perIntent": { - "blessing": { - "f1": 0.125, - "falseNegative": 84, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 6, - "recall": 0.0667, - "total": 2970, - "trueNegative": 2880, - "truePositive": 6, - "wilsonPrecisionLower95": 0.6097 - }, - "complaint": { - "f1": 0.4279, - "falseNegative": 262, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 98, - "recall": 0.2722, - "total": 2970, - "trueNegative": 2610, - "truePositive": 98, - "wilsonPrecisionLower95": 0.9623 - }, - "confirmationDecision": { - "f1": 0.6601, - "falseNegative": 80, - "falsePositive": 23, - "precision": 0.813, - "predictedPositive": 123, - "recall": 0.5556, - "total": 2970, - "trueNegative": 2767, - "truePositive": 100, - "wilsonPrecisionLower95": 0.735 - }, - "followUpReminder": { - "f1": 0.4382, - "falseNegative": 259, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 101, - "recall": 0.2806, - "total": 2970, - "trueNegative": 2610, - "truePositive": 101, - "wilsonPrecisionLower95": 0.9634 - }, - "invitation": { - "f1": 0.481, - "falseNegative": 123, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 57, - "recall": 0.3167, - "total": 2970, - "trueNegative": 2790, - "truePositive": 57, - "wilsonPrecisionLower95": 0.9369 - }, - "question": { - "f1": 0.8545, - "falseNegative": 160, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 470, - "recall": 0.746, - "total": 2970, - "trueNegative": 2340, - "truePositive": 470, - "wilsonPrecisionLower95": 0.9919 - }, - "replyableMessage": { - "f1": 0.9465, - "falseNegative": 156, - "falsePositive": 40, - "precision": 0.9775, - "predictedPositive": 1774, - "recall": 0.9175, - "total": 2970, - "trueNegative": 1040, - "truePositive": 1734, - "wilsonPrecisionLower95": 0.9694 - }, - "scheduleNegotiation": { - "f1": 0.8535, - "falseNegative": 23, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 67, - "recall": 0.7444, - "total": 2970, - "trueNegative": 2880, - "truePositive": 67, - "wilsonPrecisionLower95": 0.9458 - }, - "task": { - "f1": 0.544, - "falseNegative": 451, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 269, - "recall": 0.3736, - "total": 2970, - "trueNegative": 2250, - "truePositive": 269, - "wilsonPrecisionLower95": 0.9859 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9318, - "falseNegative": 4, - "falsePositive": 2, - "precision": 0.9535, - "predictedPositive": 43, - "recall": 0.9111, - "total": 1485, - "trueNegative": 1438, - "truePositive": 41, - "wilsonPrecisionLower95": 0.8454 - }, - "threshold": 0.915 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7568, - "falseNegative": 17, - "falsePositive": 1, - "precision": 0.9655, - "predictedPositive": 29, - "recall": 0.6222, - "total": 1485, - "trueNegative": 1439, - "truePositive": 28, - "wilsonPrecisionLower95": 0.8282 - }, - "threshold": 0.9 - } - }, - "metrics": { - "f1": 0.8375, - "falseNegative": 23, - "falsePositive": 3, - "precision": 0.9571, - "predictedPositive": 70, - "recall": 0.7444, - "total": 2970, - "trueNegative": 2877, - "truePositive": 67, - "wilsonPrecisionLower95": 0.8814 - }, - "threshold": 0.915 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 180, - "recall": 1.0, - "total": 1485, - "trueNegative": 1305, - "truePositive": 180, - "wilsonPrecisionLower95": 0.9791 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9526, - "falseNegative": 9, - "falsePositive": 8, - "precision": 0.9553, - "predictedPositive": 179, - "recall": 0.95, - "total": 1485, - "trueNegative": 1297, - "truePositive": 171, - "wilsonPrecisionLower95": 0.9143 - }, - "threshold": 0.54 - } - }, - "metrics": { - "f1": 0.9764, - "falseNegative": 9, - "falsePositive": 8, - "precision": 0.9777, - "predictedPositive": 359, - "recall": 0.975, - "total": 2970, - "trueNegative": 2602, - "truePositive": 351, - "wilsonPrecisionLower95": 0.9567 - }, - "threshold": 0.54 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8848, - "falseNegative": 17, - "falsePositive": 2, - "precision": 0.9733, - "predictedPositive": 75, - "recall": 0.8111, - "total": 1485, - "trueNegative": 1393, - "truePositive": 73, - "wilsonPrecisionLower95": 0.9079 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7947, - "falseNegative": 30, - "falsePositive": 1, - "precision": 0.9836, - "predictedPositive": 61, - "recall": 0.6667, - "total": 1485, - "trueNegative": 1394, - "truePositive": 60, - "wilsonPrecisionLower95": 0.9128 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.8418, - "falseNegative": 47, - "falsePositive": 3, - "precision": 0.9779, - "predictedPositive": 136, - "recall": 0.7389, - "total": 2970, - "trueNegative": 2787, - "truePositive": 133, - "wilsonPrecisionLower95": 0.9372 - }, - "threshold": 0.5 - }, - "followUpReminder": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8554, - "falseNegative": 41, - "falsePositive": 6, - "precision": 0.9586, - "predictedPositive": 145, - "recall": 0.7722, - "total": 1485, - "trueNegative": 1299, - "truePositive": 139, - "wilsonPrecisionLower95": 0.9127 - }, - "threshold": 0.83 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8354, - "falseNegative": 48, - "falsePositive": 4, - "precision": 0.9706, - "predictedPositive": 136, - "recall": 0.7333, - "total": 1485, - "trueNegative": 1301, - "truePositive": 132, - "wilsonPrecisionLower95": 0.9268 - }, - "threshold": 0.505 - } - }, - "metrics": { - "f1": 0.7994, - "falseNegative": 113, - "falsePositive": 11, - "precision": 0.9574, - "predictedPositive": 258, - "recall": 0.6861, - "total": 2970, - "trueNegative": 2599, - "truePositive": 247, - "wilsonPrecisionLower95": 0.9253 - }, - "threshold": 0.815 - }, - "invitation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8302, - "falseNegative": 24, - "falsePositive": 3, - "precision": 0.9565, - "predictedPositive": 69, - "recall": 0.7333, - "total": 1485, - "trueNegative": 1392, - "truePositive": 66, - "wilsonPrecisionLower95": 0.8798 - }, - "threshold": 0.935 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6765, - "falseNegative": 44, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 46, - "recall": 0.5111, - "total": 1485, - "trueNegative": 1395, - "truePositive": 46, - "wilsonPrecisionLower95": 0.9229 - }, - "threshold": 0.505 - } - }, - "metrics": { - "f1": 0.5476, - "falseNegative": 111, - "falsePositive": 3, - "precision": 0.9583, - "predictedPositive": 72, - "recall": 0.3833, - "total": 2970, - "trueNegative": 2787, - "truePositive": 69, - "wilsonPrecisionLower95": 0.8845 - }, - "threshold": 0.935 - }, - "question": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8445, - "falseNegative": 76, - "falsePositive": 12, - "precision": 0.9522, - "predictedPositive": 251, - "recall": 0.7587, - "total": 1485, - "trueNegative": 1158, - "truePositive": 239, - "wilsonPrecisionLower95": 0.9183 - }, - "threshold": 0.725 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8536, - "falseNegative": 76, - "falsePositive": 6, - "precision": 0.9755, - "predictedPositive": 245, - "recall": 0.7587, - "total": 1485, - "trueNegative": 1164, - "truePositive": 239, - "wilsonPrecisionLower95": 0.9476 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.8729, - "falseNegative": 122, - "falsePositive": 26, - "precision": 0.9513, - "predictedPositive": 534, - "recall": 0.8063, - "total": 2970, - "trueNegative": 2314, - "truePositive": 508, - "wilsonPrecisionLower95": 0.9296 - }, - "threshold": 0.515 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.948, - "falseNegative": 52, - "falsePositive": 46, - "precision": 0.951, - "predictedPositive": 939, - "recall": 0.945, - "total": 1485, - "trueNegative": 494, - "truePositive": 893, - "wilsonPrecisionLower95": 0.9353 - }, - "threshold": 0.7 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9424, - "falseNegative": 62, - "falsePositive": 46, - "precision": 0.9505, - "predictedPositive": 929, - "recall": 0.9344, - "total": 1485, - "trueNegative": 494, - "truePositive": 883, - "wilsonPrecisionLower95": 0.9346 - }, - "threshold": 0.695 - } - }, - "metrics": { - "f1": 0.9454, - "falseNegative": 115, - "falsePositive": 90, - "precision": 0.9517, - "predictedPositive": 1865, - "recall": 0.9392, - "total": 2970, - "trueNegative": 990, - "truePositive": 1775, - "wilsonPrecisionLower95": 0.9411 - }, - "threshold": 0.7 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 45, - "recall": 1.0, - "total": 1485, - "trueNegative": 1440, - "truePositive": 45, - "wilsonPrecisionLower95": 0.9213 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.967, - "falseNegative": 1, - "falsePositive": 2, - "precision": 0.9565, - "predictedPositive": 46, - "recall": 0.9778, - "total": 1485, - "trueNegative": 1438, - "truePositive": 44, - "wilsonPrecisionLower95": 0.8547 - }, - "threshold": 0.93 - } - }, - "metrics": { - "f1": 0.978, - "falseNegative": 1, - "falsePositive": 3, - "precision": 0.9674, - "predictedPositive": 92, - "recall": 0.9889, - "total": 2970, - "trueNegative": 2877, - "truePositive": 89, - "wilsonPrecisionLower95": 0.9085 - }, - "threshold": 0.91 - }, - "task": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8403, - "falseNegative": 89, - "falsePositive": 14, - "precision": 0.9509, - "predictedPositive": 285, - "recall": 0.7528, - "total": 1485, - "trueNegative": 1111, - "truePositive": 271, - "wilsonPrecisionLower95": 0.9192 - }, - "threshold": 0.835 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8278, - "falseNegative": 98, - "falsePositive": 11, - "precision": 0.9597, - "predictedPositive": 273, - "recall": 0.7278, - "total": 1485, - "trueNegative": 1114, - "truePositive": 262, - "wilsonPrecisionLower95": 0.9293 - }, - "threshold": 0.745 - } - }, - "metrics": { - "f1": 0.8637, - "falseNegative": 153, - "falsePositive": 26, - "precision": 0.9562, - "predictedPositive": 593, - "recall": 0.7875, - "total": 2970, - "trueNegative": 2224, - "truePositive": 567, - "wilsonPrecisionLower95": 0.9365 - }, - "threshold": 0.755 - } - }, - "trainingRecords": 15345, - "validation": { - "intentsMeeting95Precision": 8, - "macroF1": 0.559, - "macroPrecision": 0.9784, - "macroRecall": 0.4293, - "minimumPrecision": 0.9412, - "perIntent": { - "blessing": { - "f1": 0.3636, - "falseNegative": 70, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 20, - "recall": 0.2222, - "total": 2970, - "trueNegative": 2880, - "truePositive": 20, - "wilsonPrecisionLower95": 0.8389 - }, - "complaint": { - "f1": 0.5542, - "falseNegative": 222, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 138, - "recall": 0.3833, - "total": 2970, - "trueNegative": 2610, - "truePositive": 138, - "wilsonPrecisionLower95": 0.9729 - }, - "confirmationDecision": { - "f1": 0.6023, - "falseNegative": 102, - "falsePositive": 1, - "precision": 0.9873, - "predictedPositive": 79, - "recall": 0.4333, - "total": 2970, - "trueNegative": 2789, - "truePositive": 78, - "wilsonPrecisionLower95": 0.9317 - }, - "followUpReminder": { - "f1": 0.2884, - "falseNegative": 299, - "falsePositive": 2, - "precision": 0.9683, - "predictedPositive": 63, - "recall": 0.1694, - "total": 2970, - "trueNegative": 2608, - "truePositive": 61, - "wilsonPrecisionLower95": 0.8914 - }, - "invitation": { - "f1": 0.3256, - "falseNegative": 145, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 35, - "recall": 0.1944, - "total": 2970, - "trueNegative": 2790, - "truePositive": 35, - "wilsonPrecisionLower95": 0.9011 - }, - "question": { - "f1": 0.8386, - "falseNegative": 165, - "falsePositive": 14, - "precision": 0.9708, - "predictedPositive": 479, - "recall": 0.7381, - "total": 2970, - "trueNegative": 2326, - "truePositive": 465, - "wilsonPrecisionLower95": 0.9515 - }, - "replyableMessage": { - "f1": 0.9452, - "falseNegative": 114, - "falsePositive": 92, - "precision": 0.9507, - "predictedPositive": 1868, - "recall": 0.9397, - "total": 2970, - "trueNegative": 988, - "truePositive": 1776, - "wilsonPrecisionLower95": 0.94 - }, - "scheduleNegotiation": { - "f1": 0.5161, - "falseNegative": 58, - "falsePositive": 2, - "precision": 0.9412, - "predictedPositive": 34, - "recall": 0.3556, - "total": 2970, - "trueNegative": 2878, - "truePositive": 32, - "wilsonPrecisionLower95": 0.8091 - }, - "task": { - "f1": 0.5969, - "falseNegative": 412, - "falsePositive": 4, - "precision": 0.9872, - "predictedPositive": 312, - "recall": 0.4278, - "total": 2970, - "trueNegative": 2246, - "truePositive": 308, - "wilsonPrecisionLower95": 0.9675 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 1.5e-05, - "augmentation": "none", - "char_max": 5, - "char_min": 3, - "external_weight": 0.5900000000000001, - "hard_example_weight": 1.0, - "l1_ratio": 0.0, - "max_char_features": 50000, - "max_word_features": 25000, - "min_df": 2, - "pseudo_weight": 0.18, - "round": 2, - "word_max": 2 - }, - "featureCount": 61451, - "pseudoLabelCounts": {}, - "round": 2, - "selectionObjective": 0.6707, - "syntheticRegression": { - "intentsMeeting95Precision": 8, - "macroF1": 0.5397, - "macroPrecision": 0.9762, - "macroRecall": 0.423, - "minimumPrecision": 0.8145, - "perIntent": { - "blessing": { - "f1": 0.2178, - "falseNegative": 79, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 11, - "recall": 0.1222, - "total": 2970, - "trueNegative": 2880, - "truePositive": 11, - "wilsonPrecisionLower95": 0.7412 - }, - "complaint": { - "f1": 0.4071, - "falseNegative": 268, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 92, - "recall": 0.2556, - "total": 2970, - "trueNegative": 2610, - "truePositive": 92, - "wilsonPrecisionLower95": 0.9599 - }, - "confirmationDecision": { - "f1": 0.6645, - "falseNegative": 79, - "falsePositive": 23, - "precision": 0.8145, - "predictedPositive": 124, - "recall": 0.5611, - "total": 2970, - "trueNegative": 2767, - "truePositive": 101, - "wilsonPrecisionLower95": 0.737 - }, - "followUpReminder": { - "f1": 0.3857, - "falseNegative": 274, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 86, - "recall": 0.2389, - "total": 2970, - "trueNegative": 2610, - "truePositive": 86, - "wilsonPrecisionLower95": 0.9572 - }, - "invitation": { - "f1": 0.2, - "falseNegative": 160, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 20, - "recall": 0.1111, - "total": 2970, - "trueNegative": 2790, - "truePositive": 20, - "wilsonPrecisionLower95": 0.8389 - }, - "question": { - "f1": 0.8046, - "falseNegative": 206, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 424, - "recall": 0.673, - "total": 2970, - "trueNegative": 2340, - "truePositive": 424, - "wilsonPrecisionLower95": 0.991 - }, - "replyableMessage": { - "f1": 0.9038, - "falseNegative": 293, - "falsePositive": 47, - "precision": 0.9714, - "predictedPositive": 1644, - "recall": 0.845, - "total": 2970, - "trueNegative": 1033, - "truePositive": 1597, - "wilsonPrecisionLower95": 0.9622 - }, - "scheduleNegotiation": { - "f1": 0.8387, - "falseNegative": 25, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 65, - "recall": 0.7222, - "total": 2970, - "trueNegative": 2880, - "truePositive": 65, - "wilsonPrecisionLower95": 0.9442 - }, - "task": { - "f1": 0.4348, - "falseNegative": 520, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 200, - "recall": 0.2778, - "total": 2970, - "trueNegative": 2250, - "truePositive": 200, - "wilsonPrecisionLower95": 0.9812 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9048, - "falseNegative": 7, - "falsePositive": 1, - "precision": 0.9744, - "predictedPositive": 39, - "recall": 0.8444, - "total": 1485, - "trueNegative": 1439, - "truePositive": 38, - "wilsonPrecisionLower95": 0.8682 - }, - "threshold": 0.59 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 180, - "recall": 1.0, - "total": 1485, - "trueNegative": 1305, - "truePositive": 180, - "wilsonPrecisionLower95": 0.9791 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9226, - "falseNegative": 19, - "falsePositive": 8, - "precision": 0.9527, - "predictedPositive": 169, - "recall": 0.8944, - "total": 1485, - "trueNegative": 1297, - "truePositive": 161, - "wilsonPrecisionLower95": 0.9094 - }, - "threshold": 0.555 - } - }, - "metrics": { - "f1": 0.9649, - "falseNegative": 16, - "falsePositive": 9, - "precision": 0.9745, - "predictedPositive": 353, - "recall": 0.9556, - "total": 2970, - "trueNegative": 2601, - "truePositive": 344, - "wilsonPrecisionLower95": 0.9523 - }, - "threshold": 0.5 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.954, - "falseNegative": 7, - "falsePositive": 1, - "precision": 0.9881, - "predictedPositive": 84, - "recall": 0.9222, - "total": 1485, - "trueNegative": 1394, - "truePositive": 83, - "wilsonPrecisionLower95": 0.9356 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7183, - "falseNegative": 39, - "falsePositive": 1, - "precision": 0.9808, - "predictedPositive": 52, - "recall": 0.5667, - "total": 1485, - "trueNegative": 1394, - "truePositive": 51, - "wilsonPrecisionLower95": 0.8988 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.8481, - "falseNegative": 46, - "falsePositive": 2, - "precision": 0.9853, - "predictedPositive": 136, - "recall": 0.7444, - "total": 2970, - "trueNegative": 2788, - "truePositive": 134, - "wilsonPrecisionLower95": 0.948 - }, - "threshold": 0.5 - }, - "followUpReminder": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8328, - "falseNegative": 48, - "falsePositive": 5, - "precision": 0.9635, - "predictedPositive": 137, - "recall": 0.7333, - "total": 1485, - "trueNegative": 1300, - "truePositive": 132, - "wilsonPrecisionLower95": 0.9174 - }, - "threshold": 0.91 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6879, - "falseNegative": 83, - "falsePositive": 5, - "precision": 0.951, - "predictedPositive": 102, - "recall": 0.5389, - "total": 1485, - "trueNegative": 1300, - "truePositive": 97, - "wilsonPrecisionLower95": 0.8903 - }, - "threshold": 0.765 - } - }, - "metrics": { - "f1": 0.6243, - "falseNegative": 193, - "falsePositive": 8, - "precision": 0.9543, - "predictedPositive": 175, - "recall": 0.4639, - "total": 2970, - "trueNegative": 2602, - "truePositive": 167, - "wilsonPrecisionLower95": 0.9124 - }, - "threshold": 0.905 - }, - "invitation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9059, - "falseNegative": 13, - "falsePositive": 3, - "precision": 0.9625, - "predictedPositive": 80, - "recall": 0.8556, - "total": 1485, - "trueNegative": 1392, - "truePositive": 77, - "wilsonPrecisionLower95": 0.8955 - }, - "threshold": 0.86 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.6113, - "falseNegative": 99, - "falsePositive": 4, - "precision": 0.9529, - "predictedPositive": 85, - "recall": 0.45, - "total": 2970, - "trueNegative": 2786, - "truePositive": 81, - "wilsonPrecisionLower95": 0.8852 - }, - "threshold": 0.855 - }, - "question": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8505, - "falseNegative": 76, - "falsePositive": 8, - "precision": 0.9676, - "predictedPositive": 247, - "recall": 0.7587, - "total": 1485, - "trueNegative": 1162, - "truePositive": 239, - "wilsonPrecisionLower95": 0.9374 - }, - "threshold": 0.705 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7308, - "falseNegative": 129, - "falsePositive": 8, - "precision": 0.9588, - "predictedPositive": 194, - "recall": 0.5905, - "total": 1485, - "trueNegative": 1162, - "truePositive": 186, - "wilsonPrecisionLower95": 0.9208 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.7813, - "falseNegative": 212, - "falsePositive": 22, - "precision": 0.95, - "predictedPositive": 440, - "recall": 0.6635, - "total": 2970, - "trueNegative": 2318, - "truePositive": 418, - "wilsonPrecisionLower95": 0.9255 - }, - "threshold": 0.57 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9629, - "falseNegative": 23, - "falsePositive": 48, - "precision": 0.9505, - "predictedPositive": 970, - "recall": 0.9757, - "total": 1485, - "trueNegative": 492, - "truePositive": 922, - "wilsonPrecisionLower95": 0.935 - }, - "threshold": 0.62 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8526, - "falseNegative": 216, - "falsePositive": 36, - "precision": 0.9529, - "predictedPositive": 765, - "recall": 0.7714, - "total": 1485, - "trueNegative": 504, - "truePositive": 729, - "wilsonPrecisionLower95": 0.9355 - }, - "threshold": 0.84 - } - }, - "metrics": { - "f1": 0.9224, - "falseNegative": 197, - "falsePositive": 88, - "precision": 0.9506, - "predictedPositive": 1781, - "recall": 0.8958, - "total": 2970, - "trueNegative": 992, - "truePositive": 1693, - "wilsonPrecisionLower95": 0.9395 - }, - "threshold": 0.775 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9783, - "falseNegative": 0, - "falsePositive": 2, - "precision": 0.9574, - "predictedPositive": 47, - "recall": 1.0, - "total": 1485, - "trueNegative": 1438, - "truePositive": 45, - "wilsonPrecisionLower95": 0.8575 - }, - "threshold": 0.765 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9778, - "falseNegative": 1, - "falsePositive": 1, - "precision": 0.9778, - "predictedPositive": 45, - "recall": 0.9778, - "total": 1485, - "trueNegative": 1439, - "truePositive": 44, - "wilsonPrecisionLower95": 0.8843 - }, - "threshold": 0.855 - } - }, - "metrics": { - "f1": 0.978, - "falseNegative": 1, - "falsePositive": 3, - "precision": 0.9674, - "predictedPositive": 92, - "recall": 0.9889, - "total": 2970, - "trueNegative": 2877, - "truePositive": 89, - "wilsonPrecisionLower95": 0.9085 - }, - "threshold": 0.855 - }, - "task": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8541, - "falseNegative": 82, - "falsePositive": 13, - "precision": 0.9553, - "predictedPositive": 291, - "recall": 0.7722, - "total": 1485, - "trueNegative": 1112, - "truePositive": 278, - "wilsonPrecisionLower95": 0.9251 - }, - "threshold": 0.84 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.5418, - "falseNegative": 224, - "falsePositive": 6, - "precision": 0.9577, - "predictedPositive": 142, - "recall": 0.3778, - "total": 1485, - "trueNegative": 1119, - "truePositive": 136, - "wilsonPrecisionLower95": 0.9109 - }, - "threshold": 0.91 - } - }, - "metrics": { - "f1": 0.7246, - "falseNegative": 299, - "falsePositive": 21, - "precision": 0.9525, - "predictedPositive": 442, - "recall": 0.5847, - "total": 2970, - "trueNegative": 2229, - "truePositive": 421, - "wilsonPrecisionLower95": 0.9285 - }, - "threshold": 0.88 - } - }, - "trainingRecords": 15345, - "validation": { - "intentsMeeting95Precision": 9, - "macroF1": 0.5203, - "macroPrecision": 0.9875, - "macroRecall": 0.3893, - "minimumPrecision": 0.9516, - "perIntent": { - "blessing": { - "f1": 0.2524, - "falseNegative": 77, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 13, - "recall": 0.1444, - "total": 2970, - "trueNegative": 2880, - "truePositive": 13, - "wilsonPrecisionLower95": 0.7719 - }, - "complaint": { - "f1": 0.5455, - "falseNegative": 225, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 135, - "recall": 0.375, - "total": 2970, - "trueNegative": 2610, - "truePositive": 135, - "wilsonPrecisionLower95": 0.9723 - }, - "confirmationDecision": { - "f1": 0.6312, - "falseNegative": 97, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 83, - "recall": 0.4611, - "total": 2970, - "trueNegative": 2790, - "truePositive": 83, - "wilsonPrecisionLower95": 0.9558 - }, - "followUpReminder": { - "f1": 0.2476, - "falseNegative": 309, - "falsePositive": 1, - "precision": 0.9808, - "predictedPositive": 52, - "recall": 0.1417, - "total": 2970, - "trueNegative": 2609, - "truePositive": 51, - "wilsonPrecisionLower95": 0.8988 - }, - "invitation": { - "f1": 0.3019, - "falseNegative": 148, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 32, - "recall": 0.1778, - "total": 2970, - "trueNegative": 2790, - "truePositive": 32, - "wilsonPrecisionLower95": 0.8928 - }, - "question": { - "f1": 0.7851, - "falseNegative": 219, - "falsePositive": 6, - "precision": 0.9856, - "predictedPositive": 417, - "recall": 0.6524, - "total": 2970, - "trueNegative": 2334, - "truePositive": 411, - "wilsonPrecisionLower95": 0.969 - }, - "replyableMessage": { - "f1": 0.9109, - "falseNegative": 239, - "falsePositive": 84, - "precision": 0.9516, - "predictedPositive": 1735, - "recall": 0.8735, - "total": 2970, - "trueNegative": 996, - "truePositive": 1651, - "wilsonPrecisionLower95": 0.9404 - }, - "scheduleNegotiation": { - "f1": 0.5203, - "falseNegative": 58, - "falsePositive": 1, - "precision": 0.9697, - "predictedPositive": 33, - "recall": 0.3556, - "total": 2970, - "trueNegative": 2879, - "truePositive": 32, - "wilsonPrecisionLower95": 0.8468 - }, - "task": { - "f1": 0.4874, - "falseNegative": 488, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 232, - "recall": 0.3222, - "total": 2970, - "trueNegative": 2250, - "truePositive": 232, - "wilsonPrecisionLower95": 0.9837 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 1e-05, - "augmentation": "none", - "char_max": 6, - "char_min": 2, - "external_weight": 0.6100000000000001, - "hard_example_weight": 1.2, - "l1_ratio": 0.05, - "max_char_features": 55000, - "max_word_features": 25000, - "min_df": 2, - "pseudo_weight": 0.195, - "round": 3, - "word_max": 2 - }, - "featureCount": 66451, - "pseudoLabelCounts": {}, - "round": 3, - "selectionObjective": 0.6612, - "syntheticRegression": { - "intentsMeeting95Precision": 7, - "macroF1": 0.5557, - "macroPrecision": 0.9751, - "macroRecall": 0.4386, - "minimumPrecision": 0.8115, - "perIntent": { - "blessing": { - "f1": 0.0645, - "falseNegative": 87, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 3, - "recall": 0.0333, - "total": 2970, - "trueNegative": 2880, - "truePositive": 3, - "wilsonPrecisionLower95": 0.4385 - }, - "complaint": { - "f1": 0.4176, - "falseNegative": 265, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 95, - "recall": 0.2639, - "total": 2970, - "trueNegative": 2610, - "truePositive": 95, - "wilsonPrecisionLower95": 0.9611 - }, - "confirmationDecision": { - "f1": 0.6556, - "falseNegative": 81, - "falsePositive": 23, - "precision": 0.8115, - "predictedPositive": 122, - "recall": 0.55, - "total": 2970, - "trueNegative": 2767, - "truePositive": 99, - "wilsonPrecisionLower95": 0.733 - }, - "followUpReminder": { - "f1": 0.4211, - "falseNegative": 264, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 96, - "recall": 0.2667, - "total": 2970, - "trueNegative": 2610, - "truePositive": 96, - "wilsonPrecisionLower95": 0.9615 - }, - "invitation": { - "f1": 0.4071, - "falseNegative": 134, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 46, - "recall": 0.2556, - "total": 2970, - "trueNegative": 2790, - "truePositive": 46, - "wilsonPrecisionLower95": 0.9229 - }, - "question": { - "f1": 0.8408, - "falseNegative": 173, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 457, - "recall": 0.7254, - "total": 2970, - "trueNegative": 2340, - "truePositive": 457, - "wilsonPrecisionLower95": 0.9917 - }, - "replyableMessage": { - "f1": 0.931, - "falseNegative": 189, - "falsePositive": 63, - "precision": 0.9643, - "predictedPositive": 1764, - "recall": 0.9, - "total": 2970, - "trueNegative": 1017, - "truePositive": 1701, - "wilsonPrecisionLower95": 0.9546 - }, - "scheduleNegotiation": { - "f1": 0.7671, - "falseNegative": 34, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 56, - "recall": 0.6222, - "total": 2970, - "trueNegative": 2880, - "truePositive": 56, - "wilsonPrecisionLower95": 0.9358 - }, - "task": { - "f1": 0.4969, - "falseNegative": 482, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 238, - "recall": 0.3306, - "total": 2970, - "trueNegative": 2250, - "truePositive": 238, - "wilsonPrecisionLower95": 0.9841 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 180, - "recall": 1.0, - "total": 1485, - "trueNegative": 1305, - "truePositive": 180, - "wilsonPrecisionLower95": 0.9791 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9371, - "falseNegative": 16, - "falsePositive": 6, - "precision": 0.9647, - "predictedPositive": 170, - "recall": 0.9111, - "total": 1485, - "trueNegative": 1299, - "truePositive": 164, - "wilsonPrecisionLower95": 0.9251 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.969, - "falseNegative": 16, - "falsePositive": 6, - "precision": 0.9829, - "predictedPositive": 350, - "recall": 0.9556, - "total": 2970, - "trueNegative": 2604, - "truePositive": 344, - "wilsonPrecisionLower95": 0.9631 - }, - "threshold": 0.5 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9727, - "falseNegative": 1, - "falsePositive": 4, - "precision": 0.957, - "predictedPositive": 93, - "recall": 0.9889, - "total": 1485, - "trueNegative": 1391, - "truePositive": 89, - "wilsonPrecisionLower95": 0.8946 - }, - "threshold": 0.585 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7815, - "falseNegative": 31, - "falsePositive": 2, - "precision": 0.9672, - "predictedPositive": 61, - "recall": 0.6556, - "total": 1485, - "trueNegative": 1393, - "truePositive": 59, - "wilsonPrecisionLower95": 0.8881 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.8701, - "falseNegative": 36, - "falsePositive": 7, - "precision": 0.9536, - "predictedPositive": 151, - "recall": 0.8, - "total": 2970, - "trueNegative": 2783, - "truePositive": 144, - "wilsonPrecisionLower95": 0.9074 - }, - "threshold": 0.555 - }, - "followUpReminder": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8454, - "falseNegative": 46, - "falsePositive": 3, - "precision": 0.9781, - "predictedPositive": 137, - "recall": 0.7444, - "total": 1485, - "trueNegative": 1302, - "truePositive": 134, - "wilsonPrecisionLower95": 0.9376 - }, - "threshold": 0.97 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7525, - "falseNegative": 69, - "falsePositive": 4, - "precision": 0.9652, - "predictedPositive": 115, - "recall": 0.6167, - "total": 1485, - "trueNegative": 1301, - "truePositive": 111, - "wilsonPrecisionLower95": 0.914 - }, - "threshold": 0.785 - } - }, - "metrics": { - "f1": 0.703, - "falseNegative": 160, - "falsePositive": 9, - "precision": 0.9569, - "predictedPositive": 209, - "recall": 0.5556, - "total": 2970, - "trueNegative": 2601, - "truePositive": 200, - "wilsonPrecisionLower95": 0.9202 - }, - "threshold": 0.955 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7974, - "falseNegative": 29, - "falsePositive": 2, - "precision": 0.9683, - "predictedPositive": 63, - "recall": 0.6778, - "total": 1485, - "trueNegative": 1393, - "truePositive": 61, - "wilsonPrecisionLower95": 0.8914 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8259, - "falseNegative": 85, - "falsePositive": 12, - "precision": 0.9504, - "predictedPositive": 242, - "recall": 0.7302, - "total": 1485, - "trueNegative": 1158, - "truePositive": 230, - "wilsonPrecisionLower95": 0.9153 - }, - "threshold": 0.865 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8194, - "falseNegative": 95, - "falsePositive": 2, - "precision": 0.991, - "predictedPositive": 222, - "recall": 0.6984, - "total": 1485, - "trueNegative": 1168, - "truePositive": 220, - "wilsonPrecisionLower95": 0.9678 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.7959, - "falseNegative": 199, - "falsePositive": 22, - "precision": 0.9514, - "predictedPositive": 453, - "recall": 0.6841, - "total": 2970, - "trueNegative": 2318, - "truePositive": 431, - "wilsonPrecisionLower95": 0.9276 - }, - "threshold": 0.74 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9496, - "falseNegative": 50, - "falsePositive": 45, - "precision": 0.9521, - "predictedPositive": 940, - "recall": 0.9471, - "total": 1485, - "trueNegative": 495, - "truePositive": 895, - "wilsonPrecisionLower95": 0.9365 - }, - "threshold": 0.705 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9209, - "falseNegative": 101, - "falsePositive": 44, - "precision": 0.9505, - "predictedPositive": 888, - "recall": 0.8931, - "total": 1485, - "trueNegative": 496, - "truePositive": 844, - "wilsonPrecisionLower95": 0.9341 - }, - "threshold": 0.78 - } - }, - "metrics": { - "f1": 0.9318, - "falseNegative": 163, - "falsePositive": 90, - "precision": 0.9505, - "predictedPositive": 1817, - "recall": 0.9138, - "total": 2970, - "trueNegative": 990, - "truePositive": 1727, - "wilsonPrecisionLower95": 0.9395 - }, - "threshold": 0.77 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 45, - "recall": 1.0, - "total": 1485, - "trueNegative": 1440, - "truePositive": 45, - "wilsonPrecisionLower95": 0.9213 - }, - "threshold": 0.745 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9783, - "falseNegative": 0, - "falsePositive": 2, - "precision": 0.9574, - "predictedPositive": 47, - "recall": 1.0, - "total": 1485, - "trueNegative": 1438, - "truePositive": 45, - "wilsonPrecisionLower95": 0.8575 - }, - "threshold": 0.93 - } - }, - "metrics": { - "f1": 0.9778, - "falseNegative": 2, - "falsePositive": 2, - "precision": 0.9778, - "predictedPositive": 90, - "recall": 0.9778, - "total": 2970, - "trueNegative": 2878, - "truePositive": 88, - "wilsonPrecisionLower95": 0.9226 - }, - "threshold": 0.93 - }, - "task": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8488, - "falseNegative": 85, - "falsePositive": 13, - "precision": 0.9549, - "predictedPositive": 288, - "recall": 0.7639, - "total": 1485, - "trueNegative": 1112, - "truePositive": 275, - "wilsonPrecisionLower95": 0.9243 - }, - "threshold": 0.865 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8006, - "falseNegative": 111, - "falsePositive": 13, - "precision": 0.9504, - "predictedPositive": 262, - "recall": 0.6917, - "total": 1485, - "trueNegative": 1112, - "truePositive": 249, - "wilsonPrecisionLower95": 0.917 - }, - "threshold": 0.8 - } - }, - "metrics": { - "f1": 0.8301, - "falseNegative": 190, - "falsePositive": 27, - "precision": 0.9515, - "predictedPositive": 557, - "recall": 0.7361, - "total": 2970, - "trueNegative": 2223, - "truePositive": 530, - "wilsonPrecisionLower95": 0.9304 - }, - "threshold": 0.82 - } - }, - "trainingRecords": 15345, - "validation": { - "intentsMeeting95Precision": 6, - "macroF1": 0.5094, - "macroPrecision": 0.9758, - "macroRecall": 0.3953, - "minimumPrecision": 0.9412, - "perIntent": { - "blessing": { - "f1": 0.1443, - "falseNegative": 83, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 7, - "recall": 0.0778, - "total": 2970, - "trueNegative": 2880, - "truePositive": 7, - "wilsonPrecisionLower95": 0.6457 - }, - "complaint": { - "f1": 0.5455, - "falseNegative": 225, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 135, - "recall": 0.375, - "total": 2970, - "trueNegative": 2610, - "truePositive": 135, - "wilsonPrecisionLower95": 0.9723 - }, - "confirmationDecision": { - "f1": 0.6617, - "falseNegative": 91, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 89, - "recall": 0.4944, - "total": 2970, - "trueNegative": 2790, - "truePositive": 89, - "wilsonPrecisionLower95": 0.9586 - }, - "followUpReminder": { - "f1": 0.2464, - "falseNegative": 309, - "falsePositive": 3, - "precision": 0.9444, - "predictedPositive": 54, - "recall": 0.1417, - "total": 2970, - "trueNegative": 2607, - "truePositive": 51, - "wilsonPrecisionLower95": 0.8489 - }, - "invitation": { - "f1": 0.1347, - "falseNegative": 167, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 13, - "recall": 0.0722, - "total": 2970, - "trueNegative": 2790, - "truePositive": 13, - "wilsonPrecisionLower95": 0.7719 - }, - "question": { - "f1": 0.8158, - "falseNegative": 187, - "falsePositive": 13, - "precision": 0.9715, - "predictedPositive": 456, - "recall": 0.7032, - "total": 2970, - "trueNegative": 2327, - "truePositive": 443, - "wilsonPrecisionLower95": 0.9518 - }, - "replyableMessage": { - "f1": 0.9354, - "falseNegative": 151, - "falsePositive": 89, - "precision": 0.9513, - "predictedPositive": 1828, - "recall": 0.9201, - "total": 2970, - "trueNegative": 991, - "truePositive": 1739, - "wilsonPrecisionLower95": 0.9405 - }, - "scheduleNegotiation": { - "f1": 0.5161, - "falseNegative": 58, - "falsePositive": 2, - "precision": 0.9412, - "predictedPositive": 34, - "recall": 0.3556, - "total": 2970, - "trueNegative": 2878, - "truePositive": 32, - "wilsonPrecisionLower95": 0.8091 - }, - "task": { - "f1": 0.585, - "falseNegative": 419, - "falsePositive": 8, - "precision": 0.9741, - "predictedPositive": 309, - "recall": 0.4181, - "total": 2970, - "trueNegative": 2242, - "truePositive": 301, - "wilsonPrecisionLower95": 0.9498 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 8e-06, - "augmentation": "none", - "char_max": 6, - "char_min": 3, - "external_weight": 0.63, - "hard_example_weight": 1.2, - "l1_ratio": 0.05, - "max_char_features": 60000, - "max_word_features": 30000, - "min_df": 2, - "pseudo_weight": 0.21, - "round": 4, - "word_max": 3 - }, - "featureCount": 79458, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 0 - }, - "complaint": { - "negative": 97, - "positive": 0 - }, - "confirmationDecision": { - "negative": 51, - "positive": 2 - }, - "followUpReminder": { - "negative": 250, - "positive": 0 - }, - "invitation": { - "negative": 250, - "positive": 0 - }, - "question": { - "negative": 36, - "positive": 1 - }, - "replyableMessage": { - "negative": 3, - "positive": 130 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 0 - }, - "task": { - "negative": 85, - "positive": 0 - } - }, - "round": 4, - "selectionObjective": 0.6534, - "syntheticRegression": { - "intentsMeeting95Precision": 7, - "macroF1": 0.5001, - "macroPrecision": 0.9696, - "macroRecall": 0.3844, - "minimumPrecision": 0.7742, - "perIntent": { - "blessing": { - "f1": 0.0645, - "falseNegative": 87, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 3, - "recall": 0.0333, - "total": 2970, - "trueNegative": 2880, - "truePositive": 3, - "wilsonPrecisionLower95": 0.4385 - }, - "complaint": { - "f1": 0.4211, - "falseNegative": 264, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 96, - "recall": 0.2667, - "total": 2970, - "trueNegative": 2610, - "truePositive": 96, - "wilsonPrecisionLower95": 0.9615 - }, - "confirmationDecision": { - "f1": 0.5275, - "falseNegative": 108, - "falsePositive": 21, - "precision": 0.7742, - "predictedPositive": 93, - "recall": 0.4, - "total": 2970, - "trueNegative": 2769, - "truePositive": 72, - "wilsonPrecisionLower95": 0.6793 - }, - "followUpReminder": { - "f1": 0.4071, - "falseNegative": 268, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 92, - "recall": 0.2556, - "total": 2970, - "trueNegative": 2610, - "truePositive": 92, - "wilsonPrecisionLower95": 0.9599 - }, - "invitation": { - "f1": 0.2, - "falseNegative": 160, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 20, - "recall": 0.1111, - "total": 2970, - "trueNegative": 2790, - "truePositive": 20, - "wilsonPrecisionLower95": 0.8389 - }, - "question": { - "f1": 0.7349, - "falseNegative": 264, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 366, - "recall": 0.581, - "total": 2970, - "trueNegative": 2340, - "truePositive": 366, - "wilsonPrecisionLower95": 0.9896 - }, - "replyableMessage": { - "f1": 0.8816, - "falseNegative": 353, - "falsePositive": 60, - "precision": 0.9624, - "predictedPositive": 1597, - "recall": 0.8132, - "total": 2970, - "trueNegative": 1020, - "truePositive": 1537, - "wilsonPrecisionLower95": 0.9519 - }, - "scheduleNegotiation": { - "f1": 0.8462, - "falseNegative": 24, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 66, - "recall": 0.7333, - "total": 2970, - "trueNegative": 2880, - "truePositive": 66, - "wilsonPrecisionLower95": 0.945 - }, - "task": { - "f1": 0.4184, - "falseNegative": 529, - "falsePositive": 2, - "precision": 0.9896, - "predictedPositive": 193, - "recall": 0.2653, - "total": 2970, - "trueNegative": 2248, - "truePositive": 191, - "wilsonPrecisionLower95": 0.963 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9972, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 179, - "recall": 0.9944, - "total": 1485, - "trueNegative": 1305, - "truePositive": 179, - "wilsonPrecisionLower95": 0.979 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9253, - "falseNegative": 19, - "falsePositive": 7, - "precision": 0.9583, - "predictedPositive": 168, - "recall": 0.8944, - "total": 1485, - "trueNegative": 1298, - "truePositive": 161, - "wilsonPrecisionLower95": 0.9165 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.9618, - "falseNegative": 20, - "falsePositive": 7, - "precision": 0.9798, - "predictedPositive": 347, - "recall": 0.9444, - "total": 2970, - "trueNegative": 2603, - "truePositive": 340, - "wilsonPrecisionLower95": 0.959 - }, - "threshold": 0.5 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8765, - "falseNegative": 19, - "falsePositive": 1, - "precision": 0.9861, - "predictedPositive": 72, - "recall": 0.7889, - "total": 1485, - "trueNegative": 1394, - "truePositive": 71, - "wilsonPrecisionLower95": 0.9254 - }, - "threshold": 0.945 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6569, - "falseNegative": 45, - "falsePositive": 2, - "precision": 0.9574, - "predictedPositive": 47, - "recall": 0.5, - "total": 1485, - "trueNegative": 1393, - "truePositive": 45, - "wilsonPrecisionLower95": 0.8575 - }, - "threshold": 0.64 - } - }, - "metrics": { - "f1": 0.7108, - "falseNegative": 78, - "falsePositive": 5, - "precision": 0.9533, - "predictedPositive": 107, - "recall": 0.5667, - "total": 2970, - "trueNegative": 2785, - "truePositive": 102, - "wilsonPrecisionLower95": 0.8952 - }, - "threshold": 0.91 - }, - "followUpReminder": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8, - "falseNegative": 56, - "falsePositive": 6, - "precision": 0.9538, - "predictedPositive": 130, - "recall": 0.6889, - "total": 1485, - "trueNegative": 1299, - "truePositive": 124, - "wilsonPrecisionLower95": 0.903 - }, - "threshold": 0.98 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6165, - "falseNegative": 98, - "falsePositive": 4, - "precision": 0.9535, - "predictedPositive": 86, - "recall": 0.4556, - "total": 1485, - "trueNegative": 1301, - "truePositive": 82, - "wilsonPrecisionLower95": 0.8864 - }, - "threshold": 0.825 - } - }, - "metrics": { - "f1": 0.5912, - "falseNegative": 206, - "falsePositive": 7, - "precision": 0.9565, - "predictedPositive": 161, - "recall": 0.4278, - "total": 2970, - "trueNegative": 2603, - "truePositive": 154, - "wilsonPrecisionLower95": 0.913 - }, - "threshold": 0.98 - }, - "invitation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8302, - "falseNegative": 24, - "falsePositive": 3, - "precision": 0.9565, - "predictedPositive": 69, - "recall": 0.7333, - "total": 1485, - "trueNegative": 1392, - "truePositive": 66, - "wilsonPrecisionLower95": 0.8798 - }, - "threshold": 0.98 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.536, - "falseNegative": 113, - "falsePositive": 3, - "precision": 0.9571, - "predictedPositive": 70, - "recall": 0.3722, - "total": 2970, - "trueNegative": 2787, - "truePositive": 67, - "wilsonPrecisionLower95": 0.8814 - }, - "threshold": 0.995 - }, - "question": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.7813, - "falseNegative": 106, - "falsePositive": 11, - "precision": 0.95, - "predictedPositive": 220, - "recall": 0.6635, - "total": 1485, - "trueNegative": 1159, - "truePositive": 209, - "wilsonPrecisionLower95": 0.9127 - }, - "threshold": 0.865 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6844, - "falseNegative": 148, - "falsePositive": 6, - "precision": 0.9653, - "predictedPositive": 173, - "recall": 0.5302, - "total": 1485, - "trueNegative": 1164, - "truePositive": 167, - "wilsonPrecisionLower95": 0.9264 - }, - "threshold": 0.63 - } - }, - "metrics": { - "f1": 0.75, - "falseNegative": 240, - "falsePositive": 20, - "precision": 0.9512, - "predictedPositive": 410, - "recall": 0.619, - "total": 2970, - "trueNegative": 2320, - "truePositive": 390, - "wilsonPrecisionLower95": 0.9259 - }, - "threshold": 0.74 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9441, - "falseNegative": 59, - "falsePositive": 46, - "precision": 0.9506, - "predictedPositive": 932, - "recall": 0.9376, - "total": 1485, - "trueNegative": 494, - "truePositive": 886, - "wilsonPrecisionLower95": 0.9348 - }, - "threshold": 0.76 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8342, - "falseNegative": 243, - "falsePositive": 36, - "precision": 0.9512, - "predictedPositive": 738, - "recall": 0.7429, - "total": 1485, - "trueNegative": 504, - "truePositive": 702, - "wilsonPrecisionLower95": 0.9332 - }, - "threshold": 0.885 - } - }, - "metrics": { - "f1": 0.9056, - "falseNegative": 255, - "falsePositive": 86, - "precision": 0.95, - "predictedPositive": 1721, - "recall": 0.8651, - "total": 2970, - "trueNegative": 994, - "truePositive": 1635, - "wilsonPrecisionLower95": 0.9387 - }, - "threshold": 0.855 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 45, - "recall": 1.0, - "total": 1485, - "trueNegative": 1440, - "truePositive": 45, - "wilsonPrecisionLower95": 0.9213 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9318, - "falseNegative": 4, - "falsePositive": 2, - "precision": 0.9535, - "predictedPositive": 43, - "recall": 0.9111, - "total": 1485, - "trueNegative": 1438, - "truePositive": 41, - "wilsonPrecisionLower95": 0.8454 - }, - "threshold": 0.945 - } - }, - "metrics": { - "f1": 0.9727, - "falseNegative": 1, - "falsePositive": 4, - "precision": 0.957, - "predictedPositive": 93, - "recall": 0.9889, - "total": 2970, - "trueNegative": 2876, - "truePositive": 89, - "wilsonPrecisionLower95": 0.8946 - }, - "threshold": 0.62 - }, - "task": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.851, - "falseNegative": 83, - "falsePositive": 14, - "precision": 0.9519, - "predictedPositive": 291, - "recall": 0.7694, - "total": 1485, - "trueNegative": 1111, - "truePositive": 277, - "wilsonPrecisionLower95": 0.9209 - }, - "threshold": 0.87 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.4886, - "falseNegative": 242, - "falsePositive": 5, - "precision": 0.9593, - "predictedPositive": 123, - "recall": 0.3278, - "total": 1485, - "trueNegative": 1120, - "truePositive": 118, - "wilsonPrecisionLower95": 0.9084 - }, - "threshold": 0.935 - } - }, - "metrics": { - "f1": 0.6978, - "falseNegative": 324, - "falsePositive": 19, - "precision": 0.9542, - "predictedPositive": 415, - "recall": 0.55, - "total": 2970, - "trueNegative": 2231, - "truePositive": 396, - "wilsonPrecisionLower95": 0.9296 - }, - "threshold": 0.915 - } - }, - "trainingRecords": 15345, - "validation": { - "intentsMeeting95Precision": 8, - "macroF1": 0.4964, - "macroPrecision": 0.9825, - "macroRecall": 0.3647, - "minimumPrecision": 0.9412, - "perIntent": { - "blessing": { - "f1": 0.2, - "falseNegative": 80, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 10, - "recall": 0.1111, - "total": 2970, - "trueNegative": 2880, - "truePositive": 10, - "wilsonPrecisionLower95": 0.7225 - }, - "complaint": { - "f1": 0.5425, - "falseNegative": 226, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 134, - "recall": 0.3722, - "total": 2970, - "trueNegative": 2610, - "truePositive": 134, - "wilsonPrecisionLower95": 0.9721 - }, - "confirmationDecision": { - "f1": 0.5542, - "falseNegative": 111, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 69, - "recall": 0.3833, - "total": 2970, - "trueNegative": 2790, - "truePositive": 69, - "wilsonPrecisionLower95": 0.9473 - }, - "followUpReminder": { - "f1": 0.2602, - "falseNegative": 306, - "falsePositive": 1, - "precision": 0.9818, - "predictedPositive": 55, - "recall": 0.15, - "total": 2970, - "trueNegative": 2609, - "truePositive": 54, - "wilsonPrecisionLower95": 0.9039 - }, - "invitation": { - "f1": 0.3019, - "falseNegative": 148, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 32, - "recall": 0.1778, - "total": 2970, - "trueNegative": 2790, - "truePositive": 32, - "wilsonPrecisionLower95": 0.8928 - }, - "question": { - "f1": 0.7302, - "falseNegative": 262, - "falsePositive": 10, - "precision": 0.9735, - "predictedPositive": 378, - "recall": 0.5841, - "total": 2970, - "trueNegative": 2330, - "truePositive": 368, - "wilsonPrecisionLower95": 0.952 - }, - "replyableMessage": { - "f1": 0.8921, - "falseNegative": 302, - "falsePositive": 82, - "precision": 0.9509, - "predictedPositive": 1670, - "recall": 0.8402, - "total": 2970, - "trueNegative": 998, - "truePositive": 1588, - "wilsonPrecisionLower95": 0.9395 - }, - "scheduleNegotiation": { - "f1": 0.5161, - "falseNegative": 58, - "falsePositive": 2, - "precision": 0.9412, - "predictedPositive": 34, - "recall": 0.3556, - "total": 2970, - "trueNegative": 2878, - "truePositive": 32, - "wilsonPrecisionLower95": 0.8091 - }, - "task": { - "f1": 0.4708, - "falseNegative": 498, - "falsePositive": 1, - "precision": 0.9955, - "predictedPositive": 223, - "recall": 0.3083, - "total": 2970, - "trueNegative": 2249, - "truePositive": 222, - "wilsonPrecisionLower95": 0.975 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 1.2e-05, - "augmentation": "none", - "char_max": 5, - "char_min": 2, - "external_weight": 0.65, - "hard_example_weight": 1.4, - "l1_ratio": 0.1, - "max_char_features": 60000, - "max_word_features": 32000, - "min_df": 1, - "pseudo_weight": 0.22499999999999998, - "round": 5, - "word_max": 3 - }, - "featureCount": 92000, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 0 - }, - "complaint": { - "negative": 88, - "positive": 0 - }, - "confirmationDecision": { - "negative": 96, - "positive": 3 - }, - "followUpReminder": { - "negative": 250, - "positive": 0 - }, - "invitation": { - "negative": 250, - "positive": 0 - }, - "question": { - "negative": 40, - "positive": 1 - }, - "replyableMessage": { - "negative": 3, - "positive": 150 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 0 - }, - "task": { - "negative": 97, - "positive": 0 - } - }, - "round": 5, - "selectionObjective": 0.6361, - "syntheticRegression": { - "intentsMeeting95Precision": 7, - "macroF1": 0.5097, - "macroPrecision": 0.9762, - "macroRecall": 0.3951, - "minimumPrecision": 0.8197, - "perIntent": { - "blessing": { - "f1": 0.022, - "falseNegative": 89, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 1, - "recall": 0.0111, - "total": 2970, - "trueNegative": 2880, - "truePositive": 1, - "wilsonPrecisionLower95": 0.2065 - }, - "complaint": { - "f1": 0.4176, - "falseNegative": 265, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 95, - "recall": 0.2639, - "total": 2970, - "trueNegative": 2610, - "truePositive": 95, - "wilsonPrecisionLower95": 0.9611 - }, - "confirmationDecision": { - "f1": 0.6623, - "falseNegative": 80, - "falsePositive": 22, - "precision": 0.8197, - "predictedPositive": 122, - "recall": 0.5556, - "total": 2970, - "trueNegative": 2768, - "truePositive": 100, - "wilsonPrecisionLower95": 0.742 - }, - "followUpReminder": { - "f1": 0.341, - "falseNegative": 286, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 74, - "recall": 0.2056, - "total": 2970, - "trueNegative": 2610, - "truePositive": 74, - "wilsonPrecisionLower95": 0.9507 - }, - "invitation": { - "f1": 0.2609, - "falseNegative": 153, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 27, - "recall": 0.15, - "total": 2970, - "trueNegative": 2790, - "truePositive": 27, - "wilsonPrecisionLower95": 0.8754 - }, - "question": { - "f1": 0.8147, - "falseNegative": 197, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 433, - "recall": 0.6873, - "total": 2970, - "trueNegative": 2340, - "truePositive": 433, - "wilsonPrecisionLower95": 0.9912 - }, - "replyableMessage": { - "f1": 0.912, - "falseNegative": 257, - "falsePositive": 58, - "precision": 0.9657, - "predictedPositive": 1691, - "recall": 0.864, - "total": 2970, - "trueNegative": 1022, - "truePositive": 1633, - "wilsonPrecisionLower95": 0.9559 - }, - "scheduleNegotiation": { - "f1": 0.6364, - "falseNegative": 48, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 42, - "recall": 0.4667, - "total": 2970, - "trueNegative": 2880, - "truePositive": 42, - "wilsonPrecisionLower95": 0.9162 - }, - "task": { - "f1": 0.52, - "falseNegative": 467, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 253, - "recall": 0.3514, - "total": 2970, - "trueNegative": 2250, - "truePositive": 253, - "wilsonPrecisionLower95": 0.985 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 180, - "recall": 1.0, - "total": 1485, - "trueNegative": 1305, - "truePositive": 180, - "wilsonPrecisionLower95": 0.9791 - }, - "threshold": 0.69 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9288, - "falseNegative": 17, - "falsePositive": 8, - "precision": 0.9532, - "predictedPositive": 171, - "recall": 0.9056, - "total": 1485, - "trueNegative": 1297, - "truePositive": 163, - "wilsonPrecisionLower95": 0.9104 - }, - "threshold": 0.51 - } - }, - "metrics": { - "f1": 0.9569, - "falseNegative": 16, - "falsePositive": 15, - "precision": 0.9582, - "predictedPositive": 359, - "recall": 0.9556, - "total": 2970, - "trueNegative": 2595, - "truePositive": 344, - "wilsonPrecisionLower95": 0.9322 - }, - "threshold": 0.5 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8589, - "falseNegative": 20, - "falsePositive": 3, - "precision": 0.9589, - "predictedPositive": 73, - "recall": 0.7778, - "total": 1485, - "trueNegative": 1392, - "truePositive": 70, - "wilsonPrecisionLower95": 0.886 - }, - "threshold": 0.57 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6763, - "falseNegative": 43, - "falsePositive": 2, - "precision": 0.9592, - "predictedPositive": 49, - "recall": 0.5222, - "total": 1485, - "trueNegative": 1393, - "truePositive": 47, - "wilsonPrecisionLower95": 0.8629 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.76, - "falseNegative": 66, - "falsePositive": 6, - "precision": 0.95, - "predictedPositive": 120, - "recall": 0.6333, - "total": 2970, - "trueNegative": 2784, - "truePositive": 114, - "wilsonPrecisionLower95": 0.8952 - }, - "threshold": 0.56 - }, - "followUpReminder": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8, - "falseNegative": 56, - "falsePositive": 6, - "precision": 0.9538, - "predictedPositive": 130, - "recall": 0.6889, - "total": 1485, - "trueNegative": 1299, - "truePositive": 124, - "wilsonPrecisionLower95": 0.903 - }, - "threshold": 0.96 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7708, - "falseNegative": 64, - "falsePositive": 5, - "precision": 0.9587, - "predictedPositive": 121, - "recall": 0.6444, - "total": 1485, - "trueNegative": 1300, - "truePositive": 116, - "wilsonPrecisionLower95": 0.9069 - }, - "threshold": 0.58 - } - }, - "metrics": { - "f1": 0.6892, - "falseNegative": 166, - "falsePositive": 9, - "precision": 0.9557, - "predictedPositive": 203, - "recall": 0.5389, - "total": 2970, - "trueNegative": 2601, - "truePositive": 194, - "wilsonPrecisionLower95": 0.9179 - }, - "threshold": 0.915 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.5203, - "falseNegative": 58, - "falsePositive": 1, - "precision": 0.9697, - "predictedPositive": 33, - "recall": 0.3556, - "total": 1485, - "trueNegative": 1394, - "truePositive": 32, - "wilsonPrecisionLower95": 0.8468 - }, - "threshold": 0.88 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8225, - "falseNegative": 88, - "falsePositive": 10, - "precision": 0.9578, - "predictedPositive": 237, - "recall": 0.7206, - "total": 1485, - "trueNegative": 1160, - "truePositive": 227, - "wilsonPrecisionLower95": 0.9241 - }, - "threshold": 0.84 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8388, - "falseNegative": 86, - "falsePositive": 2, - "precision": 0.9913, - "predictedPositive": 231, - "recall": 0.727, - "total": 1485, - "trueNegative": 1168, - "truePositive": 229, - "wilsonPrecisionLower95": 0.969 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.8591, - "falseNegative": 136, - "falsePositive": 26, - "precision": 0.95, - "predictedPositive": 520, - "recall": 0.7841, - "total": 2970, - "trueNegative": 2314, - "truePositive": 494, - "wilsonPrecisionLower95": 0.9277 - }, - "threshold": 0.505 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9541, - "falseNegative": 40, - "falsePositive": 47, - "precision": 0.9506, - "predictedPositive": 952, - "recall": 0.9577, - "total": 1485, - "trueNegative": 493, - "truePositive": 905, - "wilsonPrecisionLower95": 0.935 - }, - "threshold": 0.69 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8807, - "falseNegative": 170, - "falsePositive": 40, - "precision": 0.9509, - "predictedPositive": 815, - "recall": 0.8201, - "total": 1485, - "trueNegative": 500, - "truePositive": 775, - "wilsonPrecisionLower95": 0.9339 - }, - "threshold": 0.8 - } - }, - "metrics": { - "f1": 0.914, - "falseNegative": 227, - "falsePositive": 86, - "precision": 0.9508, - "predictedPositive": 1749, - "recall": 0.8799, - "total": 2970, - "trueNegative": 994, - "truePositive": 1663, - "wilsonPrecisionLower95": 0.9397 - }, - "threshold": 0.78 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9888, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 44, - "recall": 0.9778, - "total": 1485, - "trueNegative": 1440, - "truePositive": 44, - "wilsonPrecisionLower95": 0.9197 - }, - "threshold": 0.635 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.8129, - "falseNegative": 27, - "falsePositive": 2, - "precision": 0.9692, - "predictedPositive": 65, - "recall": 0.7, - "total": 2970, - "trueNegative": 2878, - "truePositive": 63, - "wilsonPrecisionLower95": 0.8946 - }, - "threshold": 0.995 - }, - "task": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8185, - "falseNegative": 103, - "falsePositive": 11, - "precision": 0.959, - "predictedPositive": 268, - "recall": 0.7139, - "total": 1485, - "trueNegative": 1114, - "truePositive": 257, - "wilsonPrecisionLower95": 0.928 - }, - "threshold": 0.83 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7702, - "falseNegative": 127, - "falsePositive": 12, - "precision": 0.951, - "predictedPositive": 245, - "recall": 0.6472, - "total": 1485, - "trueNegative": 1113, - "truePositive": 233, - "wilsonPrecisionLower95": 0.9164 - }, - "threshold": 0.765 - } - }, - "metrics": { - "f1": 0.7816, - "falseNegative": 244, - "falsePositive": 22, - "precision": 0.9558, - "predictedPositive": 498, - "recall": 0.6611, - "total": 2970, - "trueNegative": 2228, - "truePositive": 476, - "wilsonPrecisionLower95": 0.934 - }, - "threshold": 0.805 - } - }, - "trainingRecords": 15345, - "validation": { - "intentsMeeting95Precision": 7, - "macroF1": 0.4633, - "macroPrecision": 0.987, - "macroRecall": 0.3584, - "minimumPrecision": 0.9508, - "perIntent": { - "blessing": { - "f1": 0.0851, - "falseNegative": 86, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 4, - "recall": 0.0444, - "total": 2970, - "trueNegative": 2880, - "truePositive": 4, - "wilsonPrecisionLower95": 0.5101 - }, - "complaint": { - "f1": 0.5484, - "falseNegative": 224, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 136, - "recall": 0.3778, - "total": 2970, - "trueNegative": 2610, - "truePositive": 136, - "wilsonPrecisionLower95": 0.9725 - }, - "confirmationDecision": { - "f1": 0.616, - "falseNegative": 99, - "falsePositive": 2, - "precision": 0.9759, - "predictedPositive": 83, - "recall": 0.45, - "total": 2970, - "trueNegative": 2788, - "truePositive": 81, - "wilsonPrecisionLower95": 0.9163 - }, - "followUpReminder": { - "f1": 0.2524, - "falseNegative": 308, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 52, - "recall": 0.1444, - "total": 2970, - "trueNegative": 2610, - "truePositive": 52, - "wilsonPrecisionLower95": 0.9312 - }, - "invitation": { - "f1": 0.0645, - "falseNegative": 174, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 6, - "recall": 0.0333, - "total": 2970, - "trueNegative": 2790, - "truePositive": 6, - "wilsonPrecisionLower95": 0.6097 - }, - "question": { - "f1": 0.8249, - "falseNegative": 180, - "falsePositive": 11, - "precision": 0.9761, - "predictedPositive": 461, - "recall": 0.7143, - "total": 2970, - "trueNegative": 2329, - "truePositive": 450, - "wilsonPrecisionLower95": 0.9578 - }, - "replyableMessage": { - "f1": 0.9188, - "falseNegative": 210, - "falsePositive": 87, - "precision": 0.9508, - "predictedPositive": 1767, - "recall": 0.8889, - "total": 2970, - "trueNegative": 993, - "truePositive": 1680, - "wilsonPrecisionLower95": 0.9397 - }, - "scheduleNegotiation": { - "f1": 0.2857, - "falseNegative": 75, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 15, - "recall": 0.1667, - "total": 2970, - "trueNegative": 2880, - "truePositive": 15, - "wilsonPrecisionLower95": 0.7961 - }, - "task": { - "f1": 0.5737, - "falseNegative": 428, - "falsePositive": 6, - "precision": 0.9799, - "predictedPositive": 298, - "recall": 0.4056, - "total": 2970, - "trueNegative": 2244, - "truePositive": 292, - "wilsonPrecisionLower95": 0.9568 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 8e-06, - "augmentation": "punctuation", - "char_max": 6, - "char_min": 2, - "external_weight": 0.67, - "hard_example_weight": 1.5, - "l1_ratio": 0.1, - "max_char_features": 65000, - "max_word_features": 28000, - "min_df": 2, - "pseudo_weight": 0.24, - "round": 6, - "word_max": 2 - }, - "featureCount": 78354, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 0 - }, - "complaint": { - "negative": 146, - "positive": 0 - }, - "confirmationDecision": { - "negative": 88, - "positive": 2 - }, - "followUpReminder": { - "negative": 250, - "positive": 0 - }, - "invitation": { - "negative": 250, - "positive": 0 - }, - "question": { - "negative": 36, - "positive": 5 - }, - "replyableMessage": { - "negative": 2, - "positive": 130 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 0 - }, - "task": { - "negative": 83, - "positive": 0 - } - }, - "round": 6, - "selectionObjective": 0.6598, - "syntheticRegression": { - "intentsMeeting95Precision": 7, - "macroF1": 0.5285, - "macroPrecision": 0.9709, - "macroRecall": 0.4124, - "minimumPrecision": 0.78, - "perIntent": { - "blessing": { - "f1": 0.0435, - "falseNegative": 88, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 2, - "recall": 0.0222, - "total": 2970, - "trueNegative": 2880, - "truePositive": 2, - "wilsonPrecisionLower95": 0.3424 - }, - "complaint": { - "f1": 0.4211, - "falseNegative": 264, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 96, - "recall": 0.2667, - "total": 2970, - "trueNegative": 2610, - "truePositive": 96, - "wilsonPrecisionLower95": 0.9615 - }, - "confirmationDecision": { - "f1": 0.5571, - "falseNegative": 102, - "falsePositive": 22, - "precision": 0.78, - "predictedPositive": 100, - "recall": 0.4333, - "total": 2970, - "trueNegative": 2768, - "truePositive": 78, - "wilsonPrecisionLower95": 0.6893 - }, - "followUpReminder": { - "f1": 0.3636, - "falseNegative": 280, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 80, - "recall": 0.2222, - "total": 2970, - "trueNegative": 2610, - "truePositive": 80, - "wilsonPrecisionLower95": 0.9542 - }, - "invitation": { - "f1": 0.3333, - "falseNegative": 144, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 36, - "recall": 0.2, - "total": 2970, - "trueNegative": 2790, - "truePositive": 36, - "wilsonPrecisionLower95": 0.9036 - }, - "question": { - "f1": 0.8224, - "falseNegative": 190, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 440, - "recall": 0.6984, - "total": 2970, - "trueNegative": 2340, - "truePositive": 440, - "wilsonPrecisionLower95": 0.9913 - }, - "replyableMessage": { - "f1": 0.8888, - "falseNegative": 335, - "falsePositive": 54, - "precision": 0.9664, - "predictedPositive": 1609, - "recall": 0.8228, - "total": 2970, - "trueNegative": 1026, - "truePositive": 1555, - "wilsonPrecisionLower95": 0.9565 - }, - "scheduleNegotiation": { - "f1": 0.8387, - "falseNegative": 25, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 65, - "recall": 0.7222, - "total": 2970, - "trueNegative": 2880, - "truePositive": 65, - "wilsonPrecisionLower95": 0.9442 - }, - "task": { - "f1": 0.488, - "falseNegative": 487, - "falsePositive": 2, - "precision": 0.9915, - "predictedPositive": 235, - "recall": 0.3236, - "total": 2970, - "trueNegative": 2248, - "truePositive": 233, - "wilsonPrecisionLower95": 0.9695 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9834, - "falseNegative": 2, - "falsePositive": 4, - "precision": 0.978, - "predictedPositive": 182, - "recall": 0.9889, - "total": 1485, - "trueNegative": 1301, - "truePositive": 178, - "wilsonPrecisionLower95": 0.9449 - }, - "threshold": 0.555 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9494, - "falseNegative": 11, - "falsePositive": 7, - "precision": 0.9602, - "predictedPositive": 176, - "recall": 0.9389, - "total": 1485, - "trueNegative": 1298, - "truePositive": 169, - "wilsonPrecisionLower95": 0.9202 - }, - "threshold": 0.535 - } - }, - "metrics": { - "f1": 0.9652, - "falseNegative": 13, - "falsePositive": 12, - "precision": 0.9666, - "predictedPositive": 359, - "recall": 0.9639, - "total": 2970, - "trueNegative": 2598, - "truePositive": 347, - "wilsonPrecisionLower95": 0.9425 - }, - "threshold": 0.535 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8, - "falseNegative": 28, - "falsePositive": 3, - "precision": 0.9538, - "predictedPositive": 65, - "recall": 0.6889, - "total": 1485, - "trueNegative": 1392, - "truePositive": 62, - "wilsonPrecisionLower95": 0.8729 - }, - "threshold": 0.695 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.731, - "falseNegative": 37, - "falsePositive": 2, - "precision": 0.9636, - "predictedPositive": 55, - "recall": 0.5889, - "total": 1485, - "trueNegative": 1393, - "truePositive": 53, - "wilsonPrecisionLower95": 0.8768 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.7354, - "falseNegative": 73, - "falsePositive": 4, - "precision": 0.964, - "predictedPositive": 111, - "recall": 0.5944, - "total": 2970, - "trueNegative": 2786, - "truePositive": 107, - "wilsonPrecisionLower95": 0.911 - }, - "threshold": 0.695 - }, - "followUpReminder": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8632, - "falseNegative": 38, - "falsePositive": 7, - "precision": 0.953, - "predictedPositive": 149, - "recall": 0.7889, - "total": 1485, - "trueNegative": 1298, - "truePositive": 142, - "wilsonPrecisionLower95": 0.9062 - }, - "threshold": 0.9 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.4938, - "falseNegative": 120, - "falsePositive": 3, - "precision": 0.9524, - "predictedPositive": 63, - "recall": 0.3333, - "total": 1485, - "trueNegative": 1302, - "truePositive": 60, - "wilsonPrecisionLower95": 0.8691 - }, - "threshold": 0.845 - } - }, - "metrics": { - "f1": 0.7018, - "falseNegative": 160, - "falsePositive": 10, - "precision": 0.9524, - "predictedPositive": 210, - "recall": 0.5556, - "total": 2970, - "trueNegative": 2600, - "truePositive": 200, - "wilsonPrecisionLower95": 0.9146 - }, - "threshold": 0.885 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7222, - "falseNegative": 38, - "falsePositive": 2, - "precision": 0.963, - "predictedPositive": 54, - "recall": 0.5778, - "total": 1485, - "trueNegative": 1393, - "truePositive": 52, - "wilsonPrecisionLower95": 0.8746 - }, - "threshold": 0.665 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8167, - "falseNegative": 90, - "falsePositive": 11, - "precision": 0.9534, - "predictedPositive": 236, - "recall": 0.7143, - "total": 1485, - "trueNegative": 1159, - "truePositive": 225, - "wilsonPrecisionLower95": 0.9185 - }, - "threshold": 0.865 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8205, - "falseNegative": 91, - "falsePositive": 7, - "precision": 0.9697, - "predictedPositive": 231, - "recall": 0.7111, - "total": 1485, - "trueNegative": 1163, - "truePositive": 224, - "wilsonPrecisionLower95": 0.9388 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.804, - "falseNegative": 191, - "falsePositive": 23, - "precision": 0.9502, - "predictedPositive": 462, - "recall": 0.6968, - "total": 2970, - "trueNegative": 2317, - "truePositive": 439, - "wilsonPrecisionLower95": 0.9264 - }, - "threshold": 0.68 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9268, - "falseNegative": 90, - "falsePositive": 45, - "precision": 0.95, - "predictedPositive": 900, - "recall": 0.9048, - "total": 1485, - "trueNegative": 495, - "truePositive": 855, - "wilsonPrecisionLower95": 0.9338 - }, - "threshold": 0.86 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8789, - "falseNegative": 176, - "falsePositive": 36, - "precision": 0.9553, - "predictedPositive": 805, - "recall": 0.8138, - "total": 1485, - "trueNegative": 504, - "truePositive": 769, - "wilsonPrecisionLower95": 0.9387 - }, - "threshold": 0.85 - } - }, - "metrics": { - "f1": 0.9046, - "falseNegative": 259, - "falsePositive": 85, - "precision": 0.9505, - "predictedPositive": 1716, - "recall": 0.863, - "total": 2970, - "trueNegative": 995, - "truePositive": 1631, - "wilsonPrecisionLower95": 0.9392 - }, - "threshold": 0.85 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9773, - "falseNegative": 2, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 43, - "recall": 0.9556, - "total": 1485, - "trueNegative": 1440, - "truePositive": 43, - "wilsonPrecisionLower95": 0.918 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9888, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 44, - "recall": 0.9778, - "total": 1485, - "trueNegative": 1440, - "truePositive": 44, - "wilsonPrecisionLower95": 0.9197 - }, - "threshold": 0.97 - } - }, - "metrics": { - "f1": 0.9497, - "falseNegative": 5, - "falsePositive": 4, - "precision": 0.9551, - "predictedPositive": 89, - "recall": 0.9444, - "total": 2970, - "trueNegative": 2876, - "truePositive": 85, - "wilsonPrecisionLower95": 0.8901 - }, - "threshold": 0.91 - }, - "task": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.827, - "falseNegative": 97, - "falsePositive": 13, - "precision": 0.9529, - "predictedPositive": 276, - "recall": 0.7306, - "total": 1485, - "trueNegative": 1112, - "truePositive": 263, - "wilsonPrecisionLower95": 0.9211 - }, - "threshold": 0.89 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7902, - "falseNegative": 117, - "falsePositive": 12, - "precision": 0.9529, - "predictedPositive": 255, - "recall": 0.675, - "total": 1485, - "trueNegative": 1113, - "truePositive": 243, - "wilsonPrecisionLower95": 0.9196 - }, - "threshold": 0.785 - } - }, - "metrics": { - "f1": 0.7783, - "falseNegative": 246, - "falsePositive": 24, - "precision": 0.9518, - "predictedPositive": 498, - "recall": 0.6583, - "total": 2970, - "trueNegative": 2226, - "truePositive": 474, - "wilsonPrecisionLower95": 0.9293 - }, - "threshold": 0.85 - } - }, - "trainingRecords": 17767, - "validation": { - "intentsMeeting95Precision": 8, - "macroF1": 0.5015, - "macroPrecision": 0.9886, - "macroRecall": 0.3796, - "minimumPrecision": 0.9525, - "perIntent": { - "blessing": { - "f1": 0.1633, - "falseNegative": 82, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 8, - "recall": 0.0889, - "total": 2970, - "trueNegative": 2880, - "truePositive": 8, - "wilsonPrecisionLower95": 0.6756 - }, - "complaint": { - "f1": 0.5484, - "falseNegative": 224, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 136, - "recall": 0.3778, - "total": 2970, - "trueNegative": 2610, - "truePositive": 136, - "wilsonPrecisionLower95": 0.9725 - }, - "confirmationDecision": { - "f1": 0.5714, - "falseNegative": 108, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 72, - "recall": 0.4, - "total": 2970, - "trueNegative": 2790, - "truePositive": 72, - "wilsonPrecisionLower95": 0.9493 - }, - "followUpReminder": { - "f1": 0.2396, - "falseNegative": 311, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 49, - "recall": 0.1361, - "total": 2970, - "trueNegative": 2610, - "truePositive": 49, - "wilsonPrecisionLower95": 0.9273 - }, - "invitation": { - "f1": 0.191, - "falseNegative": 161, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 19, - "recall": 0.1056, - "total": 2970, - "trueNegative": 2790, - "truePositive": 19, - "wilsonPrecisionLower95": 0.8318 - }, - "question": { - "f1": 0.8126, - "falseNegative": 190, - "falsePositive": 13, - "precision": 0.9713, - "predictedPositive": 453, - "recall": 0.6984, - "total": 2970, - "trueNegative": 2327, - "truePositive": 440, - "wilsonPrecisionLower95": 0.9515 - }, - "replyableMessage": { - "f1": 0.9035, - "falseNegative": 266, - "falsePositive": 81, - "precision": 0.9525, - "predictedPositive": 1705, - "recall": 0.8593, - "total": 2970, - "trueNegative": 999, - "truePositive": 1624, - "wilsonPrecisionLower95": 0.9413 - }, - "scheduleNegotiation": { - "f1": 0.5, - "falseNegative": 60, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 30, - "recall": 0.3333, - "total": 2970, - "trueNegative": 2880, - "truePositive": 30, - "wilsonPrecisionLower95": 0.8865 - }, - "task": { - "f1": 0.5837, - "falseNegative": 420, - "falsePositive": 8, - "precision": 0.974, - "predictedPositive": 308, - "recall": 0.4167, - "total": 2970, - "trueNegative": 2242, - "truePositive": 300, - "wilsonPrecisionLower95": 0.9496 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 6e-06, - "augmentation": "punctuation", - "char_max": 6, - "char_min": 3, - "external_weight": 0.6900000000000001, - "hard_example_weight": 1.5, - "l1_ratio": 0.1, - "max_char_features": 65000, - "max_word_features": 32000, - "min_df": 2, - "pseudo_weight": 0.255, - "round": 7, - "word_max": 3 - }, - "featureCount": 87507, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 0 - }, - "complaint": { - "negative": 141, - "positive": 0 - }, - "confirmationDecision": { - "negative": 95, - "positive": 2 - }, - "followUpReminder": { - "negative": 250, - "positive": 0 - }, - "invitation": { - "negative": 250, - "positive": 0 - }, - "question": { - "negative": 36, - "positive": 5 - }, - "replyableMessage": { - "negative": 2, - "positive": 129 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 0 - }, - "task": { - "negative": 87, - "positive": 0 - } - }, - "round": 7, - "selectionObjective": 0.6286, - "syntheticRegression": { - "intentsMeeting95Precision": 7, - "macroF1": 0.439, - "macroPrecision": 0.9718, - "macroRecall": 0.3285, - "minimumPrecision": 0.8036, - "perIntent": { - "blessing": { - "f1": 0.0851, - "falseNegative": 86, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 4, - "recall": 0.0444, - "total": 2970, - "trueNegative": 2880, - "truePositive": 4, - "wilsonPrecisionLower95": 0.5101 - }, - "complaint": { - "f1": 0.3991, - "falseNegative": 270, - "falsePositive": 1, - "precision": 0.989, - "predictedPositive": 91, - "recall": 0.25, - "total": 2970, - "trueNegative": 2609, - "truePositive": 90, - "wilsonPrecisionLower95": 0.9404 - }, - "confirmationDecision": { - "f1": 0.6164, - "falseNegative": 90, - "falsePositive": 22, - "precision": 0.8036, - "predictedPositive": 112, - "recall": 0.5, - "total": 2970, - "trueNegative": 2768, - "truePositive": 90, - "wilsonPrecisionLower95": 0.7205 - }, - "followUpReminder": { - "f1": 0.2134, - "falseNegative": 317, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 43, - "recall": 0.1194, - "total": 2970, - "trueNegative": 2610, - "truePositive": 43, - "wilsonPrecisionLower95": 0.918 - }, - "invitation": { - "f1": 0.1053, - "falseNegative": 170, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 10, - "recall": 0.0556, - "total": 2970, - "trueNegative": 2790, - "truePositive": 10, - "wilsonPrecisionLower95": 0.7225 - }, - "question": { - "f1": 0.6751, - "falseNegative": 309, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 321, - "recall": 0.5095, - "total": 2970, - "trueNegative": 2340, - "truePositive": 321, - "wilsonPrecisionLower95": 0.9882 - }, - "replyableMessage": { - "f1": 0.8083, - "falseNegative": 572, - "falsePositive": 53, - "precision": 0.9613, - "predictedPositive": 1371, - "recall": 0.6974, - "total": 2970, - "trueNegative": 1027, - "truePositive": 1318, - "wilsonPrecisionLower95": 0.9498 - }, - "scheduleNegotiation": { - "f1": 0.7586, - "falseNegative": 35, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 55, - "recall": 0.6111, - "total": 2970, - "trueNegative": 2880, - "truePositive": 55, - "wilsonPrecisionLower95": 0.9347 - }, - "task": { - "f1": 0.2894, - "falseNegative": 598, - "falsePositive": 1, - "precision": 0.9919, - "predictedPositive": 123, - "recall": 0.1694, - "total": 2970, - "trueNegative": 2249, - "truePositive": 122, - "wilsonPrecisionLower95": 0.9554 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9809, - "falseNegative": 0, - "falsePositive": 7, - "precision": 0.9626, - "predictedPositive": 187, - "recall": 1.0, - "total": 1485, - "trueNegative": 1298, - "truePositive": 180, - "wilsonPrecisionLower95": 0.9248 - }, - "threshold": 0.74 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8974, - "falseNegative": 27, - "falsePositive": 8, - "precision": 0.9503, - "predictedPositive": 161, - "recall": 0.85, - "total": 1485, - "trueNegative": 1297, - "truePositive": 153, - "wilsonPrecisionLower95": 0.905 - }, - "threshold": 0.665 - } - }, - "metrics": { - "f1": 0.9529, - "falseNegative": 16, - "falsePositive": 18, - "precision": 0.9503, - "predictedPositive": 362, - "recall": 0.9556, - "total": 2970, - "trueNegative": 2592, - "truePositive": 344, - "wilsonPrecisionLower95": 0.9228 - }, - "threshold": 0.51 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.7763, - "falseNegative": 31, - "falsePositive": 3, - "precision": 0.9516, - "predictedPositive": 62, - "recall": 0.6556, - "total": 1485, - "trueNegative": 1392, - "truePositive": 59, - "wilsonPrecisionLower95": 0.8671 - }, - "threshold": 0.895 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7568, - "falseNegative": 34, - "falsePositive": 2, - "precision": 0.9655, - "predictedPositive": 58, - "recall": 0.6222, - "total": 1485, - "trueNegative": 1393, - "truePositive": 56, - "wilsonPrecisionLower95": 0.8827 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.6569, - "falseNegative": 90, - "falsePositive": 4, - "precision": 0.9574, - "predictedPositive": 94, - "recall": 0.5, - "total": 2970, - "trueNegative": 2786, - "truePositive": 90, - "wilsonPrecisionLower95": 0.8956 - }, - "threshold": 0.895 - }, - "followUpReminder": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8421, - "falseNegative": 44, - "falsePositive": 7, - "precision": 0.951, - "predictedPositive": 143, - "recall": 0.7556, - "total": 1485, - "trueNegative": 1298, - "truePositive": 136, - "wilsonPrecisionLower95": 0.9024 - }, - "threshold": 0.985 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.5985, - "falseNegative": 205, - "falsePositive": 3, - "precision": 0.981, - "predictedPositive": 158, - "recall": 0.4306, - "total": 2970, - "trueNegative": 2607, - "truePositive": 155, - "wilsonPrecisionLower95": 0.9457 - }, - "threshold": 0.995 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.7143, - "falseNegative": 135, - "falsePositive": 9, - "precision": 0.9524, - "predictedPositive": 189, - "recall": 0.5714, - "total": 1485, - "trueNegative": 1161, - "truePositive": 180, - "wilsonPrecisionLower95": 0.912 - }, - "threshold": 0.965 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7168, - "falseNegative": 134, - "falsePositive": 9, - "precision": 0.9526, - "predictedPositive": 190, - "recall": 0.5746, - "total": 1485, - "trueNegative": 1161, - "truePositive": 181, - "wilsonPrecisionLower95": 0.9124 - }, - "threshold": 0.52 - } - }, - "metrics": { - "f1": 0.7149, - "falseNegative": 269, - "falsePositive": 19, - "precision": 0.95, - "predictedPositive": 380, - "recall": 0.573, - "total": 2970, - "trueNegative": 2321, - "truePositive": 361, - "wilsonPrecisionLower95": 0.9232 - }, - "threshold": 0.795 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8846, - "falseNegative": 163, - "falsePositive": 41, - "precision": 0.9502, - "predictedPositive": 823, - "recall": 0.8275, - "total": 1485, - "trueNegative": 499, - "truePositive": 782, - "wilsonPrecisionLower95": 0.9331 - }, - "threshold": 0.89 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7476, - "falseNegative": 363, - "falsePositive": 30, - "precision": 0.951, - "predictedPositive": 612, - "recall": 0.6159, - "total": 1485, - "trueNegative": 510, - "truePositive": 582, - "wilsonPrecisionLower95": 0.9309 - }, - "threshold": 0.92 - } - }, - "metrics": { - "f1": 0.828, - "falseNegative": 506, - "falsePositive": 69, - "precision": 0.9525, - "predictedPositive": 1453, - "recall": 0.7323, - "total": 2970, - "trueNegative": 1011, - "truePositive": 1384, - "wilsonPrecisionLower95": 0.9403 - }, - "threshold": 0.91 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 45, - "recall": 1.0, - "total": 1485, - "trueNegative": 1440, - "truePositive": 45, - "wilsonPrecisionLower95": 0.9213 - }, - "threshold": 0.66 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 45, - "recall": 1.0, - "total": 1485, - "trueNegative": 1440, - "truePositive": 45, - "wilsonPrecisionLower95": 0.9213 - }, - "threshold": 0.55 - } - }, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 90, - "recall": 1.0, - "total": 2970, - "trueNegative": 2880, - "truePositive": 90, - "wilsonPrecisionLower95": 0.9591 - }, - "threshold": 0.66 - }, - "task": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8615, - "falseNegative": 77, - "falsePositive": 14, - "precision": 0.9529, - "predictedPositive": 297, - "recall": 0.7861, - "total": 1485, - "trueNegative": 1111, - "truePositive": 283, - "wilsonPrecisionLower95": 0.9224 - }, - "threshold": 0.805 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.6868, - "falseNegative": 333, - "falsePositive": 20, - "precision": 0.9509, - "predictedPositive": 407, - "recall": 0.5375, - "total": 2970, - "trueNegative": 2230, - "truePositive": 387, - "wilsonPrecisionLower95": 0.9253 - }, - "threshold": 0.925 - } - }, - "trainingRecords": 17708, - "validation": { - "intentsMeeting95Precision": 9, - "macroF1": 0.4573, - "macroPrecision": 0.9836, - "macroRecall": 0.3282, - "minimumPrecision": 0.9505, - "perIntent": { - "blessing": { - "f1": 0.2353, - "falseNegative": 78, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 12, - "recall": 0.1333, - "total": 2970, - "trueNegative": 2880, - "truePositive": 12, - "wilsonPrecisionLower95": 0.7575 - }, - "complaint": { - "f1": 0.5366, - "falseNegative": 228, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 132, - "recall": 0.3667, - "total": 2970, - "trueNegative": 2610, - "truePositive": 132, - "wilsonPrecisionLower95": 0.9717 - }, - "confirmationDecision": { - "f1": 0.5781, - "falseNegative": 106, - "falsePositive": 2, - "precision": 0.9737, - "predictedPositive": 76, - "recall": 0.4111, - "total": 2970, - "trueNegative": 2788, - "truePositive": 74, - "wilsonPrecisionLower95": 0.909 - }, - "followUpReminder": { - "f1": 0.2035, - "falseNegative": 319, - "falsePositive": 2, - "precision": 0.9535, - "predictedPositive": 43, - "recall": 0.1139, - "total": 2970, - "trueNegative": 2608, - "truePositive": 41, - "wilsonPrecisionLower95": 0.8454 - }, - "invitation": { - "f1": 0.1818, - "falseNegative": 162, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 18, - "recall": 0.1, - "total": 2970, - "trueNegative": 2790, - "truePositive": 18, - "wilsonPrecisionLower95": 0.8241 - }, - "question": { - "f1": 0.7078, - "falseNegative": 280, - "falsePositive": 9, - "precision": 0.9749, - "predictedPositive": 359, - "recall": 0.5556, - "total": 2970, - "trueNegative": 2331, - "truePositive": 350, - "wilsonPrecisionLower95": 0.953 - }, - "replyableMessage": { - "f1": 0.8205, - "falseNegative": 526, - "falsePositive": 71, - "precision": 0.9505, - "predictedPositive": 1435, - "recall": 0.7217, - "total": 2970, - "trueNegative": 1009, - "truePositive": 1364, - "wilsonPrecisionLower95": 0.9381 - }, - "scheduleNegotiation": { - "f1": 0.5246, - "falseNegative": 58, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 32, - "recall": 0.3556, - "total": 2970, - "trueNegative": 2880, - "truePositive": 32, - "wilsonPrecisionLower95": 0.8928 - }, - "task": { - "f1": 0.3275, - "falseNegative": 579, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 141, - "recall": 0.1958, - "total": 2970, - "trueNegative": 2250, - "truePositive": 141, - "wilsonPrecisionLower95": 0.9735 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 5e-06, - "augmentation": "punctuation", - "char_max": 7, - "char_min": 2, - "external_weight": 0.7100000000000001, - "hard_example_weight": 1.7, - "l1_ratio": 0.15, - "max_char_features": 70000, - "max_word_features": 28000, - "min_df": 2, - "pseudo_weight": 0.27, - "round": 8, - "word_max": 2 - }, - "featureCount": 83313, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 0 - }, - "complaint": { - "negative": 187, - "positive": 0 - }, - "confirmationDecision": { - "negative": 107, - "positive": 2 - }, - "followUpReminder": { - "negative": 250, - "positive": 0 - }, - "invitation": { - "negative": 250, - "positive": 0 - }, - "question": { - "negative": 40, - "positive": 5 - }, - "replyableMessage": { - "negative": 2, - "positive": 135 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 0 - }, - "task": { - "negative": 94, - "positive": 0 - } - }, - "round": 8, - "selectionObjective": 0.6543, - "syntheticRegression": { - "intentsMeeting95Precision": 7, - "macroF1": 0.5256, - "macroPrecision": 0.9725, - "macroRecall": 0.4039, - "minimumPrecision": 0.8017, - "perIntent": { - "blessing": { - "f1": 0.125, - "falseNegative": 84, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 6, - "recall": 0.0667, - "total": 2970, - "trueNegative": 2880, - "truePositive": 6, - "wilsonPrecisionLower95": 0.6097 - }, - "complaint": { - "f1": 0.4279, - "falseNegative": 262, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 98, - "recall": 0.2722, - "total": 2970, - "trueNegative": 2610, - "truePositive": 98, - "wilsonPrecisionLower95": 0.9623 - }, - "confirmationDecision": { - "f1": 0.6284, - "falseNegative": 87, - "falsePositive": 23, - "precision": 0.8017, - "predictedPositive": 116, - "recall": 0.5167, - "total": 2970, - "trueNegative": 2767, - "truePositive": 93, - "wilsonPrecisionLower95": 0.72 - }, - "followUpReminder": { - "f1": 0.2686, - "falseNegative": 304, - "falsePositive": 1, - "precision": 0.9825, - "predictedPositive": 57, - "recall": 0.1556, - "total": 2970, - "trueNegative": 2609, - "truePositive": 56, - "wilsonPrecisionLower95": 0.9071 - }, - "invitation": { - "f1": 0.341, - "falseNegative": 143, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 37, - "recall": 0.2056, - "total": 2970, - "trueNegative": 2790, - "truePositive": 37, - "wilsonPrecisionLower95": 0.9059 - }, - "question": { - "f1": 0.7525, - "falseNegative": 250, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 380, - "recall": 0.6032, - "total": 2970, - "trueNegative": 2340, - "truePositive": 380, - "wilsonPrecisionLower95": 0.99 - }, - "replyableMessage": { - "f1": 0.8832, - "falseNegative": 366, - "falsePositive": 37, - "precision": 0.9763, - "predictedPositive": 1561, - "recall": 0.8063, - "total": 2970, - "trueNegative": 1043, - "truePositive": 1524, - "wilsonPrecisionLower95": 0.9675 - }, - "scheduleNegotiation": { - "f1": 0.8079, - "falseNegative": 29, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 61, - "recall": 0.6778, - "total": 2970, - "trueNegative": 2880, - "truePositive": 61, - "wilsonPrecisionLower95": 0.9408 - }, - "task": { - "f1": 0.4958, - "falseNegative": 482, - "falsePositive": 2, - "precision": 0.9917, - "predictedPositive": 240, - "recall": 0.3306, - "total": 2970, - "trueNegative": 2248, - "truePositive": 238, - "wilsonPrecisionLower95": 0.9701 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.989, - "falseNegative": 1, - "falsePositive": 3, - "precision": 0.9835, - "predictedPositive": 182, - "recall": 0.9944, - "total": 1485, - "trueNegative": 1302, - "truePositive": 179, - "wilsonPrecisionLower95": 0.9527 - }, - "threshold": 0.97 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9405, - "falseNegative": 14, - "falsePositive": 7, - "precision": 0.9595, - "predictedPositive": 173, - "recall": 0.9222, - "total": 1485, - "trueNegative": 1298, - "truePositive": 166, - "wilsonPrecisionLower95": 0.9188 - }, - "threshold": 0.54 - } - }, - "metrics": { - "f1": 0.9237, - "falseNegative": 39, - "falsePositive": 14, - "precision": 0.9582, - "predictedPositive": 335, - "recall": 0.8917, - "total": 2970, - "trueNegative": 2596, - "truePositive": 321, - "wilsonPrecisionLower95": 0.9311 - }, - "threshold": 0.925 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9613, - "falseNegative": 3, - "falsePositive": 4, - "precision": 0.956, - "predictedPositive": 91, - "recall": 0.9667, - "total": 1485, - "trueNegative": 1391, - "truePositive": 87, - "wilsonPrecisionLower95": 0.8924 - }, - "threshold": 0.72 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.5556, - "falseNegative": 55, - "falsePositive": 1, - "precision": 0.9722, - "predictedPositive": 36, - "recall": 0.3889, - "total": 1485, - "trueNegative": 1394, - "truePositive": 35, - "wilsonPrecisionLower95": 0.8583 - }, - "threshold": 0.905 - } - }, - "metrics": { - "f1": 0.7922, - "falseNegative": 58, - "falsePositive": 6, - "precision": 0.9531, - "predictedPositive": 128, - "recall": 0.6778, - "total": 2970, - "trueNegative": 2784, - "truePositive": 122, - "wilsonPrecisionLower95": 0.9015 - }, - "threshold": 0.88 - }, - "followUpReminder": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8437, - "falseNegative": 45, - "falsePositive": 5, - "precision": 0.9643, - "predictedPositive": 140, - "recall": 0.75, - "total": 1485, - "trueNegative": 1300, - "truePositive": 135, - "wilsonPrecisionLower95": 0.9191 - }, - "threshold": 0.96 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.7063, - "falseNegative": 158, - "falsePositive": 10, - "precision": 0.9528, - "predictedPositive": 212, - "recall": 0.5611, - "total": 2970, - "trueNegative": 2600, - "truePositive": 202, - "wilsonPrecisionLower95": 0.9154 - }, - "threshold": 0.96 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.7157, - "falseNegative": 135, - "falsePositive": 8, - "precision": 0.9574, - "predictedPositive": 188, - "recall": 0.5714, - "total": 1485, - "trueNegative": 1162, - "truePositive": 180, - "wilsonPrecisionLower95": 0.9183 - }, - "threshold": 0.975 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.822, - "falseNegative": 91, - "falsePositive": 6, - "precision": 0.9739, - "predictedPositive": 230, - "recall": 0.7111, - "total": 1485, - "trueNegative": 1164, - "truePositive": 224, - "wilsonPrecisionLower95": 0.9443 - }, - "threshold": 0.505 - } - }, - "metrics": { - "f1": 0.8095, - "falseNegative": 186, - "falsePositive": 23, - "precision": 0.9507, - "predictedPositive": 467, - "recall": 0.7048, - "total": 2970, - "trueNegative": 2317, - "truePositive": 444, - "wilsonPrecisionLower95": 0.9272 - }, - "threshold": 0.71 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8741, - "falseNegative": 181, - "falsePositive": 39, - "precision": 0.9514, - "predictedPositive": 803, - "recall": 0.8085, - "total": 1485, - "trueNegative": 501, - "truePositive": 764, - "wilsonPrecisionLower95": 0.9343 - }, - "threshold": 0.95 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8775, - "falseNegative": 175, - "falsePositive": 40, - "precision": 0.9506, - "predictedPositive": 810, - "recall": 0.8148, - "total": 1485, - "trueNegative": 500, - "truePositive": 770, - "wilsonPrecisionLower95": 0.9335 - }, - "threshold": 0.88 - } - }, - "metrics": { - "f1": 0.8762, - "falseNegative": 354, - "falsePositive": 80, - "precision": 0.9505, - "predictedPositive": 1616, - "recall": 0.8127, - "total": 2970, - "trueNegative": 1000, - "truePositive": 1536, - "wilsonPrecisionLower95": 0.9388 - }, - "threshold": 0.905 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 45, - "recall": 1.0, - "total": 1485, - "trueNegative": 1440, - "truePositive": 45, - "wilsonPrecisionLower95": 0.9213 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9556, - "falseNegative": 2, - "falsePositive": 2, - "precision": 0.9556, - "predictedPositive": 45, - "recall": 0.9556, - "total": 1485, - "trueNegative": 1438, - "truePositive": 43, - "wilsonPrecisionLower95": 0.8517 - }, - "threshold": 0.75 - } - }, - "metrics": { - "f1": 0.9724, - "falseNegative": 2, - "falsePositive": 3, - "precision": 0.967, - "predictedPositive": 91, - "recall": 0.9778, - "total": 2970, - "trueNegative": 2877, - "truePositive": 88, - "wilsonPrecisionLower95": 0.9075 - }, - "threshold": 0.505 - }, - "task": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8331, - "falseNegative": 93, - "falsePositive": 14, - "precision": 0.9502, - "predictedPositive": 281, - "recall": 0.7417, - "total": 1485, - "trueNegative": 1111, - "truePositive": 267, - "wilsonPrecisionLower95": 0.9181 - }, - "threshold": 0.95 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7445, - "falseNegative": 140, - "falsePositive": 11, - "precision": 0.9524, - "predictedPositive": 231, - "recall": 0.6111, - "total": 1485, - "trueNegative": 1114, - "truePositive": 220, - "wilsonPrecisionLower95": 0.9168 - }, - "threshold": 0.86 - } - }, - "metrics": { - "f1": 0.7672, - "falseNegative": 257, - "falsePositive": 24, - "precision": 0.9507, - "predictedPositive": 487, - "recall": 0.6431, - "total": 2970, - "trueNegative": 2226, - "truePositive": 463, - "wilsonPrecisionLower95": 0.9277 - }, - "threshold": 0.92 - } - }, - "trainingRecords": 17762, - "validation": { - "intentsMeeting95Precision": 7, - "macroF1": 0.5016, - "macroPrecision": 0.9741, - "macroRecall": 0.3747, - "minimumPrecision": 0.9333, - "perIntent": { - "blessing": { - "f1": 0.2524, - "falseNegative": 77, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 13, - "recall": 0.1444, - "total": 2970, - "trueNegative": 2880, - "truePositive": 13, - "wilsonPrecisionLower95": 0.7719 - }, - "complaint": { - "f1": 0.5513, - "falseNegative": 223, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 137, - "recall": 0.3806, - "total": 2970, - "trueNegative": 2610, - "truePositive": 137, - "wilsonPrecisionLower95": 0.9727 - }, - "confirmationDecision": { - "f1": 0.6107, - "falseNegative": 100, - "falsePositive": 2, - "precision": 0.9756, - "predictedPositive": 82, - "recall": 0.4444, - "total": 2970, - "trueNegative": 2788, - "truePositive": 80, - "wilsonPrecisionLower95": 0.9154 - }, - "followUpReminder": { - "f1": 0.2074, - "falseNegative": 318, - "falsePositive": 3, - "precision": 0.9333, - "predictedPositive": 45, - "recall": 0.1167, - "total": 2970, - "trueNegative": 2607, - "truePositive": 42, - "wilsonPrecisionLower95": 0.8214 - }, - "invitation": { - "f1": 0.1633, - "falseNegative": 164, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 16, - "recall": 0.0889, - "total": 2970, - "trueNegative": 2790, - "truePositive": 16, - "wilsonPrecisionLower95": 0.8064 - }, - "question": { - "f1": 0.759, - "falseNegative": 238, - "falsePositive": 11, - "precision": 0.9727, - "predictedPositive": 403, - "recall": 0.6222, - "total": 2970, - "trueNegative": 2329, - "truePositive": 392, - "wilsonPrecisionLower95": 0.9518 - }, - "replyableMessage": { - "f1": 0.8758, - "falseNegative": 356, - "falsePositive": 79, - "precision": 0.951, - "predictedPositive": 1613, - "recall": 0.8116, - "total": 2970, - "trueNegative": 1001, - "truePositive": 1534, - "wilsonPrecisionLower95": 0.9394 - }, - "scheduleNegotiation": { - "f1": 0.5161, - "falseNegative": 58, - "falsePositive": 2, - "precision": 0.9412, - "predictedPositive": 34, - "recall": 0.3556, - "total": 2970, - "trueNegative": 2878, - "truePositive": 32, - "wilsonPrecisionLower95": 0.8091 - }, - "task": { - "f1": 0.5787, - "falseNegative": 426, - "falsePositive": 2, - "precision": 0.9932, - "predictedPositive": 296, - "recall": 0.4083, - "total": 2970, - "trueNegative": 2248, - "truePositive": 294, - "wilsonPrecisionLower95": 0.9757 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 4e-06, - "augmentation": "punctuation", - "char_max": 7, - "char_min": 3, - "external_weight": 0.73, - "hard_example_weight": 1.7, - "l1_ratio": 0.15, - "max_char_features": 70000, - "max_word_features": 35000, - "min_df": 2, - "pseudo_weight": 0.28500000000000003, - "round": 9, - "word_max": 3 - }, - "featureCount": 92641, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 0 - }, - "complaint": { - "negative": 250, - "positive": 0 - }, - "confirmationDecision": { - "negative": 248, - "positive": 6 - }, - "followUpReminder": { - "negative": 250, - "positive": 1 - }, - "invitation": { - "negative": 250, - "positive": 0 - }, - "question": { - "negative": 146, - "positive": 8 - }, - "replyableMessage": { - "negative": 8, - "positive": 250 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 0 - }, - "task": { - "negative": 208, - "positive": 0 - } - }, - "round": 9, - "selectionObjective": 0.6362, - "syntheticRegression": { - "intentsMeeting95Precision": 6, - "macroF1": 0.473, - "macroPrecision": 0.9657, - "macroRecall": 0.3476, - "minimumPrecision": 0.7955, - "perIntent": { - "blessing": { - "f1": 0.125, - "falseNegative": 84, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 6, - "recall": 0.0667, - "total": 2970, - "trueNegative": 2880, - "truePositive": 6, - "wilsonPrecisionLower95": 0.6097 - }, - "complaint": { - "f1": 0.4141, - "falseNegative": 266, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 94, - "recall": 0.2611, - "total": 2970, - "trueNegative": 2610, - "truePositive": 94, - "wilsonPrecisionLower95": 0.9607 - }, - "confirmationDecision": { - "f1": 0.5224, - "falseNegative": 110, - "falsePositive": 18, - "precision": 0.7955, - "predictedPositive": 88, - "recall": 0.3889, - "total": 2970, - "trueNegative": 2772, - "truePositive": 70, - "wilsonPrecisionLower95": 0.6997 - }, - "followUpReminder": { - "f1": 0.2255, - "falseNegative": 314, - "falsePositive": 2, - "precision": 0.9583, - "predictedPositive": 48, - "recall": 0.1278, - "total": 2970, - "trueNegative": 2608, - "truePositive": 46, - "wilsonPrecisionLower95": 0.8602 - }, - "invitation": { - "f1": 0.4071, - "falseNegative": 134, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 46, - "recall": 0.2556, - "total": 2970, - "trueNegative": 2790, - "truePositive": 46, - "wilsonPrecisionLower95": 0.9229 - }, - "question": { - "f1": 0.6452, - "falseNegative": 330, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 300, - "recall": 0.4762, - "total": 2970, - "trueNegative": 2340, - "truePositive": 300, - "wilsonPrecisionLower95": 0.9874 - }, - "replyableMessage": { - "f1": 0.7919, - "falseNegative": 594, - "falsePositive": 87, - "precision": 0.9371, - "predictedPositive": 1383, - "recall": 0.6857, - "total": 2970, - "trueNegative": 993, - "truePositive": 1296, - "wilsonPrecisionLower95": 0.923 - }, - "scheduleNegotiation": { - "f1": 0.8079, - "falseNegative": 29, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 61, - "recall": 0.6778, - "total": 2970, - "trueNegative": 2880, - "truePositive": 61, - "wilsonPrecisionLower95": 0.9408 - }, - "task": { - "f1": 0.3178, - "falseNegative": 584, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 136, - "recall": 0.1889, - "total": 2970, - "trueNegative": 2250, - "truePositive": 136, - "wilsonPrecisionLower95": 0.9725 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9862, - "falseNegative": 1, - "falsePositive": 4, - "precision": 0.9781, - "predictedPositive": 183, - "recall": 0.9944, - "total": 1485, - "trueNegative": 1301, - "truePositive": 179, - "wilsonPrecisionLower95": 0.9452 - }, - "threshold": 0.74 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9222, - "falseNegative": 20, - "falsePositive": 7, - "precision": 0.9581, - "predictedPositive": 167, - "recall": 0.8889, - "total": 1485, - "trueNegative": 1298, - "truePositive": 160, - "wilsonPrecisionLower95": 0.916 - }, - "threshold": 0.62 - } - }, - "metrics": { - "f1": 0.9528, - "falseNegative": 17, - "falsePositive": 17, - "precision": 0.9528, - "predictedPositive": 360, - "recall": 0.9528, - "total": 2970, - "trueNegative": 2593, - "truePositive": 343, - "wilsonPrecisionLower95": 0.9257 - }, - "threshold": 0.5 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9492, - "falseNegative": 6, - "falsePositive": 3, - "precision": 0.9655, - "predictedPositive": 87, - "recall": 0.9333, - "total": 1485, - "trueNegative": 1392, - "truePositive": 84, - "wilsonPrecisionLower95": 0.9035 - }, - "threshold": 0.965 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7042, - "falseNegative": 40, - "falsePositive": 2, - "precision": 0.9615, - "predictedPositive": 52, - "recall": 0.5556, - "total": 1485, - "trueNegative": 1393, - "truePositive": 50, - "wilsonPrecisionLower95": 0.8702 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.76, - "falseNegative": 66, - "falsePositive": 6, - "precision": 0.95, - "predictedPositive": 120, - "recall": 0.6333, - "total": 2970, - "trueNegative": 2784, - "truePositive": 114, - "wilsonPrecisionLower95": 0.8952 - }, - "threshold": 0.945 - }, - "followUpReminder": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8243, - "falseNegative": 51, - "falsePositive": 4, - "precision": 0.9699, - "predictedPositive": 133, - "recall": 0.7167, - "total": 1485, - "trueNegative": 1301, - "truePositive": 129, - "wilsonPrecisionLower95": 0.9252 - }, - "threshold": 0.99 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.6316, - "falseNegative": 192, - "falsePositive": 4, - "precision": 0.9767, - "predictedPositive": 172, - "recall": 0.4667, - "total": 2970, - "trueNegative": 2606, - "truePositive": 168, - "wilsonPrecisionLower95": 0.9417 - }, - "threshold": 0.995 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.544, - "falseNegative": 56, - "falsePositive": 1, - "precision": 0.9714, - "predictedPositive": 35, - "recall": 0.3778, - "total": 1485, - "trueNegative": 1394, - "truePositive": 34, - "wilsonPrecisionLower95": 0.8547 - }, - "threshold": 0.82 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.6694, - "falseNegative": 153, - "falsePositive": 7, - "precision": 0.9586, - "predictedPositive": 169, - "recall": 0.5143, - "total": 1485, - "trueNegative": 1163, - "truePositive": 162, - "wilsonPrecisionLower95": 0.917 - }, - "threshold": 0.985 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6762, - "falseNegative": 150, - "falsePositive": 8, - "precision": 0.9538, - "predictedPositive": 173, - "recall": 0.5238, - "total": 1485, - "trueNegative": 1162, - "truePositive": 165, - "wilsonPrecisionLower95": 0.9114 - }, - "threshold": 0.64 - } - }, - "metrics": { - "f1": 0.5392, - "falseNegative": 393, - "falsePositive": 12, - "precision": 0.9518, - "predictedPositive": 249, - "recall": 0.3762, - "total": 2970, - "trueNegative": 2328, - "truePositive": 237, - "wilsonPrecisionLower95": 0.9177 - }, - "threshold": 0.965 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9407, - "falseNegative": 65, - "falsePositive": 46, - "precision": 0.9503, - "predictedPositive": 926, - "recall": 0.9312, - "total": 1485, - "trueNegative": 494, - "truePositive": 880, - "wilsonPrecisionLower95": 0.9344 - }, - "threshold": 0.745 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6118, - "falseNegative": 521, - "falsePositive": 17, - "precision": 0.9615, - "predictedPositive": 441, - "recall": 0.4487, - "total": 1485, - "trueNegative": 523, - "truePositive": 424, - "wilsonPrecisionLower95": 0.9391 - }, - "threshold": 0.965 - } - }, - "metrics": { - "f1": 0.8037, - "falseNegative": 576, - "falsePositive": 66, - "precision": 0.9522, - "predictedPositive": 1380, - "recall": 0.6952, - "total": 2970, - "trueNegative": 1014, - "truePositive": 1314, - "wilsonPrecisionLower95": 0.9396 - }, - "threshold": 0.95 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9286, - "falseNegative": 6, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 39, - "recall": 0.8667, - "total": 1485, - "trueNegative": 1440, - "truePositive": 39, - "wilsonPrecisionLower95": 0.9103 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9302, - "falseNegative": 5, - "falsePositive": 1, - "precision": 0.9756, - "predictedPositive": 41, - "recall": 0.8889, - "total": 1485, - "trueNegative": 1439, - "truePositive": 40, - "wilsonPrecisionLower95": 0.874 - }, - "threshold": 0.61 - } - }, - "metrics": { - "f1": 0.9294, - "falseNegative": 11, - "falsePositive": 1, - "precision": 0.9875, - "predictedPositive": 80, - "recall": 0.8778, - "total": 2970, - "trueNegative": 2879, - "truePositive": 79, - "wilsonPrecisionLower95": 0.9325 - }, - "threshold": 0.61 - }, - "task": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8403, - "falseNegative": 89, - "falsePositive": 14, - "precision": 0.9509, - "predictedPositive": 285, - "recall": 0.7528, - "total": 1485, - "trueNegative": 1111, - "truePositive": 271, - "wilsonPrecisionLower95": 0.9192 - }, - "threshold": 0.93 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.2384, - "falseNegative": 311, - "falsePositive": 2, - "precision": 0.9608, - "predictedPositive": 51, - "recall": 0.1361, - "total": 1485, - "trueNegative": 1123, - "truePositive": 49, - "wilsonPrecisionLower95": 0.8678 - }, - "threshold": 0.995 - } - }, - "metrics": { - "f1": 0.6415, - "falseNegative": 372, - "falsePositive": 17, - "precision": 0.9534, - "predictedPositive": 365, - "recall": 0.4833, - "total": 2970, - "trueNegative": 2233, - "truePositive": 348, - "wilsonPrecisionLower95": 0.9267 - }, - "threshold": 0.975 - } - }, - "trainingRecords": 17823, - "validation": { - "intentsMeeting95Precision": 8, - "macroF1": 0.4697, - "macroPrecision": 0.9844, - "macroRecall": 0.3337, - "minimumPrecision": 0.9268, - "perIntent": { - "blessing": { - "f1": 0.2857, - "falseNegative": 75, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 15, - "recall": 0.1667, - "total": 2970, - "trueNegative": 2880, - "truePositive": 15, - "wilsonPrecisionLower95": 0.7961 - }, - "complaint": { - "f1": 0.5366, - "falseNegative": 228, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 132, - "recall": 0.3667, - "total": 2970, - "trueNegative": 2610, - "truePositive": 132, - "wilsonPrecisionLower95": 0.9717 - }, - "confirmationDecision": { - "f1": 0.6415, - "falseNegative": 95, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 85, - "recall": 0.4722, - "total": 2970, - "trueNegative": 2790, - "truePositive": 85, - "wilsonPrecisionLower95": 0.9568 - }, - "followUpReminder": { - "f1": 0.1895, - "falseNegative": 322, - "falsePositive": 3, - "precision": 0.9268, - "predictedPositive": 41, - "recall": 0.1056, - "total": 2970, - "trueNegative": 2607, - "truePositive": 38, - "wilsonPrecisionLower95": 0.8057 - }, - "invitation": { - "f1": 0.2266, - "falseNegative": 157, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 23, - "recall": 0.1278, - "total": 2970, - "trueNegative": 2790, - "truePositive": 23, - "wilsonPrecisionLower95": 0.8569 - }, - "question": { - "f1": 0.6701, - "falseNegative": 309, - "falsePositive": 7, - "precision": 0.9787, - "predictedPositive": 328, - "recall": 0.5095, - "total": 2970, - "trueNegative": 2333, - "truePositive": 321, - "wilsonPrecisionLower95": 0.9566 - }, - "replyableMessage": { - "f1": 0.8007, - "falseNegative": 586, - "falsePositive": 63, - "precision": 0.9539, - "predictedPositive": 1367, - "recall": 0.6899, - "total": 2970, - "trueNegative": 1017, - "truePositive": 1304, - "wilsonPrecisionLower95": 0.9415 - }, - "scheduleNegotiation": { - "f1": 0.5, - "falseNegative": 60, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 30, - "recall": 0.3333, - "total": 2970, - "trueNegative": 2880, - "truePositive": 30, - "wilsonPrecisionLower95": 0.8865 - }, - "task": { - "f1": 0.3766, - "falseNegative": 553, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 167, - "recall": 0.2319, - "total": 2970, - "trueNegative": 2250, - "truePositive": 167, - "wilsonPrecisionLower95": 0.9775 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 6e-06, - "augmentation": "punctuation", - "char_max": 6, - "char_min": 2, - "external_weight": 0.75, - "hard_example_weight": 1.8, - "l1_ratio": 0.2, - "max_char_features": 75000, - "max_word_features": 38000, - "min_df": 1, - "pseudo_weight": 0.3, - "round": 10, - "word_max": 3 - }, - "featureCount": 113000, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 0 - }, - "complaint": { - "negative": 250, - "positive": 0 - }, - "confirmationDecision": { - "negative": 250, - "positive": 11 - }, - "followUpReminder": { - "negative": 250, - "positive": 4 - }, - "invitation": { - "negative": 250, - "positive": 0 - }, - "question": { - "negative": 182, - "positive": 10 - }, - "replyableMessage": { - "negative": 11, - "positive": 250 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 0 - }, - "task": { - "negative": 250, - "positive": 1 - } - }, - "round": 10, - "selectionObjective": 0.6131, - "syntheticRegression": { - "intentsMeeting95Precision": 7, - "macroF1": 0.4796, - "macroPrecision": 0.972, - "macroRecall": 0.3442, - "minimumPrecision": 0.8198, - "perIntent": { - "blessing": { - "f1": 0.125, - "falseNegative": 84, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 6, - "recall": 0.0667, - "total": 2970, - "trueNegative": 2880, - "truePositive": 6, - "wilsonPrecisionLower95": 0.6097 - }, - "complaint": { - "f1": 0.4148, - "falseNegative": 265, - "falsePositive": 3, - "precision": 0.9694, - "predictedPositive": 98, - "recall": 0.2639, - "total": 2970, - "trueNegative": 2607, - "truePositive": 95, - "wilsonPrecisionLower95": 0.9138 - }, - "confirmationDecision": { - "f1": 0.6254, - "falseNegative": 89, - "falsePositive": 20, - "precision": 0.8198, - "predictedPositive": 111, - "recall": 0.5056, - "total": 2970, - "trueNegative": 2770, - "truePositive": 91, - "wilsonPrecisionLower95": 0.738 - }, - "followUpReminder": { - "f1": 0.2768, - "falseNegative": 302, - "falsePositive": 1, - "precision": 0.9831, - "predictedPositive": 59, - "recall": 0.1611, - "total": 2970, - "trueNegative": 2609, - "truePositive": 58, - "wilsonPrecisionLower95": 0.91 - }, - "invitation": { - "f1": 0.3636, - "falseNegative": 140, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 40, - "recall": 0.2222, - "total": 2970, - "trueNegative": 2790, - "truePositive": 40, - "wilsonPrecisionLower95": 0.9124 - }, - "question": { - "f1": 0.5383, - "falseNegative": 398, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 232, - "recall": 0.3683, - "total": 2970, - "trueNegative": 2340, - "truePositive": 232, - "wilsonPrecisionLower95": 0.9837 - }, - "replyableMessage": { - "f1": 0.7216, - "falseNegative": 808, - "falsePositive": 27, - "precision": 0.9757, - "predictedPositive": 1109, - "recall": 0.5725, - "total": 2970, - "trueNegative": 1053, - "truePositive": 1082, - "wilsonPrecisionLower95": 0.9648 - }, - "scheduleNegotiation": { - "f1": 0.7586, - "falseNegative": 35, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 55, - "recall": 0.6111, - "total": 2970, - "trueNegative": 2880, - "truePositive": 55, - "wilsonPrecisionLower95": 0.9347 - }, - "task": { - "f1": 0.4921, - "falseNegative": 485, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 235, - "recall": 0.3264, - "total": 2970, - "trueNegative": 2250, - "truePositive": 235, - "wilsonPrecisionLower95": 0.9839 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9807, - "falseNegative": 2, - "falsePositive": 5, - "precision": 0.9727, - "predictedPositive": 183, - "recall": 0.9889, - "total": 1485, - "trueNegative": 1300, - "truePositive": 178, - "wilsonPrecisionLower95": 0.9376 - }, - "threshold": 0.765 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9288, - "falseNegative": 17, - "falsePositive": 8, - "precision": 0.9532, - "predictedPositive": 171, - "recall": 0.9056, - "total": 1485, - "trueNegative": 1297, - "truePositive": 163, - "wilsonPrecisionLower95": 0.9104 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.9455, - "falseNegative": 22, - "falsePositive": 17, - "precision": 0.9521, - "predictedPositive": 355, - "recall": 0.9389, - "total": 2970, - "trueNegative": 2593, - "truePositive": 338, - "wilsonPrecisionLower95": 0.9247 - }, - "threshold": 0.595 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.7568, - "falseNegative": 34, - "falsePositive": 2, - "precision": 0.9655, - "predictedPositive": 58, - "recall": 0.6222, - "total": 1485, - "trueNegative": 1393, - "truePositive": 56, - "wilsonPrecisionLower95": 0.8827 - }, - "threshold": 0.955 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7733, - "falseNegative": 32, - "falsePositive": 2, - "precision": 0.9667, - "predictedPositive": 60, - "recall": 0.6444, - "total": 1485, - "trueNegative": 1393, - "truePositive": 58, - "wilsonPrecisionLower95": 0.8864 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.7723, - "falseNegative": 63, - "falsePositive": 6, - "precision": 0.9512, - "predictedPositive": 123, - "recall": 0.65, - "total": 2970, - "trueNegative": 2784, - "truePositive": 117, - "wilsonPrecisionLower95": 0.8977 - }, - "threshold": 0.765 - }, - "followUpReminder": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8483, - "falseNegative": 43, - "falsePositive": 6, - "precision": 0.958, - "predictedPositive": 143, - "recall": 0.7611, - "total": 1485, - "trueNegative": 1299, - "truePositive": 137, - "wilsonPrecisionLower95": 0.9115 - }, - "threshold": 0.995 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.3529, - "falseNegative": 141, - "falsePositive": 2, - "precision": 0.9512, - "predictedPositive": 41, - "recall": 0.2167, - "total": 1485, - "trueNegative": 1303, - "truePositive": 39, - "wilsonPrecisionLower95": 0.8386 - }, - "threshold": 0.99 - } - }, - "metrics": { - "f1": 0.6292, - "falseNegative": 192, - "falsePositive": 6, - "precision": 0.9655, - "predictedPositive": 174, - "recall": 0.4667, - "total": 2970, - "trueNegative": 2604, - "truePositive": 168, - "wilsonPrecisionLower95": 0.9268 - }, - "threshold": 0.995 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.3894, - "falseNegative": 68, - "falsePositive": 1, - "precision": 0.9565, - "predictedPositive": 23, - "recall": 0.2444, - "total": 1485, - "trueNegative": 1394, - "truePositive": 22, - "wilsonPrecisionLower95": 0.7901 - }, - "threshold": 0.98 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8175, - "falseNegative": 91, - "falsePositive": 9, - "precision": 0.9614, - "predictedPositive": 233, - "recall": 0.7111, - "total": 1485, - "trueNegative": 1161, - "truePositive": 224, - "wilsonPrecisionLower95": 0.9282 - }, - "threshold": 0.6 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.7182, - "falseNegative": 401, - "falsePositive": 26, - "precision": 0.9544, - "predictedPositive": 570, - "recall": 0.5757, - "total": 1485, - "trueNegative": 514, - "truePositive": 544, - "wilsonPrecisionLower95": 0.934 - }, - "threshold": 0.985 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6667, - "falseNegative": 460, - "falsePositive": 25, - "precision": 0.951, - "predictedPositive": 510, - "recall": 0.5132, - "total": 1485, - "trueNegative": 515, - "truePositive": 485, - "wilsonPrecisionLower95": 0.9286 - }, - "threshold": 0.945 - } - }, - "metrics": { - "f1": 0.7151, - "falseNegative": 807, - "falsePositive": 56, - "precision": 0.9508, - "predictedPositive": 1139, - "recall": 0.573, - "total": 2970, - "trueNegative": 1024, - "truePositive": 1083, - "wilsonPrecisionLower95": 0.9367 - }, - "threshold": 0.96 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9412, - "falseNegative": 5, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 40, - "recall": 0.8889, - "total": 1485, - "trueNegative": 1440, - "truePositive": 40, - "wilsonPrecisionLower95": 0.9124 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8052, - "falseNegative": 14, - "falsePositive": 1, - "precision": 0.9688, - "predictedPositive": 32, - "recall": 0.6889, - "total": 1485, - "trueNegative": 1439, - "truePositive": 31, - "wilsonPrecisionLower95": 0.8426 - }, - "threshold": 0.98 - } - }, - "metrics": { - "f1": 0.8375, - "falseNegative": 23, - "falsePositive": 3, - "precision": 0.9571, - "predictedPositive": 70, - "recall": 0.7444, - "total": 2970, - "trueNegative": 2877, - "truePositive": 67, - "wilsonPrecisionLower95": 0.8814 - }, - "threshold": 0.92 - }, - "task": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.7783, - "falseNegative": 123, - "falsePositive": 12, - "precision": 0.9518, - "predictedPositive": 249, - "recall": 0.6583, - "total": 1485, - "trueNegative": 1113, - "truePositive": 237, - "wilsonPrecisionLower95": 0.9177 - }, - "threshold": 0.84 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7994, - "falseNegative": 113, - "falsePositive": 11, - "precision": 0.9574, - "predictedPositive": 258, - "recall": 0.6861, - "total": 1485, - "trueNegative": 1114, - "truePositive": 247, - "wilsonPrecisionLower95": 0.9253 - }, - "threshold": 0.755 - } - }, - "metrics": { - "f1": 0.7977, - "falseNegative": 225, - "falsePositive": 26, - "precision": 0.9501, - "predictedPositive": 521, - "recall": 0.6875, - "total": 2970, - "trueNegative": 2224, - "truePositive": 495, - "wilsonPrecisionLower95": 0.9279 - }, - "threshold": 0.77 - } - }, - "trainingRecords": 17763, - "validation": { - "intentsMeeting95Precision": 7, - "macroF1": 0.4424, - "macroPrecision": 0.9693, - "macroRecall": 0.3051, - "minimumPrecision": 0.8889, - "perIntent": { - "blessing": { - "f1": 0.1818, - "falseNegative": 81, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 9, - "recall": 0.1, - "total": 2970, - "trueNegative": 2880, - "truePositive": 9, - "wilsonPrecisionLower95": 0.7009 - }, - "complaint": { - "f1": 0.5414, - "falseNegative": 226, - "falsePositive": 1, - "precision": 0.9926, - "predictedPositive": 135, - "recall": 0.3722, - "total": 2970, - "trueNegative": 2609, - "truePositive": 134, - "wilsonPrecisionLower95": 0.9592 - }, - "confirmationDecision": { - "f1": 0.5946, - "falseNegative": 103, - "falsePositive": 2, - "precision": 0.9747, - "predictedPositive": 79, - "recall": 0.4278, - "total": 2970, - "trueNegative": 2788, - "truePositive": 77, - "wilsonPrecisionLower95": 0.9123 - }, - "followUpReminder": { - "f1": 0.1975, - "falseNegative": 320, - "falsePositive": 5, - "precision": 0.8889, - "predictedPositive": 45, - "recall": 0.1111, - "total": 2970, - "trueNegative": 2605, - "truePositive": 40, - "wilsonPrecisionLower95": 0.765 - }, - "invitation": { - "f1": 0.209, - "falseNegative": 159, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 21, - "recall": 0.1167, - "total": 2970, - "trueNegative": 2790, - "truePositive": 21, - "wilsonPrecisionLower95": 0.8454 - }, - "question": { - "f1": 0.5216, - "falseNegative": 407, - "falsePositive": 2, - "precision": 0.9911, - "predictedPositive": 225, - "recall": 0.354, - "total": 2970, - "trueNegative": 2338, - "truePositive": 223, - "wilsonPrecisionLower95": 0.9682 - }, - "replyableMessage": { - "f1": 0.6929, - "falseNegative": 861, - "falsePositive": 51, - "precision": 0.9528, - "predictedPositive": 1080, - "recall": 0.5444, - "total": 2970, - "trueNegative": 1029, - "truePositive": 1029, - "wilsonPrecisionLower95": 0.9384 - }, - "scheduleNegotiation": { - "f1": 0.4706, - "falseNegative": 62, - "falsePositive": 1, - "precision": 0.9655, - "predictedPositive": 29, - "recall": 0.3111, - "total": 2970, - "trueNegative": 2879, - "truePositive": 28, - "wilsonPrecisionLower95": 0.8282 - }, - "task": { - "f1": 0.5725, - "falseNegative": 426, - "falsePositive": 13, - "precision": 0.9577, - "predictedPositive": 307, - "recall": 0.4083, - "total": 2970, - "trueNegative": 2237, - "truePositive": 294, - "wilsonPrecisionLower95": 0.9289 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 5e-06, - "augmentation": "prefix", - "char_max": 6, - "char_min": 2, - "external_weight": 0.77, - "hard_example_weight": 1.8, - "l1_ratio": 0.1, - "max_char_features": 70000, - "max_word_features": 30000, - "min_df": 2, - "pseudo_weight": 0.31499999999999995, - "round": 11, - "word_max": 2 - }, - "featureCount": 81451, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 0 - }, - "complaint": { - "negative": 250, - "positive": 0 - }, - "confirmationDecision": { - "negative": 250, - "positive": 14 - }, - "followUpReminder": { - "negative": 250, - "positive": 2 - }, - "invitation": { - "negative": 250, - "positive": 1 - }, - "question": { - "negative": 233, - "positive": 0 - }, - "replyableMessage": { - "negative": 8, - "positive": 250 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 0 - }, - "task": { - "negative": 250, - "positive": 0 - } - }, - "round": 11, - "selectionObjective": 0.6682, - "syntheticRegression": { - "intentsMeeting95Precision": 6, - "macroF1": 0.5232, - "macroPrecision": 0.9846, - "macroRecall": 0.4153, - "minimumPrecision": 0.93, - "perIntent": { - "blessing": { - "f1": 0.125, - "falseNegative": 84, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 6, - "recall": 0.0667, - "total": 2970, - "trueNegative": 2880, - "truePositive": 6, - "wilsonPrecisionLower95": 0.6097 - }, - "complaint": { - "f1": 0.4245, - "falseNegative": 263, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 97, - "recall": 0.2694, - "total": 2970, - "trueNegative": 2610, - "truePositive": 97, - "wilsonPrecisionLower95": 0.9619 - }, - "confirmationDecision": { - "f1": 0.6643, - "falseNegative": 87, - "falsePositive": 7, - "precision": 0.93, - "predictedPositive": 100, - "recall": 0.5167, - "total": 2970, - "trueNegative": 2783, - "truePositive": 93, - "wilsonPrecisionLower95": 0.8625 - }, - "followUpReminder": { - "f1": 0.0902, - "falseNegative": 343, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 17, - "recall": 0.0472, - "total": 2970, - "trueNegative": 2610, - "truePositive": 17, - "wilsonPrecisionLower95": 0.8157 - }, - "invitation": { - "f1": 0.341, - "falseNegative": 143, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 37, - "recall": 0.2056, - "total": 2970, - "trueNegative": 2790, - "truePositive": 37, - "wilsonPrecisionLower95": 0.9059 - }, - "question": { - "f1": 0.8191, - "falseNegative": 193, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 437, - "recall": 0.6937, - "total": 2970, - "trueNegative": 2340, - "truePositive": 437, - "wilsonPrecisionLower95": 0.9913 - }, - "replyableMessage": { - "f1": 0.8975, - "falseNegative": 253, - "falsePositive": 121, - "precision": 0.9312, - "predictedPositive": 1758, - "recall": 0.8661, - "total": 2970, - "trueNegative": 959, - "truePositive": 1637, - "wilsonPrecisionLower95": 0.9184 - }, - "scheduleNegotiation": { - "f1": 0.8535, - "falseNegative": 23, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 67, - "recall": 0.7444, - "total": 2970, - "trueNegative": 2880, - "truePositive": 67, - "wilsonPrecisionLower95": 0.9458 - }, - "task": { - "f1": 0.4937, - "falseNegative": 484, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 236, - "recall": 0.3278, - "total": 2970, - "trueNegative": 2250, - "truePositive": 236, - "wilsonPrecisionLower95": 0.984 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9917, - "falseNegative": 0, - "falsePositive": 3, - "precision": 0.9836, - "predictedPositive": 183, - "recall": 1.0, - "total": 1485, - "trueNegative": 1302, - "truePositive": 180, - "wilsonPrecisionLower95": 0.9529 - }, - "threshold": 0.99 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9318, - "falseNegative": 16, - "falsePositive": 8, - "precision": 0.9535, - "predictedPositive": 172, - "recall": 0.9111, - "total": 1485, - "trueNegative": 1297, - "truePositive": 164, - "wilsonPrecisionLower95": 0.9109 - }, - "threshold": 0.84 - } - }, - "metrics": { - "f1": 0.9529, - "falseNegative": 16, - "falsePositive": 18, - "precision": 0.9503, - "predictedPositive": 362, - "recall": 0.9556, - "total": 2970, - "trueNegative": 2592, - "truePositive": 344, - "wilsonPrecisionLower95": 0.9228 - }, - "threshold": 0.84 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.931, - "falseNegative": 9, - "falsePositive": 3, - "precision": 0.9643, - "predictedPositive": 84, - "recall": 0.9, - "total": 1485, - "trueNegative": 1392, - "truePositive": 81, - "wilsonPrecisionLower95": 0.9002 - }, - "threshold": 0.97 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7222, - "falseNegative": 38, - "falsePositive": 2, - "precision": 0.963, - "predictedPositive": 54, - "recall": 0.5778, - "total": 1485, - "trueNegative": 1393, - "truePositive": 52, - "wilsonPrecisionLower95": 0.8746 - }, - "threshold": 0.86 - } - }, - "metrics": { - "f1": 0.8077, - "falseNegative": 54, - "falsePositive": 6, - "precision": 0.9545, - "predictedPositive": 132, - "recall": 0.7, - "total": 2970, - "trueNegative": 2784, - "truePositive": 126, - "wilsonPrecisionLower95": 0.9044 - }, - "threshold": 0.95 - }, - "followUpReminder": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.4833, - "falseNegative": 61, - "falsePositive": 1, - "precision": 0.9667, - "predictedPositive": 30, - "recall": 0.3222, - "total": 1485, - "trueNegative": 1394, - "truePositive": 29, - "wilsonPrecisionLower95": 0.8333 - }, - "threshold": 0.965 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.7993, - "falseNegative": 98, - "falsePositive": 11, - "precision": 0.9518, - "predictedPositive": 228, - "recall": 0.6889, - "total": 1485, - "trueNegative": 1159, - "truePositive": 217, - "wilsonPrecisionLower95": 0.9157 - }, - "threshold": 0.92 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7572, - "falseNegative": 117, - "falsePositive": 10, - "precision": 0.9519, - "predictedPositive": 208, - "recall": 0.6286, - "total": 1485, - "trueNegative": 1160, - "truePositive": 198, - "wilsonPrecisionLower95": 0.9138 - }, - "threshold": 0.735 - } - }, - "metrics": { - "f1": 0.797, - "falseNegative": 198, - "falsePositive": 22, - "precision": 0.9515, - "predictedPositive": 454, - "recall": 0.6857, - "total": 2970, - "trueNegative": 2318, - "truePositive": 432, - "wilsonPrecisionLower95": 0.9277 - }, - "threshold": 0.8 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9373, - "falseNegative": 71, - "falsePositive": 46, - "precision": 0.95, - "predictedPositive": 920, - "recall": 0.9249, - "total": 1485, - "trueNegative": 494, - "truePositive": 874, - "wilsonPrecisionLower95": 0.934 - }, - "threshold": 0.79 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8754, - "falseNegative": 179, - "falsePositive": 39, - "precision": 0.9516, - "predictedPositive": 805, - "recall": 0.8106, - "total": 1485, - "trueNegative": 501, - "truePositive": 766, - "wilsonPrecisionLower95": 0.9345 - }, - "threshold": 0.91 - } - }, - "metrics": { - "f1": 0.9105, - "falseNegative": 242, - "falsePositive": 82, - "precision": 0.9526, - "predictedPositive": 1730, - "recall": 0.872, - "total": 2970, - "trueNegative": 998, - "truePositive": 1648, - "wilsonPrecisionLower95": 0.9415 - }, - "threshold": 0.895 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9773, - "falseNegative": 2, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 43, - "recall": 0.9556, - "total": 1485, - "trueNegative": 1440, - "truePositive": 43, - "wilsonPrecisionLower95": 0.918 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9318, - "falseNegative": 4, - "falsePositive": 2, - "precision": 0.9535, - "predictedPositive": 43, - "recall": 0.9111, - "total": 1485, - "trueNegative": 1438, - "truePositive": 41, - "wilsonPrecisionLower95": 0.8454 - }, - "threshold": 0.9 - } - }, - "metrics": { - "f1": 0.9551, - "falseNegative": 5, - "falsePositive": 3, - "precision": 0.9659, - "predictedPositive": 88, - "recall": 0.9444, - "total": 2970, - "trueNegative": 2877, - "truePositive": 85, - "wilsonPrecisionLower95": 0.9045 - }, - "threshold": 0.885 - }, - "task": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8315, - "falseNegative": 96, - "falsePositive": 11, - "precision": 0.96, - "predictedPositive": 275, - "recall": 0.7333, - "total": 1485, - "trueNegative": 1114, - "truePositive": 264, - "wilsonPrecisionLower95": 0.9298 - }, - "threshold": 0.985 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8349, - "falseNegative": 92, - "falsePositive": 14, - "precision": 0.9504, - "predictedPositive": 282, - "recall": 0.7444, - "total": 1485, - "trueNegative": 1111, - "truePositive": 268, - "wilsonPrecisionLower95": 0.9184 - }, - "threshold": 0.845 - } - }, - "metrics": { - "f1": 0.7688, - "falseNegative": 256, - "falsePositive": 23, - "precision": 0.9528, - "predictedPositive": 487, - "recall": 0.6444, - "total": 2970, - "trueNegative": 2227, - "truePositive": 464, - "wilsonPrecisionLower95": 0.9301 - }, - "threshold": 0.975 - } - }, - "trainingRecords": 15345, - "validation": { - "intentsMeeting95Precision": 8, - "macroF1": 0.5184, - "macroPrecision": 0.982, - "macroRecall": 0.3943, - "minimumPrecision": 0.9412, - "perIntent": { - "blessing": { - "f1": 0.2692, - "falseNegative": 76, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 14, - "recall": 0.1556, - "total": 2970, - "trueNegative": 2880, - "truePositive": 14, - "wilsonPrecisionLower95": 0.7847 - }, - "complaint": { - "f1": 0.5542, - "falseNegative": 222, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 138, - "recall": 0.3833, - "total": 2970, - "trueNegative": 2610, - "truePositive": 138, - "wilsonPrecisionLower95": 0.9729 - }, - "confirmationDecision": { - "f1": 0.6642, - "falseNegative": 90, - "falsePositive": 1, - "precision": 0.989, - "predictedPositive": 91, - "recall": 0.5, - "total": 2970, - "trueNegative": 2789, - "truePositive": 90, - "wilsonPrecisionLower95": 0.9404 - }, - "followUpReminder": { - "f1": 0.1347, - "falseNegative": 334, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 26, - "recall": 0.0722, - "total": 2970, - "trueNegative": 2610, - "truePositive": 26, - "wilsonPrecisionLower95": 0.8713 - }, - "invitation": { - "f1": 0.2524, - "falseNegative": 154, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 26, - "recall": 0.1444, - "total": 2970, - "trueNegative": 2790, - "truePositive": 26, - "wilsonPrecisionLower95": 0.8713 - }, - "question": { - "f1": 0.7757, - "falseNegative": 222, - "falsePositive": 14, - "precision": 0.9668, - "predictedPositive": 422, - "recall": 0.6476, - "total": 2970, - "trueNegative": 2326, - "truePositive": 408, - "wilsonPrecisionLower95": 0.9451 - }, - "replyableMessage": { - "f1": 0.9073, - "falseNegative": 250, - "falsePositive": 85, - "precision": 0.9507, - "predictedPositive": 1725, - "recall": 0.8677, - "total": 2970, - "trueNegative": 995, - "truePositive": 1640, - "wilsonPrecisionLower95": 0.9395 - }, - "scheduleNegotiation": { - "f1": 0.5161, - "falseNegative": 58, - "falsePositive": 2, - "precision": 0.9412, - "predictedPositive": 34, - "recall": 0.3556, - "total": 2970, - "trueNegative": 2878, - "truePositive": 32, - "wilsonPrecisionLower95": 0.8091 - }, - "task": { - "f1": 0.592, - "falseNegative": 416, - "falsePositive": 3, - "precision": 0.9902, - "predictedPositive": 307, - "recall": 0.4222, - "total": 2970, - "trueNegative": 2247, - "truePositive": 304, - "wilsonPrecisionLower95": 0.9717 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 4e-06, - "augmentation": "prefix", - "char_max": 6, - "char_min": 3, - "external_weight": 0.79, - "hard_example_weight": 2.0, - "l1_ratio": 0.15, - "max_char_features": 75000, - "max_word_features": 35000, - "min_df": 2, - "pseudo_weight": 0.32999999999999996, - "round": 12, - "word_max": 3 - }, - "featureCount": 94458, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 0 - }, - "complaint": { - "negative": 250, - "positive": 0 - }, - "confirmationDecision": { - "negative": 250, - "positive": 15 - }, - "followUpReminder": { - "negative": 250, - "positive": 0 - }, - "invitation": { - "negative": 250, - "positive": 0 - }, - "question": { - "negative": 232, - "positive": 0 - }, - "replyableMessage": { - "negative": 10, - "positive": 250 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 0 - }, - "task": { - "negative": 250, - "positive": 0 - } - }, - "round": 12, - "selectionObjective": 0.6224, - "syntheticRegression": { - "intentsMeeting95Precision": 7, - "macroF1": 0.4599, - "macroPrecision": 0.9738, - "macroRecall": 0.3356, - "minimumPrecision": 0.8053, - "perIntent": { - "blessing": { - "f1": 0.1053, - "falseNegative": 85, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 5, - "recall": 0.0556, - "total": 2970, - "trueNegative": 2880, - "truePositive": 5, - "wilsonPrecisionLower95": 0.5655 - }, - "complaint": { - "f1": 0.3964, - "falseNegative": 271, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 89, - "recall": 0.2472, - "total": 2970, - "trueNegative": 2610, - "truePositive": 89, - "wilsonPrecisionLower95": 0.9586 - }, - "confirmationDecision": { - "f1": 0.6212, - "falseNegative": 89, - "falsePositive": 22, - "precision": 0.8053, - "predictedPositive": 113, - "recall": 0.5056, - "total": 2970, - "trueNegative": 2768, - "truePositive": 91, - "wilsonPrecisionLower95": 0.7228 - }, - "followUpReminder": { - "f1": 0.2651, - "falseNegative": 305, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 55, - "recall": 0.1528, - "total": 2970, - "trueNegative": 2610, - "truePositive": 55, - "wilsonPrecisionLower95": 0.9347 - }, - "invitation": { - "f1": 0.341, - "falseNegative": 143, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 37, - "recall": 0.2056, - "total": 2970, - "trueNegative": 2790, - "truePositive": 37, - "wilsonPrecisionLower95": 0.9059 - }, - "question": { - "f1": 0.427, - "falseNegative": 459, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 171, - "recall": 0.2714, - "total": 2970, - "trueNegative": 2340, - "truePositive": 171, - "wilsonPrecisionLower95": 0.978 - }, - "replyableMessage": { - "f1": 0.7888, - "falseNegative": 624, - "falsePositive": 54, - "precision": 0.9591, - "predictedPositive": 1320, - "recall": 0.6698, - "total": 2970, - "trueNegative": 1026, - "truePositive": 1266, - "wilsonPrecisionLower95": 0.947 - }, - "scheduleNegotiation": { - "f1": 0.8, - "falseNegative": 30, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 60, - "recall": 0.6667, - "total": 2970, - "trueNegative": 2880, - "truePositive": 60, - "wilsonPrecisionLower95": 0.9398 - }, - "task": { - "f1": 0.3946, - "falseNegative": 543, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 177, - "recall": 0.2458, - "total": 2970, - "trueNegative": 2250, - "truePositive": 177, - "wilsonPrecisionLower95": 0.9788 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9917, - "falseNegative": 0, - "falsePositive": 3, - "precision": 0.9836, - "predictedPositive": 183, - "recall": 1.0, - "total": 1485, - "trueNegative": 1302, - "truePositive": 180, - "wilsonPrecisionLower95": 0.9529 - }, - "threshold": 0.885 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8974, - "falseNegative": 27, - "falsePositive": 8, - "precision": 0.9503, - "predictedPositive": 161, - "recall": 0.85, - "total": 1485, - "trueNegative": 1297, - "truePositive": 153, - "wilsonPrecisionLower95": 0.905 - }, - "threshold": 0.9 - } - }, - "metrics": { - "f1": 0.9515, - "falseNegative": 17, - "falsePositive": 18, - "precision": 0.9501, - "predictedPositive": 361, - "recall": 0.9528, - "total": 2970, - "trueNegative": 2592, - "truePositive": 343, - "wilsonPrecisionLower95": 0.9226 - }, - "threshold": 0.77 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9944, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 89, - "recall": 0.9889, - "total": 1485, - "trueNegative": 1395, - "truePositive": 89, - "wilsonPrecisionLower95": 0.9586 - }, - "threshold": 0.915 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.4959, - "falseNegative": 60, - "falsePositive": 1, - "precision": 0.9677, - "predictedPositive": 31, - "recall": 0.3333, - "total": 1485, - "trueNegative": 1394, - "truePositive": 30, - "wilsonPrecisionLower95": 0.8381 - }, - "threshold": 0.985 - } - }, - "metrics": { - "f1": 0.8217, - "falseNegative": 51, - "falsePositive": 5, - "precision": 0.9627, - "predictedPositive": 134, - "recall": 0.7167, - "total": 2970, - "trueNegative": 2785, - "truePositive": 129, - "wilsonPrecisionLower95": 0.9156 - }, - "threshold": 0.915 - }, - "followUpReminder": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.5647, - "falseNegative": 108, - "falsePositive": 3, - "precision": 0.96, - "predictedPositive": 75, - "recall": 0.4, - "total": 1485, - "trueNegative": 1302, - "truePositive": 72, - "wilsonPrecisionLower95": 0.8889 - }, - "threshold": 0.995 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6542, - "falseNegative": 158, - "falsePositive": 8, - "precision": 0.9515, - "predictedPositive": 165, - "recall": 0.4984, - "total": 1485, - "trueNegative": 1162, - "truePositive": 157, - "wilsonPrecisionLower95": 0.9073 - }, - "threshold": 0.805 - } - }, - "metrics": { - "f1": 0.5711, - "falseNegative": 373, - "falsePositive": 13, - "precision": 0.9519, - "predictedPositive": 270, - "recall": 0.4079, - "total": 2970, - "trueNegative": 2327, - "truePositive": 257, - "wilsonPrecisionLower95": 0.9194 - }, - "threshold": 0.985 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8956, - "falseNegative": 147, - "falsePositive": 39, - "precision": 0.9534, - "predictedPositive": 837, - "recall": 0.8444, - "total": 1485, - "trueNegative": 501, - "truePositive": 798, - "wilsonPrecisionLower95": 0.9369 - }, - "threshold": 0.975 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6735, - "falseNegative": 453, - "falsePositive": 24, - "precision": 0.9535, - "predictedPositive": 516, - "recall": 0.5206, - "total": 1485, - "trueNegative": 516, - "truePositive": 492, - "wilsonPrecisionLower95": 0.9317 - }, - "threshold": 0.985 - } - }, - "metrics": { - "f1": 0.7866, - "falseNegative": 626, - "falsePositive": 60, - "precision": 0.9547, - "predictedPositive": 1324, - "recall": 0.6688, - "total": 2970, - "trueNegative": 1020, - "truePositive": 1264, - "wilsonPrecisionLower95": 0.9421 - }, - "threshold": 0.985 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9888, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 44, - "recall": 0.9778, - "total": 1485, - "trueNegative": 1440, - "truePositive": 44, - "wilsonPrecisionLower95": 0.9197 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8052, - "falseNegative": 14, - "falsePositive": 1, - "precision": 0.9688, - "predictedPositive": 32, - "recall": 0.6889, - "total": 1485, - "trueNegative": 1439, - "truePositive": 31, - "wilsonPrecisionLower95": 0.8426 - }, - "threshold": 0.94 - } - }, - "metrics": { - "f1": 0.9364, - "falseNegative": 9, - "falsePositive": 2, - "precision": 0.9759, - "predictedPositive": 83, - "recall": 0.9, - "total": 2970, - "trueNegative": 2878, - "truePositive": 81, - "wilsonPrecisionLower95": 0.9163 - }, - "threshold": 0.715 - }, - "task": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8475, - "falseNegative": 85, - "falsePositive": 14, - "precision": 0.9516, - "predictedPositive": 289, - "recall": 0.7639, - "total": 1485, - "trueNegative": 1111, - "truePositive": 275, - "wilsonPrecisionLower95": 0.9203 - }, - "threshold": 0.985 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.501, - "falseNegative": 238, - "falsePositive": 5, - "precision": 0.9606, - "predictedPositive": 127, - "recall": 0.3389, - "total": 1485, - "trueNegative": 1120, - "truePositive": 122, - "wilsonPrecisionLower95": 0.9111 - }, - "threshold": 0.985 - } - }, - "metrics": { - "f1": 0.6989, - "falseNegative": 323, - "falsePositive": 19, - "precision": 0.9543, - "predictedPositive": 416, - "recall": 0.5514, - "total": 2970, - "trueNegative": 2231, - "truePositive": 397, - "wilsonPrecisionLower95": 0.9298 - }, - "threshold": 0.985 - } - }, - "trainingRecords": 15345, - "validation": { - "intentsMeeting95Precision": 9, - "macroF1": 0.445, - "macroPrecision": 0.9912, - "macroRecall": 0.307, - "minimumPrecision": 0.9534, - "perIntent": { - "blessing": { - "f1": 0.2857, - "falseNegative": 75, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 15, - "recall": 0.1667, - "total": 2970, - "trueNegative": 2880, - "truePositive": 15, - "wilsonPrecisionLower95": 0.7961 - }, - "complaint": { - "f1": 0.5355, - "falseNegative": 228, - "falsePositive": 1, - "precision": 0.9925, - "predictedPositive": 133, - "recall": 0.3667, - "total": 2970, - "trueNegative": 2609, - "truePositive": 132, - "wilsonPrecisionLower95": 0.9586 - }, - "confirmationDecision": { - "f1": 0.5827, - "falseNegative": 106, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 74, - "recall": 0.4111, - "total": 2970, - "trueNegative": 2790, - "truePositive": 74, - "wilsonPrecisionLower95": 0.9507 - }, - "followUpReminder": { - "f1": 0.2045, - "falseNegative": 319, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 41, - "recall": 0.1139, - "total": 2970, - "trueNegative": 2610, - "truePositive": 41, - "wilsonPrecisionLower95": 0.9143 - }, - "invitation": { - "f1": 0.2266, - "falseNegative": 157, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 23, - "recall": 0.1278, - "total": 2970, - "trueNegative": 2790, - "truePositive": 23, - "wilsonPrecisionLower95": 0.8569 - }, - "question": { - "f1": 0.42, - "falseNegative": 462, - "falsePositive": 2, - "precision": 0.9882, - "predictedPositive": 170, - "recall": 0.2667, - "total": 2970, - "trueNegative": 2338, - "truePositive": 168, - "wilsonPrecisionLower95": 0.9581 - }, - "replyableMessage": { - "f1": 0.7956, - "falseNegative": 600, - "falsePositive": 63, - "precision": 0.9534, - "predictedPositive": 1353, - "recall": 0.6825, - "total": 2970, - "trueNegative": 1017, - "truePositive": 1290, - "wilsonPrecisionLower95": 0.9409 - }, - "scheduleNegotiation": { - "f1": 0.4874, - "falseNegative": 61, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 29, - "recall": 0.3222, - "total": 2970, - "trueNegative": 2880, - "truePositive": 29, - "wilsonPrecisionLower95": 0.883 - }, - "task": { - "f1": 0.4666, - "falseNegative": 500, - "falsePositive": 3, - "precision": 0.9865, - "predictedPositive": 223, - "recall": 0.3056, - "total": 2970, - "trueNegative": 2247, - "truePositive": 220, - "wilsonPrecisionLower95": 0.9612 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 3e-06, - "augmentation": "prefix", - "char_max": 7, - "char_min": 2, - "external_weight": 0.81, - "hard_example_weight": 2.0, - "l1_ratio": 0.15, - "max_char_features": 80000, - "max_word_features": 38000, - "min_df": 2, - "pseudo_weight": 0.345, - "round": 13, - "word_max": 3 - }, - "featureCount": 99458, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 0 - }, - "complaint": { - "negative": 250, - "positive": 1 - }, - "confirmationDecision": { - "negative": 250, - "positive": 18 - }, - "followUpReminder": { - "negative": 250, - "positive": 0 - }, - "invitation": { - "negative": 250, - "positive": 0 - }, - "question": { - "negative": 250, - "positive": 0 - }, - "replyableMessage": { - "negative": 12, - "positive": 250 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 0 - }, - "task": { - "negative": 250, - "positive": 0 - } - }, - "round": 13, - "selectionObjective": 0.6381, - "syntheticRegression": { - "intentsMeeting95Precision": 6, - "macroF1": 0.4906, - "macroPrecision": 0.9736, - "macroRecall": 0.3748, - "minimumPrecision": 0.8182, - "perIntent": { - "blessing": { - "f1": 0.1443, - "falseNegative": 83, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 7, - "recall": 0.0778, - "total": 2970, - "trueNegative": 2880, - "truePositive": 7, - "wilsonPrecisionLower95": 0.6457 - }, - "complaint": { - "f1": 0.4279, - "falseNegative": 262, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 98, - "recall": 0.2722, - "total": 2970, - "trueNegative": 2610, - "truePositive": 98, - "wilsonPrecisionLower95": 0.9623 - }, - "confirmationDecision": { - "f1": 0.6207, - "falseNegative": 90, - "falsePositive": 20, - "precision": 0.8182, - "predictedPositive": 110, - "recall": 0.5, - "total": 2970, - "trueNegative": 2770, - "truePositive": 90, - "wilsonPrecisionLower95": 0.7358 - }, - "followUpReminder": { - "f1": 0.1201, - "falseNegative": 337, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 23, - "recall": 0.0639, - "total": 2970, - "trueNegative": 2610, - "truePositive": 23, - "wilsonPrecisionLower95": 0.8569 - }, - "invitation": { - "f1": 0.3486, - "falseNegative": 142, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 38, - "recall": 0.2111, - "total": 2970, - "trueNegative": 2790, - "truePositive": 38, - "wilsonPrecisionLower95": 0.9082 - }, - "question": { - "f1": 0.5124, - "falseNegative": 413, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 217, - "recall": 0.3444, - "total": 2970, - "trueNegative": 2340, - "truePositive": 217, - "wilsonPrecisionLower95": 0.9826 - }, - "replyableMessage": { - "f1": 0.9179, - "falseNegative": 202, - "falsePositive": 100, - "precision": 0.9441, - "predictedPositive": 1788, - "recall": 0.8931, - "total": 2970, - "trueNegative": 980, - "truePositive": 1688, - "wilsonPrecisionLower95": 0.9324 - }, - "scheduleNegotiation": { - "f1": 0.7755, - "falseNegative": 33, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 57, - "recall": 0.6333, - "total": 2970, - "trueNegative": 2880, - "truePositive": 57, - "wilsonPrecisionLower95": 0.9369 - }, - "task": { - "f1": 0.5484, - "falseNegative": 448, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 272, - "recall": 0.3778, - "total": 2970, - "trueNegative": 2250, - "truePositive": 272, - "wilsonPrecisionLower95": 0.9861 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9727, - "falseNegative": 2, - "falsePositive": 8, - "precision": 0.957, - "predictedPositive": 186, - "recall": 0.9889, - "total": 1485, - "trueNegative": 1297, - "truePositive": 178, - "wilsonPrecisionLower95": 0.9174 - }, - "threshold": 0.98 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.95, - "falseNegative": 9, - "falsePositive": 9, - "precision": 0.95, - "predictedPositive": 180, - "recall": 0.95, - "total": 1485, - "trueNegative": 1296, - "truePositive": 171, - "wilsonPrecisionLower95": 0.9077 - }, - "threshold": 0.905 - } - }, - "metrics": { - "f1": 0.9529, - "falseNegative": 16, - "falsePositive": 18, - "precision": 0.9503, - "predictedPositive": 362, - "recall": 0.9556, - "total": 2970, - "trueNegative": 2592, - "truePositive": 344, - "wilsonPrecisionLower95": 0.9228 - }, - "threshold": 0.965 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.828, - "falseNegative": 25, - "falsePositive": 2, - "precision": 0.9701, - "predictedPositive": 67, - "recall": 0.7222, - "total": 1485, - "trueNegative": 1393, - "truePositive": 65, - "wilsonPrecisionLower95": 0.8975 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6763, - "falseNegative": 43, - "falsePositive": 2, - "precision": 0.9592, - "predictedPositive": 49, - "recall": 0.5222, - "total": 1485, - "trueNegative": 1393, - "truePositive": 47, - "wilsonPrecisionLower95": 0.8629 - }, - "threshold": 0.93 - } - }, - "metrics": { - "f1": 0.7855, - "falseNegative": 61, - "falsePositive": 4, - "precision": 0.9675, - "predictedPositive": 123, - "recall": 0.6611, - "total": 2970, - "trueNegative": 2786, - "truePositive": 119, - "wilsonPrecisionLower95": 0.9194 - }, - "threshold": 0.82 - }, - "followUpReminder": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7452, - "falseNegative": 122, - "falsePositive": 10, - "precision": 0.9507, - "predictedPositive": 203, - "recall": 0.6127, - "total": 1485, - "trueNegative": 1160, - "truePositive": 193, - "wilsonPrecisionLower95": 0.9117 - }, - "threshold": 0.915 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9441, - "falseNegative": 59, - "falsePositive": 46, - "precision": 0.9506, - "predictedPositive": 932, - "recall": 0.9376, - "total": 1485, - "trueNegative": 494, - "truePositive": 886, - "wilsonPrecisionLower95": 0.9348 - }, - "threshold": 0.935 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9067, - "falseNegative": 129, - "falsePositive": 39, - "precision": 0.9544, - "predictedPositive": 855, - "recall": 0.8635, - "total": 1485, - "trueNegative": 501, - "truePositive": 816, - "wilsonPrecisionLower95": 0.9383 - }, - "threshold": 0.905 - } - }, - "metrics": { - "f1": 0.9267, - "falseNegative": 182, - "falsePositive": 88, - "precision": 0.951, - "predictedPositive": 1796, - "recall": 0.9037, - "total": 2970, - "trueNegative": 992, - "truePositive": 1708, - "wilsonPrecisionLower95": 0.94 - }, - "threshold": 0.905 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9888, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 44, - "recall": 0.9778, - "total": 1485, - "trueNegative": 1440, - "truePositive": 44, - "wilsonPrecisionLower95": 0.9197 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9655, - "falseNegative": 3, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 42, - "recall": 0.9333, - "total": 1485, - "trueNegative": 1440, - "truePositive": 42, - "wilsonPrecisionLower95": 0.9162 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.9773, - "falseNegative": 4, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 86, - "recall": 0.9556, - "total": 2970, - "trueNegative": 2880, - "truePositive": 86, - "wilsonPrecisionLower95": 0.9572 - }, - "threshold": 0.5 - }, - "task": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.7876, - "falseNegative": 119, - "falsePositive": 11, - "precision": 0.9563, - "predictedPositive": 252, - "recall": 0.6694, - "total": 1485, - "trueNegative": 1114, - "truePositive": 241, - "wilsonPrecisionLower95": 0.9235 - }, - "threshold": 0.995 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8416, - "falseNegative": 89, - "falsePositive": 13, - "precision": 0.9542, - "predictedPositive": 284, - "recall": 0.7528, - "total": 1485, - "trueNegative": 1112, - "truePositive": 271, - "wilsonPrecisionLower95": 0.9233 - }, - "threshold": 0.895 - } - }, - "metrics": { - "f1": 0.6962, - "falseNegative": 327, - "falsePositive": 16, - "precision": 0.9609, - "predictedPositive": 409, - "recall": 0.5458, - "total": 2970, - "trueNegative": 2234, - "truePositive": 393, - "wilsonPrecisionLower95": 0.9374 - }, - "threshold": 0.995 - } - }, - "trainingRecords": 15346, - "validation": { - "intentsMeeting95Precision": 9, - "macroF1": 0.4673, - "macroPrecision": 0.991, - "macroRecall": 0.3426, - "minimumPrecision": 0.9524, - "perIntent": { - "blessing": { - "f1": 0.2178, - "falseNegative": 79, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 11, - "recall": 0.1222, - "total": 2970, - "trueNegative": 2880, - "truePositive": 11, - "wilsonPrecisionLower95": 0.7412 - }, - "complaint": { - "f1": 0.5531, - "falseNegative": 222, - "falsePositive": 1, - "precision": 0.9928, - "predictedPositive": 139, - "recall": 0.3833, - "total": 2970, - "trueNegative": 2609, - "truePositive": 138, - "wilsonPrecisionLower95": 0.9604 - }, - "confirmationDecision": { - "f1": 0.5657, - "falseNegative": 109, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 71, - "recall": 0.3944, - "total": 2970, - "trueNegative": 2790, - "truePositive": 71, - "wilsonPrecisionLower95": 0.9487 - }, - "followUpReminder": { - "f1": 0.1538, - "falseNegative": 330, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 30, - "recall": 0.0833, - "total": 2970, - "trueNegative": 2610, - "truePositive": 30, - "wilsonPrecisionLower95": 0.8865 - }, - "invitation": { - "f1": 0.2, - "falseNegative": 160, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 20, - "recall": 0.1111, - "total": 2970, - "trueNegative": 2790, - "truePositive": 20, - "wilsonPrecisionLower95": 0.8389 - }, - "question": { - "f1": 0.4934, - "falseNegative": 423, - "falsePositive": 2, - "precision": 0.9904, - "predictedPositive": 209, - "recall": 0.3286, - "total": 2970, - "trueNegative": 2338, - "truePositive": 207, - "wilsonPrecisionLower95": 0.9658 - }, - "replyableMessage": { - "f1": 0.9258, - "falseNegative": 188, - "falsePositive": 85, - "precision": 0.9524, - "predictedPositive": 1787, - "recall": 0.9005, - "total": 2970, - "trueNegative": 995, - "truePositive": 1702, - "wilsonPrecisionLower95": 0.9416 - }, - "scheduleNegotiation": { - "f1": 0.5124, - "falseNegative": 59, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 31, - "recall": 0.3444, - "total": 2970, - "trueNegative": 2880, - "truePositive": 31, - "wilsonPrecisionLower95": 0.8897 - }, - "task": { - "f1": 0.584, - "falseNegative": 421, - "falsePositive": 5, - "precision": 0.9836, - "predictedPositive": 304, - "recall": 0.4153, - "total": 2970, - "trueNegative": 2245, - "truePositive": 299, - "wilsonPrecisionLower95": 0.9621 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 2.5e-06, - "augmentation": "prefix", - "char_max": 7, - "char_min": 3, - "external_weight": 0.8300000000000001, - "hard_example_weight": 2.2, - "l1_ratio": 0.2, - "max_char_features": 80000, - "max_word_features": 40000, - "min_df": 1, - "pseudo_weight": 0.36, - "round": 14, - "word_max": 3 - }, - "featureCount": 120000, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 0 - }, - "complaint": { - "negative": 250, - "positive": 8 - }, - "confirmationDecision": { - "negative": 250, - "positive": 26 - }, - "followUpReminder": { - "negative": 250, - "positive": 0 - }, - "invitation": { - "negative": 250, - "positive": 0 - }, - "question": { - "negative": 250, - "positive": 0 - }, - "replyableMessage": { - "negative": 34, - "positive": 250 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 0 - }, - "task": { - "negative": 250, - "positive": 3 - } - }, - "round": 14, - "selectionObjective": 0.6118, - "syntheticRegression": { - "intentsMeeting95Precision": 6, - "macroF1": 0.4596, - "macroPrecision": 0.9735, - "macroRecall": 0.3465, - "minimumPrecision": 0.8258, - "perIntent": { - "blessing": { - "f1": 0.1443, - "falseNegative": 83, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 7, - "recall": 0.0778, - "total": 2970, - "trueNegative": 2880, - "truePositive": 7, - "wilsonPrecisionLower95": 0.6457 - }, - "complaint": { - "f1": 0.2775, - "falseNegative": 302, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 58, - "recall": 0.1611, - "total": 2970, - "trueNegative": 2610, - "truePositive": 58, - "wilsonPrecisionLower95": 0.9379 - }, - "confirmationDecision": { - "f1": 0.6987, - "falseNegative": 71, - "falsePositive": 23, - "precision": 0.8258, - "predictedPositive": 132, - "recall": 0.6056, - "total": 2970, - "trueNegative": 2767, - "truePositive": 109, - "wilsonPrecisionLower95": 0.7521 - }, - "followUpReminder": { - "f1": 0.3991, - "falseNegative": 270, - "falsePositive": 1, - "precision": 0.989, - "predictedPositive": 91, - "recall": 0.25, - "total": 2970, - "trueNegative": 2609, - "truePositive": 90, - "wilsonPrecisionLower95": 0.9404 - }, - "invitation": { - "f1": 0.3178, - "falseNegative": 146, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 34, - "recall": 0.1889, - "total": 2970, - "trueNegative": 2790, - "truePositive": 34, - "wilsonPrecisionLower95": 0.8985 - }, - "question": { - "f1": 0.7537, - "falseNegative": 249, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 381, - "recall": 0.6048, - "total": 2970, - "trueNegative": 2340, - "truePositive": 381, - "wilsonPrecisionLower95": 0.99 - }, - "replyableMessage": { - "f1": 0.6113, - "falseNegative": 1037, - "falsePositive": 48, - "precision": 0.9467, - "predictedPositive": 901, - "recall": 0.4513, - "total": 2970, - "trueNegative": 1032, - "truePositive": 853, - "wilsonPrecisionLower95": 0.9301 - }, - "scheduleNegotiation": { - "f1": 0.8462, - "falseNegative": 24, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 66, - "recall": 0.7333, - "total": 2970, - "trueNegative": 2880, - "truePositive": 66, - "wilsonPrecisionLower95": 0.945 - }, - "task": { - "f1": 0.0876, - "falseNegative": 687, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 33, - "recall": 0.0458, - "total": 2970, - "trueNegative": 2250, - "truePositive": 33, - "wilsonPrecisionLower95": 0.8957 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9809, - "falseNegative": 0, - "falsePositive": 7, - "precision": 0.9626, - "predictedPositive": 187, - "recall": 1.0, - "total": 1485, - "trueNegative": 1298, - "truePositive": 180, - "wilsonPrecisionLower95": 0.9248 - }, - "threshold": 0.99 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8625, - "falseNegative": 21, - "falsePositive": 1, - "precision": 0.9857, - "predictedPositive": 70, - "recall": 0.7667, - "total": 1485, - "trueNegative": 1394, - "truePositive": 69, - "wilsonPrecisionLower95": 0.9234 - }, - "threshold": 0.905 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.828, - "falseNegative": 25, - "falsePositive": 2, - "precision": 0.9701, - "predictedPositive": 67, - "recall": 0.7222, - "total": 1485, - "trueNegative": 1393, - "truePositive": 65, - "wilsonPrecisionLower95": 0.8975 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.8328, - "falseNegative": 48, - "falsePositive": 5, - "precision": 0.9635, - "predictedPositive": 137, - "recall": 0.7333, - "total": 2970, - "trueNegative": 2785, - "truePositive": 132, - "wilsonPrecisionLower95": 0.9174 - }, - "threshold": 0.685 - }, - "followUpReminder": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8788, - "falseNegative": 35, - "falsePositive": 5, - "precision": 0.9667, - "predictedPositive": 150, - "recall": 0.8056, - "total": 1485, - "trueNegative": 1300, - "truePositive": 145, - "wilsonPrecisionLower95": 0.9243 - }, - "threshold": 0.845 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6113, - "falseNegative": 99, - "falsePositive": 4, - "precision": 0.9529, - "predictedPositive": 85, - "recall": 0.45, - "total": 1485, - "trueNegative": 1301, - "truePositive": 81, - "wilsonPrecisionLower95": 0.8852 - }, - "threshold": 0.97 - } - }, - "metrics": { - "f1": 0.7682, - "falseNegative": 128, - "falsePositive": 12, - "precision": 0.9508, - "predictedPositive": 244, - "recall": 0.6444, - "total": 2970, - "trueNegative": 2598, - "truePositive": 232, - "wilsonPrecisionLower95": 0.916 - }, - "threshold": 0.91 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.847, - "falseNegative": 77, - "falsePositive": 9, - "precision": 0.9636, - "predictedPositive": 247, - "recall": 0.7556, - "total": 1485, - "trueNegative": 1161, - "truePositive": 238, - "wilsonPrecisionLower95": 0.9322 - }, - "threshold": 0.995 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7168, - "falseNegative": 134, - "falsePositive": 9, - "precision": 0.9526, - "predictedPositive": 190, - "recall": 0.5746, - "total": 1485, - "trueNegative": 1161, - "truePositive": 181, - "wilsonPrecisionLower95": 0.9124 - }, - "threshold": 0.865 - } - }, - "metrics": { - "f1": 0.7524, - "falseNegative": 238, - "falsePositive": 20, - "precision": 0.9515, - "predictedPositive": 412, - "recall": 0.6222, - "total": 2970, - "trueNegative": 2320, - "truePositive": 392, - "wilsonPrecisionLower95": 0.9262 - }, - "threshold": 0.975 - }, - "replyableMessage": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9349, - "falseNegative": 76, - "falsePositive": 45, - "precision": 0.9508, - "predictedPositive": 914, - "recall": 0.9196, - "total": 1485, - "trueNegative": 495, - "truePositive": 869, - "wilsonPrecisionLower95": 0.9348 - }, - "threshold": 0.955 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 45, - "recall": 1.0, - "total": 1485, - "trueNegative": 1440, - "truePositive": 45, - "wilsonPrecisionLower95": 0.9213 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.967, - "falseNegative": 1, - "falsePositive": 2, - "precision": 0.9565, - "predictedPositive": 46, - "recall": 0.9778, - "total": 1485, - "trueNegative": 1438, - "truePositive": 44, - "wilsonPrecisionLower95": 0.8547 - }, - "threshold": 0.995 - } - }, - "metrics": { - "f1": 0.9727, - "falseNegative": 1, - "falsePositive": 4, - "precision": 0.957, - "predictedPositive": 93, - "recall": 0.9889, - "total": 2970, - "trueNegative": 2876, - "truePositive": 89, - "wilsonPrecisionLower95": 0.8946 - }, - "threshold": 0.64 - }, - "task": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "trainingRecords": 15345, - "validation": { - "intentsMeeting95Precision": 9, - "macroF1": 0.4308, - "macroPrecision": 0.9856, - "macroRecall": 0.2986, - "minimumPrecision": 0.9516, - "perIntent": { - "blessing": { - "f1": 0.3019, - "falseNegative": 74, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 16, - "recall": 0.1778, - "total": 2970, - "trueNegative": 2880, - "truePositive": 16, - "wilsonPrecisionLower95": 0.8064 - }, - "complaint": { - "f1": 0.4, - "falseNegative": 270, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 90, - "recall": 0.25, - "total": 2970, - "trueNegative": 2610, - "truePositive": 90, - "wilsonPrecisionLower95": 0.9591 - }, - "confirmationDecision": { - "f1": 0.5859, - "falseNegative": 105, - "falsePositive": 1, - "precision": 0.9868, - "predictedPositive": 76, - "recall": 0.4167, - "total": 2970, - "trueNegative": 2789, - "truePositive": 75, - "wilsonPrecisionLower95": 0.9292 - }, - "followUpReminder": { - "f1": 0.3012, - "falseNegative": 296, - "falsePositive": 1, - "precision": 0.9846, - "predictedPositive": 65, - "recall": 0.1778, - "total": 2970, - "trueNegative": 2609, - "truePositive": 64, - "wilsonPrecisionLower95": 0.9179 - }, - "invitation": { - "f1": 0.1726, - "falseNegative": 163, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 17, - "recall": 0.0944, - "total": 2970, - "trueNegative": 2790, - "truePositive": 17, - "wilsonPrecisionLower95": 0.8157 - }, - "question": { - "f1": 0.7782, - "falseNegative": 223, - "falsePositive": 9, - "precision": 0.9784, - "predictedPositive": 416, - "recall": 0.646, - "total": 2970, - "trueNegative": 2331, - "truePositive": 407, - "wilsonPrecisionLower95": 0.9594 - }, - "replyableMessage": { - "f1": 0.6272, - "falseNegative": 1006, - "falsePositive": 45, - "precision": 0.9516, - "predictedPositive": 929, - "recall": 0.4677, - "total": 2970, - "trueNegative": 1035, - "truePositive": 884, - "wilsonPrecisionLower95": 0.9358 - }, - "scheduleNegotiation": { - "f1": 0.5082, - "falseNegative": 59, - "falsePositive": 1, - "precision": 0.9688, - "predictedPositive": 32, - "recall": 0.3444, - "total": 2970, - "trueNegative": 2879, - "truePositive": 31, - "wilsonPrecisionLower95": 0.8426 - }, - "task": { - "f1": 0.2022, - "falseNegative": 639, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 81, - "recall": 0.1125, - "total": 2970, - "trueNegative": 2250, - "truePositive": 81, - "wilsonPrecisionLower95": 0.9547 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 3.5e-06, - "augmentation": "prefix", - "char_max": 6, - "char_min": 2, - "external_weight": 0.8500000000000001, - "hard_example_weight": 2.2, - "l1_ratio": 0.2, - "max_char_features": 85000, - "max_word_features": 42000, - "min_df": 1, - "pseudo_weight": 0.375, - "round": 15, - "word_max": 3 - }, - "featureCount": 127000, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 1 - }, - "complaint": { - "negative": 250, - "positive": 8 - }, - "confirmationDecision": { - "negative": 250, - "positive": 30 - }, - "followUpReminder": { - "negative": 250, - "positive": 0 - }, - "invitation": { - "negative": 250, - "positive": 0 - }, - "question": { - "negative": 250, - "positive": 0 - }, - "replyableMessage": { - "negative": 39, - "positive": 250 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 0 - }, - "task": { - "negative": 250, - "positive": 0 - } - }, - "round": 15, - "selectionObjective": 0.6461, - "syntheticRegression": { - "intentsMeeting95Precision": 8, - "macroF1": 0.4969, - "macroPrecision": 0.9744, - "macroRecall": 0.3753, - "minimumPrecision": 0.8151, - "perIntent": { - "blessing": { - "f1": 0.2, - "falseNegative": 80, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 10, - "recall": 0.1111, - "total": 2970, - "trueNegative": 2880, - "truePositive": 10, - "wilsonPrecisionLower95": 0.7225 - }, - "complaint": { - "f1": 0.4211, - "falseNegative": 264, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 96, - "recall": 0.2667, - "total": 2970, - "trueNegative": 2610, - "truePositive": 96, - "wilsonPrecisionLower95": 0.9615 - }, - "confirmationDecision": { - "f1": 0.6488, - "falseNegative": 83, - "falsePositive": 22, - "precision": 0.8151, - "predictedPositive": 119, - "recall": 0.5389, - "total": 2970, - "trueNegative": 2768, - "truePositive": 97, - "wilsonPrecisionLower95": 0.7359 - }, - "followUpReminder": { - "f1": 0.2651, - "falseNegative": 305, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 55, - "recall": 0.1528, - "total": 2970, - "trueNegative": 2610, - "truePositive": 55, - "wilsonPrecisionLower95": 0.9347 - }, - "invitation": { - "f1": 0.2, - "falseNegative": 160, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 20, - "recall": 0.1111, - "total": 2970, - "trueNegative": 2790, - "truePositive": 20, - "wilsonPrecisionLower95": 0.8389 - }, - "question": { - "f1": 0.5366, - "falseNegative": 399, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 231, - "recall": 0.3667, - "total": 2970, - "trueNegative": 2340, - "truePositive": 231, - "wilsonPrecisionLower95": 0.9836 - }, - "replyableMessage": { - "f1": 0.862, - "falseNegative": 410, - "falsePositive": 64, - "precision": 0.9585, - "predictedPositive": 1544, - "recall": 0.7831, - "total": 2970, - "trueNegative": 1016, - "truePositive": 1480, - "wilsonPrecisionLower95": 0.9474 - }, - "scheduleNegotiation": { - "f1": 0.8235, - "falseNegative": 27, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 63, - "recall": 0.7, - "total": 2970, - "trueNegative": 2880, - "truePositive": 63, - "wilsonPrecisionLower95": 0.9425 - }, - "task": { - "f1": 0.5149, - "falseNegative": 470, - "falsePositive": 1, - "precision": 0.996, - "predictedPositive": 251, - "recall": 0.3472, - "total": 2970, - "trueNegative": 2249, - "truePositive": 250, - "wilsonPrecisionLower95": 0.9778 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9836, - "falseNegative": 0, - "falsePositive": 6, - "precision": 0.9677, - "predictedPositive": 186, - "recall": 1.0, - "total": 1485, - "trueNegative": 1299, - "truePositive": 180, - "wilsonPrecisionLower95": 0.9314 - }, - "threshold": 0.945 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.928, - "falseNegative": 19, - "falsePositive": 6, - "precision": 0.9641, - "predictedPositive": 167, - "recall": 0.8944, - "total": 1485, - "trueNegative": 1299, - "truePositive": 161, - "wilsonPrecisionLower95": 0.9238 - }, - "threshold": 0.77 - } - }, - "metrics": { - "f1": 0.9365, - "falseNegative": 28, - "falsePositive": 17, - "precision": 0.9513, - "predictedPositive": 349, - "recall": 0.9222, - "total": 2970, - "trueNegative": 2593, - "truePositive": 332, - "wilsonPrecisionLower95": 0.9234 - }, - "threshold": 0.88 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.954, - "falseNegative": 7, - "falsePositive": 1, - "precision": 0.9881, - "predictedPositive": 84, - "recall": 0.9222, - "total": 1485, - "trueNegative": 1394, - "truePositive": 83, - "wilsonPrecisionLower95": 0.9356 - }, - "threshold": 0.85 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6906, - "falseNegative": 42, - "falsePositive": 1, - "precision": 0.9796, - "predictedPositive": 49, - "recall": 0.5333, - "total": 1485, - "trueNegative": 1394, - "truePositive": 48, - "wilsonPrecisionLower95": 0.8931 - }, - "threshold": 0.985 - } - }, - "metrics": { - "f1": 0.8598, - "falseNegative": 39, - "falsePositive": 7, - "precision": 0.9527, - "predictedPositive": 148, - "recall": 0.7833, - "total": 2970, - "trueNegative": 2783, - "truePositive": 141, - "wilsonPrecisionLower95": 0.9056 - }, - "threshold": 0.935 - }, - "followUpReminder": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8328, - "falseNegative": 48, - "falsePositive": 5, - "precision": 0.9635, - "predictedPositive": 137, - "recall": 0.7333, - "total": 1485, - "trueNegative": 1300, - "truePositive": 132, - "wilsonPrecisionLower95": 0.9174 - }, - "threshold": 0.99 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9133, - "falseNegative": 11, - "falsePositive": 4, - "precision": 0.9518, - "predictedPositive": 83, - "recall": 0.8778, - "total": 1485, - "trueNegative": 1391, - "truePositive": 79, - "wilsonPrecisionLower95": 0.8825 - }, - "threshold": 0.97 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7548, - "falseNegative": 118, - "falsePositive": 10, - "precision": 0.9517, - "predictedPositive": 207, - "recall": 0.6254, - "total": 1485, - "trueNegative": 1160, - "truePositive": 197, - "wilsonPrecisionLower95": 0.9134 - }, - "threshold": 0.8 - } - }, - "metrics": { - "f1": 0.7678, - "falseNegative": 225, - "falsePositive": 20, - "precision": 0.9529, - "predictedPositive": 425, - "recall": 0.6429, - "total": 2970, - "trueNegative": 2320, - "truePositive": 405, - "wilsonPrecisionLower95": 0.9284 - }, - "threshold": 0.985 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.907, - "falseNegative": 126, - "falsePositive": 42, - "precision": 0.9512, - "predictedPositive": 861, - "recall": 0.8667, - "total": 1485, - "trueNegative": 498, - "truePositive": 819, - "wilsonPrecisionLower95": 0.9347 - }, - "threshold": 0.98 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8252, - "falseNegative": 258, - "falsePositive": 33, - "precision": 0.9542, - "predictedPositive": 720, - "recall": 0.727, - "total": 1485, - "trueNegative": 507, - "truePositive": 687, - "wilsonPrecisionLower95": 0.9363 - }, - "threshold": 0.96 - } - }, - "metrics": { - "f1": 0.8678, - "falseNegative": 384, - "falsePositive": 75, - "precision": 0.9526, - "predictedPositive": 1581, - "recall": 0.7968, - "total": 2970, - "trueNegative": 1005, - "truePositive": 1506, - "wilsonPrecisionLower95": 0.9409 - }, - "threshold": 0.965 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9157, - "falseNegative": 7, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 38, - "recall": 0.8444, - "total": 1485, - "trueNegative": 1440, - "truePositive": 38, - "wilsonPrecisionLower95": 0.9082 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9556, - "falseNegative": 2, - "falsePositive": 2, - "precision": 0.9556, - "predictedPositive": 45, - "recall": 0.9556, - "total": 1485, - "trueNegative": 1438, - "truePositive": 43, - "wilsonPrecisionLower95": 0.8517 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.9364, - "falseNegative": 9, - "falsePositive": 2, - "precision": 0.9759, - "predictedPositive": 83, - "recall": 0.9, - "total": 2970, - "trueNegative": 2878, - "truePositive": 81, - "wilsonPrecisionLower95": 0.9163 - }, - "threshold": 0.5 - }, - "task": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8593, - "falseNegative": 79, - "falsePositive": 13, - "precision": 0.9558, - "predictedPositive": 294, - "recall": 0.7806, - "total": 1485, - "trueNegative": 1112, - "truePositive": 281, - "wilsonPrecisionLower95": 0.9258 - }, - "threshold": 0.98 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8166, - "falseNegative": 104, - "falsePositive": 11, - "precision": 0.9588, - "predictedPositive": 267, - "recall": 0.7111, - "total": 1485, - "trueNegative": 1114, - "truePositive": 256, - "wilsonPrecisionLower95": 0.9277 - }, - "threshold": 0.96 - } - }, - "metrics": { - "f1": 0.8434, - "falseNegative": 176, - "falsePositive": 26, - "precision": 0.9544, - "predictedPositive": 570, - "recall": 0.7556, - "total": 2970, - "trueNegative": 2224, - "truePositive": 544, - "wilsonPrecisionLower95": 0.934 - }, - "threshold": 0.965 - } - }, - "trainingRecords": 15345, - "validation": { - "intentsMeeting95Precision": 7, - "macroF1": 0.4919, - "macroPrecision": 0.9716, - "macroRecall": 0.3554, - "minimumPrecision": 0.9024, - "perIntent": { - "blessing": { - "f1": 0.3333, - "falseNegative": 72, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 18, - "recall": 0.2, - "total": 2970, - "trueNegative": 2880, - "truePositive": 18, - "wilsonPrecisionLower95": 0.8241 - }, - "complaint": { - "f1": 0.5455, - "falseNegative": 225, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 135, - "recall": 0.375, - "total": 2970, - "trueNegative": 2610, - "truePositive": 135, - "wilsonPrecisionLower95": 0.9723 - }, - "confirmationDecision": { - "f1": 0.6493, - "falseNegative": 93, - "falsePositive": 1, - "precision": 0.9886, - "predictedPositive": 88, - "recall": 0.4833, - "total": 2970, - "trueNegative": 2789, - "truePositive": 87, - "wilsonPrecisionLower95": 0.9384 - }, - "followUpReminder": { - "f1": 0.1845, - "falseNegative": 323, - "falsePositive": 4, - "precision": 0.9024, - "predictedPositive": 41, - "recall": 0.1028, - "total": 2970, - "trueNegative": 2606, - "truePositive": 37, - "wilsonPrecisionLower95": 0.7745 - }, - "invitation": { - "f1": 0.2775, - "falseNegative": 151, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 29, - "recall": 0.1611, - "total": 2970, - "trueNegative": 2790, - "truePositive": 29, - "wilsonPrecisionLower95": 0.883 - }, - "question": { - "f1": 0.4893, - "falseNegative": 425, - "falsePositive": 3, - "precision": 0.9856, - "predictedPositive": 208, - "recall": 0.3254, - "total": 2970, - "trueNegative": 2337, - "truePositive": 205, - "wilsonPrecisionLower95": 0.9585 - }, - "replyableMessage": { - "f1": 0.8678, - "falseNegative": 384, - "falsePositive": 75, - "precision": 0.9526, - "predictedPositive": 1581, - "recall": 0.7968, - "total": 2970, - "trueNegative": 1005, - "truePositive": 1506, - "wilsonPrecisionLower95": 0.9409 - }, - "scheduleNegotiation": { - "f1": 0.4918, - "falseNegative": 60, - "falsePositive": 2, - "precision": 0.9375, - "predictedPositive": 32, - "recall": 0.3333, - "total": 2970, - "trueNegative": 2878, - "truePositive": 30, - "wilsonPrecisionLower95": 0.7985 - }, - "task": { - "f1": 0.5883, - "falseNegative": 417, - "falsePositive": 7, - "precision": 0.9774, - "predictedPositive": 310, - "recall": 0.4208, - "total": 2970, - "trueNegative": 2243, - "truePositive": 303, - "wilsonPrecisionLower95": 0.9541 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 3e-06, - "augmentation": "numeric", - "char_max": 7, - "char_min": 2, - "external_weight": 0.8700000000000001, - "hard_example_weight": 2.3, - "l1_ratio": 0.15, - "max_char_features": 80000, - "max_word_features": 38000, - "min_df": 2, - "pseudo_weight": 0.39, - "round": 16, - "word_max": 3 - }, - "featureCount": 100252, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 3 - }, - "complaint": { - "negative": 250, - "positive": 11 - }, - "confirmationDecision": { - "negative": 250, - "positive": 25 - }, - "followUpReminder": { - "negative": 250, - "positive": 0 - }, - "invitation": { - "negative": 250, - "positive": 0 - }, - "question": { - "negative": 250, - "positive": 0 - }, - "replyableMessage": { - "negative": 76, - "positive": 250 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 0 - }, - "task": { - "negative": 250, - "positive": 0 - } - }, - "round": 16, - "selectionObjective": 0.6666, - "syntheticRegression": { - "intentsMeeting95Precision": 7, - "macroF1": 0.5355, - "macroPrecision": 0.9681, - "macroRecall": 0.4128, - "minimumPrecision": 0.7609, - "perIntent": { - "blessing": { - "f1": 0.125, - "falseNegative": 84, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 6, - "recall": 0.0667, - "total": 2970, - "trueNegative": 2880, - "truePositive": 6, - "wilsonPrecisionLower95": 0.6097 - }, - "complaint": { - "f1": 0.3673, - "falseNegative": 279, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 81, - "recall": 0.225, - "total": 2970, - "trueNegative": 2610, - "truePositive": 81, - "wilsonPrecisionLower95": 0.9547 - }, - "confirmationDecision": { - "f1": 0.5147, - "falseNegative": 110, - "falsePositive": 22, - "precision": 0.7609, - "predictedPositive": 92, - "recall": 0.3889, - "total": 2970, - "trueNegative": 2768, - "truePositive": 70, - "wilsonPrecisionLower95": 0.6644 - }, - "followUpReminder": { - "f1": 0.3486, - "falseNegative": 284, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 76, - "recall": 0.2111, - "total": 2970, - "trueNegative": 2610, - "truePositive": 76, - "wilsonPrecisionLower95": 0.9519 - }, - "invitation": { - "f1": 0.4211, - "falseNegative": 132, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 48, - "recall": 0.2667, - "total": 2970, - "trueNegative": 2790, - "truePositive": 48, - "wilsonPrecisionLower95": 0.9259 - }, - "question": { - "f1": 0.8117, - "falseNegative": 199, - "falsePositive": 1, - "precision": 0.9977, - "predictedPositive": 432, - "recall": 0.6841, - "total": 2970, - "trueNegative": 2339, - "truePositive": 431, - "wilsonPrecisionLower95": 0.987 - }, - "replyableMessage": { - "f1": 0.8725, - "falseNegative": 371, - "falsePositive": 73, - "precision": 0.9541, - "predictedPositive": 1592, - "recall": 0.8037, - "total": 2970, - "trueNegative": 1007, - "truePositive": 1519, - "wilsonPrecisionLower95": 0.9427 - }, - "scheduleNegotiation": { - "f1": 0.8312, - "falseNegative": 26, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 64, - "recall": 0.7111, - "total": 2970, - "trueNegative": 2880, - "truePositive": 64, - "wilsonPrecisionLower95": 0.9434 - }, - "task": { - "f1": 0.5276, - "falseNegative": 462, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 258, - "recall": 0.3583, - "total": 2970, - "trueNegative": 2250, - "truePositive": 258, - "wilsonPrecisionLower95": 0.9853 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9371, - "falseNegative": 16, - "falsePositive": 6, - "precision": 0.9647, - "predictedPositive": 170, - "recall": 0.9111, - "total": 1485, - "trueNegative": 1299, - "truePositive": 164, - "wilsonPrecisionLower95": 0.9251 - }, - "threshold": 0.995 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8836, - "falseNegative": 32, - "falsePositive": 7, - "precision": 0.9548, - "predictedPositive": 155, - "recall": 0.8222, - "total": 1485, - "trueNegative": 1298, - "truePositive": 148, - "wilsonPrecisionLower95": 0.9097 - }, - "threshold": 0.99 - } - }, - "metrics": { - "f1": 0.8994, - "falseNegative": 56, - "falsePositive": 12, - "precision": 0.962, - "predictedPositive": 316, - "recall": 0.8444, - "total": 2970, - "trueNegative": 2598, - "truePositive": 304, - "wilsonPrecisionLower95": 0.9348 - }, - "threshold": 0.995 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8625, - "falseNegative": 21, - "falsePositive": 1, - "precision": 0.9857, - "predictedPositive": 70, - "recall": 0.7667, - "total": 1485, - "trueNegative": 1394, - "truePositive": 69, - "wilsonPrecisionLower95": 0.9234 - }, - "threshold": 0.915 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.7415, - "falseNegative": 71, - "falsePositive": 5, - "precision": 0.9561, - "predictedPositive": 114, - "recall": 0.6056, - "total": 2970, - "trueNegative": 2785, - "truePositive": 109, - "wilsonPrecisionLower95": 0.9014 - }, - "threshold": 0.96 - }, - "followUpReminder": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8667, - "falseNegative": 37, - "falsePositive": 7, - "precision": 0.9533, - "predictedPositive": 150, - "recall": 0.7944, - "total": 1485, - "trueNegative": 1298, - "truePositive": 143, - "wilsonPrecisionLower95": 0.9068 - }, - "threshold": 0.96 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6031, - "falseNegative": 101, - "falsePositive": 3, - "precision": 0.9634, - "predictedPositive": 82, - "recall": 0.4389, - "total": 1485, - "trueNegative": 1302, - "truePositive": 79, - "wilsonPrecisionLower95": 0.8979 - }, - "threshold": 0.97 - } - }, - "metrics": { - "f1": 0.7492, - "falseNegative": 139, - "falsePositive": 9, - "precision": 0.9609, - "predictedPositive": 230, - "recall": 0.6139, - "total": 2970, - "trueNegative": 2601, - "truePositive": 221, - "wilsonPrecisionLower95": 0.9273 - }, - "threshold": 0.97 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.3604, - "falseNegative": 70, - "falsePositive": 1, - "precision": 0.9524, - "predictedPositive": 21, - "recall": 0.2222, - "total": 1485, - "trueNegative": 1394, - "truePositive": 20, - "wilsonPrecisionLower95": 0.7733 - }, - "threshold": 0.965 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8273, - "falseNegative": 85, - "falsePositive": 11, - "precision": 0.9544, - "predictedPositive": 241, - "recall": 0.7302, - "total": 1485, - "trueNegative": 1159, - "truePositive": 230, - "wilsonPrecisionLower95": 0.9201 - }, - "threshold": 0.985 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.797, - "falseNegative": 99, - "falsePositive": 11, - "precision": 0.9515, - "predictedPositive": 227, - "recall": 0.6857, - "total": 1485, - "trueNegative": 1159, - "truePositive": 216, - "wilsonPrecisionLower95": 0.9153 - }, - "threshold": 0.835 - } - }, - "metrics": { - "f1": 0.8051, - "falseNegative": 190, - "falsePositive": 23, - "precision": 0.9503, - "predictedPositive": 463, - "recall": 0.6984, - "total": 2970, - "trueNegative": 2317, - "truePositive": 440, - "wilsonPrecisionLower95": 0.9266 - }, - "threshold": 0.94 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9343, - "falseNegative": 77, - "falsePositive": 45, - "precision": 0.9507, - "predictedPositive": 913, - "recall": 0.9185, - "total": 1485, - "trueNegative": 495, - "truePositive": 868, - "wilsonPrecisionLower95": 0.9347 - }, - "threshold": 0.91 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8207, - "falseNegative": 263, - "falsePositive": 35, - "precision": 0.9512, - "predictedPositive": 717, - "recall": 0.7217, - "total": 1485, - "trueNegative": 505, - "truePositive": 682, - "wilsonPrecisionLower95": 0.9329 - }, - "threshold": 0.98 - } - }, - "metrics": { - "f1": 0.8572, - "falseNegative": 419, - "falsePositive": 71, - "precision": 0.954, - "predictedPositive": 1542, - "recall": 0.7783, - "total": 2970, - "trueNegative": 1009, - "truePositive": 1471, - "wilsonPrecisionLower95": 0.9423 - }, - "threshold": 0.98 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9888, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 44, - "recall": 0.9778, - "total": 1485, - "trueNegative": 1440, - "truePositive": 44, - "wilsonPrecisionLower95": 0.9197 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.989, - "falseNegative": 0, - "falsePositive": 1, - "precision": 0.9783, - "predictedPositive": 46, - "recall": 1.0, - "total": 1485, - "trueNegative": 1439, - "truePositive": 45, - "wilsonPrecisionLower95": 0.8866 - }, - "threshold": 0.955 - } - }, - "metrics": { - "f1": 0.9778, - "falseNegative": 2, - "falsePositive": 2, - "precision": 0.9778, - "predictedPositive": 90, - "recall": 0.9778, - "total": 2970, - "trueNegative": 2878, - "truePositive": 88, - "wilsonPrecisionLower95": 0.9226 - }, - "threshold": 0.935 - }, - "task": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8836, - "falseNegative": 64, - "falsePositive": 14, - "precision": 0.9548, - "predictedPositive": 310, - "recall": 0.8222, - "total": 1485, - "trueNegative": 1111, - "truePositive": 296, - "wilsonPrecisionLower95": 0.9256 - }, - "threshold": 0.975 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7896, - "falseNegative": 118, - "falsePositive": 11, - "precision": 0.9565, - "predictedPositive": 253, - "recall": 0.6722, - "total": 1485, - "trueNegative": 1114, - "truePositive": 242, - "wilsonPrecisionLower95": 0.9238 - }, - "threshold": 0.96 - } - }, - "metrics": { - "f1": 0.8316, - "falseNegative": 189, - "falsePositive": 26, - "precision": 0.9533, - "predictedPositive": 557, - "recall": 0.7375, - "total": 2970, - "trueNegative": 2224, - "truePositive": 531, - "wilsonPrecisionLower95": 0.9325 - }, - "threshold": 0.97 - } - }, - "trainingRecords": 16726, - "validation": { - "intentsMeeting95Precision": 9, - "macroF1": 0.514, - "macroPrecision": 0.9879, - "macroRecall": 0.3812, - "minimumPrecision": 0.9509, - "perIntent": { - "blessing": { - "f1": 0.3636, - "falseNegative": 70, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 20, - "recall": 0.2222, - "total": 2970, - "trueNegative": 2880, - "truePositive": 20, - "wilsonPrecisionLower95": 0.8389 - }, - "complaint": { - "f1": 0.5155, - "falseNegative": 235, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 125, - "recall": 0.3472, - "total": 2970, - "trueNegative": 2610, - "truePositive": 125, - "wilsonPrecisionLower95": 0.9702 - }, - "confirmationDecision": { - "f1": 0.4549, - "falseNegative": 127, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 53, - "recall": 0.2944, - "total": 2970, - "trueNegative": 2790, - "truePositive": 53, - "wilsonPrecisionLower95": 0.9324 - }, - "followUpReminder": { - "f1": 0.2347, - "falseNegative": 312, - "falsePositive": 1, - "precision": 0.9796, - "predictedPositive": 49, - "recall": 0.1333, - "total": 2970, - "trueNegative": 2609, - "truePositive": 48, - "wilsonPrecisionLower95": 0.8931 - }, - "invitation": { - "f1": 0.2609, - "falseNegative": 153, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 27, - "recall": 0.15, - "total": 2970, - "trueNegative": 2790, - "truePositive": 27, - "wilsonPrecisionLower95": 0.8754 - }, - "question": { - "f1": 0.8133, - "falseNegative": 190, - "falsePositive": 12, - "precision": 0.9735, - "predictedPositive": 452, - "recall": 0.6984, - "total": 2970, - "trueNegative": 2328, - "truePositive": 440, - "wilsonPrecisionLower95": 0.9542 - }, - "replyableMessage": { - "f1": 0.8807, - "falseNegative": 340, - "falsePositive": 80, - "precision": 0.9509, - "predictedPositive": 1630, - "recall": 0.8201, - "total": 2970, - "trueNegative": 1000, - "truePositive": 1550, - "wilsonPrecisionLower95": 0.9393 - }, - "scheduleNegotiation": { - "f1": 0.5124, - "falseNegative": 59, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 31, - "recall": 0.3444, - "total": 2970, - "trueNegative": 2880, - "truePositive": 31, - "wilsonPrecisionLower95": 0.8897 - }, - "task": { - "f1": 0.5901, - "falseNegative": 417, - "falsePositive": 4, - "precision": 0.987, - "predictedPositive": 307, - "recall": 0.4208, - "total": 2970, - "trueNegative": 2246, - "truePositive": 303, - "wilsonPrecisionLower95": 0.967 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 2e-06, - "augmentation": "numeric", - "char_max": 7, - "char_min": 3, - "external_weight": 0.8900000000000001, - "hard_example_weight": 2.5, - "l1_ratio": 0.2, - "max_char_features": 85000, - "max_word_features": 42000, - "min_df": 1, - "pseudo_weight": 0.405, - "round": 17, - "word_max": 3 - }, - "featureCount": 127000, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 1 - }, - "complaint": { - "negative": 250, - "positive": 4 - }, - "confirmationDecision": { - "negative": 250, - "positive": 23 - }, - "followUpReminder": { - "negative": 250, - "positive": 3 - }, - "invitation": { - "negative": 250, - "positive": 0 - }, - "question": { - "negative": 250, - "positive": 0 - }, - "replyableMessage": { - "negative": 70, - "positive": 250 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 0 - }, - "task": { - "negative": 250, - "positive": 0 - } - }, - "round": 17, - "selectionObjective": 0.5722, - "syntheticRegression": { - "intentsMeeting95Precision": 7, - "macroF1": 0.3646, - "macroPrecision": 0.9708, - "macroRecall": 0.2459, - "minimumPrecision": 0.7816, - "perIntent": { - "blessing": { - "f1": 0.1633, - "falseNegative": 82, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 8, - "recall": 0.0889, - "total": 2970, - "trueNegative": 2880, - "truePositive": 8, - "wilsonPrecisionLower95": 0.6756 - }, - "complaint": { - "f1": 0.4035, - "falseNegative": 269, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 91, - "recall": 0.2528, - "total": 2970, - "trueNegative": 2610, - "truePositive": 91, - "wilsonPrecisionLower95": 0.9595 - }, - "confirmationDecision": { - "f1": 0.5094, - "falseNegative": 112, - "falsePositive": 19, - "precision": 0.7816, - "predictedPositive": 87, - "recall": 0.3778, - "total": 2970, - "trueNegative": 2771, - "truePositive": 68, - "wilsonPrecisionLower95": 0.6839 - }, - "followUpReminder": { - "f1": 0.1679, - "falseNegative": 327, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 33, - "recall": 0.0917, - "total": 2970, - "trueNegative": 2610, - "truePositive": 33, - "wilsonPrecisionLower95": 0.8957 - }, - "invitation": { - "f1": 0.371, - "falseNegative": 139, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 41, - "recall": 0.2278, - "total": 2970, - "trueNegative": 2790, - "truePositive": 41, - "wilsonPrecisionLower95": 0.9143 - }, - "question": { - "f1": 0.4783, - "falseNegative": 432, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 198, - "recall": 0.3143, - "total": 2970, - "trueNegative": 2340, - "truePositive": 198, - "wilsonPrecisionLower95": 0.981 - }, - "replyableMessage": { - "f1": 0.2549, - "falseNegative": 1612, - "falsePositive": 13, - "precision": 0.9553, - "predictedPositive": 291, - "recall": 0.1471, - "total": 2970, - "trueNegative": 1067, - "truePositive": 278, - "wilsonPrecisionLower95": 0.9251 - }, - "scheduleNegotiation": { - "f1": 0.7671, - "falseNegative": 34, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 56, - "recall": 0.6222, - "total": 2970, - "trueNegative": 2880, - "truePositive": 56, - "wilsonPrecisionLower95": 0.9358 - }, - "task": { - "f1": 0.1656, - "falseNegative": 655, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 65, - "recall": 0.0903, - "total": 2970, - "trueNegative": 2250, - "truePositive": 65, - "wilsonPrecisionLower95": 0.9442 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.989, - "falseNegative": 0, - "falsePositive": 4, - "precision": 0.9783, - "predictedPositive": 184, - "recall": 1.0, - "total": 1485, - "trueNegative": 1301, - "truePositive": 180, - "wilsonPrecisionLower95": 0.9454 - }, - "threshold": 0.995 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8869, - "falseNegative": 31, - "falsePositive": 7, - "precision": 0.9551, - "predictedPositive": 156, - "recall": 0.8278, - "total": 1485, - "trueNegative": 1298, - "truePositive": 149, - "wilsonPrecisionLower95": 0.9103 - }, - "threshold": 0.995 - } - }, - "metrics": { - "f1": 0.944, - "falseNegative": 23, - "falsePositive": 17, - "precision": 0.952, - "predictedPositive": 354, - "recall": 0.9361, - "total": 2970, - "trueNegative": 2593, - "truePositive": 337, - "wilsonPrecisionLower95": 0.9244 - }, - "threshold": 0.975 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8447, - "falseNegative": 22, - "falsePositive": 3, - "precision": 0.9577, - "predictedPositive": 71, - "recall": 0.7556, - "total": 1485, - "trueNegative": 1392, - "truePositive": 68, - "wilsonPrecisionLower95": 0.883 - }, - "threshold": 0.825 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.7723, - "falseNegative": 63, - "falsePositive": 6, - "precision": 0.9512, - "predictedPositive": 123, - "recall": 0.65, - "total": 2970, - "trueNegative": 2784, - "truePositive": 117, - "wilsonPrecisionLower95": 0.8977 - }, - "threshold": 0.965 - }, - "followUpReminder": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6625, - "falseNegative": 155, - "falsePositive": 8, - "precision": 0.9524, - "predictedPositive": 168, - "recall": 0.5079, - "total": 1485, - "trueNegative": 1162, - "truePositive": 160, - "wilsonPrecisionLower95": 0.9089 - }, - "threshold": 0.97 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "replyableMessage": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9655, - "falseNegative": 3, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 42, - "recall": 0.9333, - "total": 1485, - "trueNegative": 1440, - "truePositive": 42, - "wilsonPrecisionLower95": 0.9162 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9425, - "falseNegative": 4, - "falsePositive": 1, - "precision": 0.9762, - "predictedPositive": 42, - "recall": 0.9111, - "total": 1485, - "trueNegative": 1439, - "truePositive": 41, - "wilsonPrecisionLower95": 0.8768 - }, - "threshold": 0.73 - } - }, - "metrics": { - "f1": 0.9486, - "falseNegative": 7, - "falsePositive": 2, - "precision": 0.9765, - "predictedPositive": 85, - "recall": 0.9222, - "total": 2970, - "trueNegative": 2878, - "truePositive": 83, - "wilsonPrecisionLower95": 0.9182 - }, - "threshold": 0.5 - }, - "task": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "trainingRecords": 16820, - "validation": { - "intentsMeeting95Precision": 9, - "macroF1": 0.3704, - "macroPrecision": 0.9857, - "macroRecall": 0.235, - "minimumPrecision": 0.9651, - "perIntent": { - "blessing": { - "f1": 0.3019, - "falseNegative": 74, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 16, - "recall": 0.1778, - "total": 2970, - "trueNegative": 2880, - "truePositive": 16, - "wilsonPrecisionLower95": 0.8064 - }, - "complaint": { - "f1": 0.5325, - "falseNegative": 229, - "falsePositive": 1, - "precision": 0.9924, - "predictedPositive": 132, - "recall": 0.3639, - "total": 2970, - "trueNegative": 2609, - "truePositive": 131, - "wilsonPrecisionLower95": 0.9583 - }, - "confirmationDecision": { - "f1": 0.4329, - "falseNegative": 130, - "falsePositive": 1, - "precision": 0.9804, - "predictedPositive": 51, - "recall": 0.2778, - "total": 2970, - "trueNegative": 2789, - "truePositive": 50, - "wilsonPrecisionLower95": 0.897 - }, - "followUpReminder": { - "f1": 0.204, - "falseNegative": 319, - "falsePositive": 1, - "precision": 0.9762, - "predictedPositive": 42, - "recall": 0.1139, - "total": 2970, - "trueNegative": 2609, - "truePositive": 41, - "wilsonPrecisionLower95": 0.8768 - }, - "invitation": { - "f1": 0.2609, - "falseNegative": 153, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 27, - "recall": 0.15, - "total": 2970, - "trueNegative": 2790, - "truePositive": 27, - "wilsonPrecisionLower95": 0.8754 - }, - "question": { - "f1": 0.5471, - "falseNegative": 392, - "falsePositive": 2, - "precision": 0.9917, - "predictedPositive": 240, - "recall": 0.3778, - "total": 2970, - "trueNegative": 2338, - "truePositive": 238, - "wilsonPrecisionLower95": 0.9701 - }, - "replyableMessage": { - "f1": 0.2757, - "falseNegative": 1586, - "falsePositive": 11, - "precision": 0.9651, - "predictedPositive": 315, - "recall": 0.1608, - "total": 2970, - "trueNegative": 1069, - "truePositive": 304, - "wilsonPrecisionLower95": 0.9386 - }, - "scheduleNegotiation": { - "f1": 0.4706, - "falseNegative": 62, - "falsePositive": 1, - "precision": 0.9655, - "predictedPositive": 29, - "recall": 0.3111, - "total": 2970, - "trueNegative": 2879, - "truePositive": 28, - "wilsonPrecisionLower95": 0.8282 - }, - "task": { - "f1": 0.3079, - "falseNegative": 589, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 131, - "recall": 0.1819, - "total": 2970, - "trueNegative": 2250, - "truePositive": 131, - "wilsonPrecisionLower95": 0.9715 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 1.5e-06, - "augmentation": "numeric", - "char_max": 8, - "char_min": 2, - "external_weight": 0.91, - "hard_example_weight": 2.5, - "l1_ratio": 0.2, - "max_char_features": 90000, - "max_word_features": 45000, - "min_df": 1, - "pseudo_weight": 0.42000000000000004, - "round": 18, - "word_max": 3 - }, - "featureCount": 135000, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 1 - }, - "complaint": { - "negative": 250, - "positive": 8 - }, - "confirmationDecision": { - "negative": 250, - "positive": 27 - }, - "followUpReminder": { - "negative": 250, - "positive": 0 - }, - "invitation": { - "negative": 250, - "positive": 0 - }, - "question": { - "negative": 250, - "positive": 0 - }, - "replyableMessage": { - "negative": 90, - "positive": 32 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 0 - }, - "task": { - "negative": 250, - "positive": 0 - } - }, - "round": 18, - "selectionObjective": 0.6413, - "syntheticRegression": { - "intentsMeeting95Precision": 7, - "macroF1": 0.4676, - "macroPrecision": 0.9604, - "macroRecall": 0.3553, - "minimumPrecision": 0.7403, - "perIntent": { - "blessing": { - "f1": 0.2178, - "falseNegative": 79, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 11, - "recall": 0.1222, - "total": 2970, - "trueNegative": 2880, - "truePositive": 11, - "wilsonPrecisionLower95": 0.7412 - }, - "complaint": { - "f1": 0.3059, - "falseNegative": 295, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 65, - "recall": 0.1806, - "total": 2970, - "trueNegative": 2610, - "truePositive": 65, - "wilsonPrecisionLower95": 0.9442 - }, - "confirmationDecision": { - "f1": 0.4436, - "falseNegative": 123, - "falsePositive": 20, - "precision": 0.7403, - "predictedPositive": 77, - "recall": 0.3167, - "total": 2970, - "trueNegative": 2770, - "truePositive": 57, - "wilsonPrecisionLower95": 0.6326 - }, - "followUpReminder": { - "f1": 0.2844, - "falseNegative": 300, - "falsePositive": 2, - "precision": 0.9677, - "predictedPositive": 62, - "recall": 0.1667, - "total": 2970, - "trueNegative": 2608, - "truePositive": 60, - "wilsonPrecisionLower95": 0.8898 - }, - "invitation": { - "f1": 0.3178, - "falseNegative": 146, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 34, - "recall": 0.1889, - "total": 2970, - "trueNegative": 2790, - "truePositive": 34, - "wilsonPrecisionLower95": 0.8985 - }, - "question": { - "f1": 0.6957, - "falseNegative": 294, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 336, - "recall": 0.5333, - "total": 2970, - "trueNegative": 2340, - "truePositive": 336, - "wilsonPrecisionLower95": 0.9887 - }, - "replyableMessage": { - "f1": 0.8939, - "falseNegative": 273, - "falsePositive": 111, - "precision": 0.9358, - "predictedPositive": 1728, - "recall": 0.8556, - "total": 2970, - "trueNegative": 969, - "truePositive": 1617, - "wilsonPrecisionLower95": 0.9232 - }, - "scheduleNegotiation": { - "f1": 0.8312, - "falseNegative": 26, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 64, - "recall": 0.7111, - "total": 2970, - "trueNegative": 2880, - "truePositive": 64, - "wilsonPrecisionLower95": 0.9434 - }, - "task": { - "f1": 0.2178, - "falseNegative": 632, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 88, - "recall": 0.1222, - "total": 2970, - "trueNegative": 2250, - "truePositive": 88, - "wilsonPrecisionLower95": 0.9582 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9486, - "falseNegative": 7, - "falsePositive": 2, - "precision": 0.9765, - "predictedPositive": 85, - "recall": 0.9222, - "total": 1485, - "trueNegative": 1393, - "truePositive": 83, - "wilsonPrecisionLower95": 0.9182 - }, - "threshold": 0.9 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.8447, - "falseNegative": 44, - "falsePositive": 6, - "precision": 0.9577, - "predictedPositive": 142, - "recall": 0.7556, - "total": 2970, - "trueNegative": 2784, - "truePositive": 136, - "wilsonPrecisionLower95": 0.9109 - }, - "threshold": 0.955 - }, - "followUpReminder": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8822, - "falseNegative": 34, - "falsePositive": 5, - "precision": 0.9669, - "predictedPositive": 151, - "recall": 0.8111, - "total": 1485, - "trueNegative": 1300, - "truePositive": 146, - "wilsonPrecisionLower95": 0.9248 - }, - "threshold": 0.98 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.7641, - "falseNegative": 130, - "falsePositive": 12, - "precision": 0.9504, - "predictedPositive": 242, - "recall": 0.6389, - "total": 2970, - "trueNegative": 2598, - "truePositive": 230, - "wilsonPrecisionLower95": 0.9153 - }, - "threshold": 0.99 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8103, - "falseNegative": 95, - "falsePositive": 8, - "precision": 0.9649, - "predictedPositive": 228, - "recall": 0.6984, - "total": 1485, - "trueNegative": 1162, - "truePositive": 220, - "wilsonPrecisionLower95": 0.9323 - }, - "threshold": 0.915 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9575, - "falseNegative": 33, - "falsePositive": 48, - "precision": 0.95, - "predictedPositive": 960, - "recall": 0.9651, - "total": 1485, - "trueNegative": 492, - "truePositive": 912, - "wilsonPrecisionLower95": 0.9343 - }, - "threshold": 0.845 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8361, - "falseNegative": 241, - "falsePositive": 35, - "precision": 0.9526, - "predictedPositive": 739, - "recall": 0.745, - "total": 1485, - "trueNegative": 505, - "truePositive": 704, - "wilsonPrecisionLower95": 0.9348 - }, - "threshold": 0.98 - } - }, - "metrics": { - "f1": 0.901, - "falseNegative": 274, - "falsePositive": 81, - "precision": 0.9523, - "predictedPositive": 1697, - "recall": 0.855, - "total": 2970, - "trueNegative": 999, - "truePositive": 1616, - "wilsonPrecisionLower95": 0.9411 - }, - "threshold": 0.975 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9655, - "falseNegative": 3, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 42, - "recall": 0.9333, - "total": 1485, - "trueNegative": 1440, - "truePositive": 42, - "wilsonPrecisionLower95": 0.9162 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 45, - "recall": 1.0, - "total": 1485, - "trueNegative": 1440, - "truePositive": 45, - "wilsonPrecisionLower95": 0.9213 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.9831, - "falseNegative": 3, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 87, - "recall": 0.9667, - "total": 2970, - "trueNegative": 2880, - "truePositive": 87, - "wilsonPrecisionLower95": 0.9577 - }, - "threshold": 0.5 - }, - "task": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "trainingRecords": 16841, - "validation": { - "intentsMeeting95Precision": 8, - "macroF1": 0.478, - "macroPrecision": 0.982, - "macroRecall": 0.3466, - "minimumPrecision": 0.9074, - "perIntent": { - "blessing": { - "f1": 0.3486, - "falseNegative": 71, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 19, - "recall": 0.2111, - "total": 2970, - "trueNegative": 2880, - "truePositive": 19, - "wilsonPrecisionLower95": 0.8318 - }, - "complaint": { - "f1": 0.4506, - "falseNegative": 255, - "falsePositive": 1, - "precision": 0.9906, - "predictedPositive": 106, - "recall": 0.2917, - "total": 2970, - "trueNegative": 2609, - "truePositive": 105, - "wilsonPrecisionLower95": 0.9485 - }, - "confirmationDecision": { - "f1": 0.5366, - "falseNegative": 114, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 66, - "recall": 0.3667, - "total": 2970, - "trueNegative": 2790, - "truePositive": 66, - "wilsonPrecisionLower95": 0.945 - }, - "followUpReminder": { - "f1": 0.2367, - "falseNegative": 311, - "falsePositive": 5, - "precision": 0.9074, - "predictedPositive": 54, - "recall": 0.1361, - "total": 2970, - "trueNegative": 2605, - "truePositive": 49, - "wilsonPrecisionLower95": 0.8009 - }, - "invitation": { - "f1": 0.2692, - "falseNegative": 152, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 28, - "recall": 0.1556, - "total": 2970, - "trueNegative": 2790, - "truePositive": 28, - "wilsonPrecisionLower95": 0.8794 - }, - "question": { - "f1": 0.714, - "falseNegative": 278, - "falsePositive": 4, - "precision": 0.9888, - "predictedPositive": 356, - "recall": 0.5587, - "total": 2970, - "trueNegative": 2336, - "truePositive": 352, - "wilsonPrecisionLower95": 0.9715 - }, - "replyableMessage": { - "f1": 0.9005, - "falseNegative": 274, - "falsePositive": 83, - "precision": 0.9511, - "predictedPositive": 1699, - "recall": 0.855, - "total": 2970, - "trueNegative": 997, - "truePositive": 1616, - "wilsonPrecisionLower95": 0.9398 - }, - "scheduleNegotiation": { - "f1": 0.5124, - "falseNegative": 59, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 31, - "recall": 0.3444, - "total": 2970, - "trueNegative": 2880, - "truePositive": 31, - "wilsonPrecisionLower95": 0.8897 - }, - "task": { - "f1": 0.3333, - "falseNegative": 576, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 144, - "recall": 0.2, - "total": 2970, - "trueNegative": 2250, - "truePositive": 144, - "wilsonPrecisionLower95": 0.974 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 1.2e-06, - "augmentation": "numeric", - "char_max": 8, - "char_min": 3, - "external_weight": 0.93, - "hard_example_weight": 2.7, - "l1_ratio": 0.25, - "max_char_features": 95000, - "max_word_features": 45000, - "min_df": 1, - "pseudo_weight": 0.43499999999999994, - "round": 19, - "word_max": 3 - }, - "featureCount": 140000, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 1 - }, - "complaint": { - "negative": 250, - "positive": 2 - }, - "confirmationDecision": { - "negative": 250, - "positive": 36 - }, - "followUpReminder": { - "negative": 250, - "positive": 0 - }, - "invitation": { - "negative": 250, - "positive": 0 - }, - "question": { - "negative": 250, - "positive": 4 - }, - "replyableMessage": { - "negative": 99, - "positive": 32 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 0 - }, - "task": { - "negative": 250, - "positive": 0 - } - }, - "round": 19, - "selectionObjective": 0.6027, - "syntheticRegression": { - "intentsMeeting95Precision": 7, - "macroF1": 0.4167, - "macroPrecision": 0.9816, - "macroRecall": 0.2818, - "minimumPrecision": 0.9094, - "perIntent": { - "blessing": { - "f1": 0.2, - "falseNegative": 80, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 10, - "recall": 0.1111, - "total": 2970, - "trueNegative": 2880, - "truePositive": 10, - "wilsonPrecisionLower95": 0.7225 - }, - "complaint": { - "f1": 0.371, - "falseNegative": 278, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 82, - "recall": 0.2278, - "total": 2970, - "trueNegative": 2610, - "truePositive": 82, - "wilsonPrecisionLower95": 0.9552 - }, - "confirmationDecision": { - "f1": 0.502, - "falseNegative": 118, - "falsePositive": 5, - "precision": 0.9254, - "predictedPositive": 67, - "recall": 0.3444, - "total": 2970, - "trueNegative": 2785, - "truePositive": 62, - "wilsonPrecisionLower95": 0.8369 - }, - "followUpReminder": { - "f1": 0.231, - "falseNegative": 313, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 47, - "recall": 0.1306, - "total": 2970, - "trueNegative": 2610, - "truePositive": 47, - "wilsonPrecisionLower95": 0.9244 - }, - "invitation": { - "f1": 0.3636, - "falseNegative": 140, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 40, - "recall": 0.2222, - "total": 2970, - "trueNegative": 2790, - "truePositive": 40, - "wilsonPrecisionLower95": 0.9124 - }, - "question": { - "f1": 0.4387, - "falseNegative": 453, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 177, - "recall": 0.281, - "total": 2970, - "trueNegative": 2340, - "truePositive": 177, - "wilsonPrecisionLower95": 0.9788 - }, - "replyableMessage": { - "f1": 0.6399, - "falseNegative": 957, - "falsePositive": 93, - "precision": 0.9094, - "predictedPositive": 1026, - "recall": 0.4937, - "total": 2970, - "trueNegative": 987, - "truePositive": 933, - "wilsonPrecisionLower95": 0.8902 - }, - "scheduleNegotiation": { - "f1": 0.7143, - "falseNegative": 40, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 50, - "recall": 0.5556, - "total": 2970, - "trueNegative": 2880, - "truePositive": 50, - "wilsonPrecisionLower95": 0.9287 - }, - "task": { - "f1": 0.2898, - "falseNegative": 598, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 122, - "recall": 0.1694, - "total": 2970, - "trueNegative": 2250, - "truePositive": 122, - "wilsonPrecisionLower95": 0.9695 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "confirmationDecision": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "followUpReminder": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "replyableMessage": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9309, - "falseNegative": 83, - "falsePositive": 45, - "precision": 0.9504, - "predictedPositive": 907, - "recall": 0.9122, - "total": 1485, - "trueNegative": 495, - "truePositive": 862, - "wilsonPrecisionLower95": 0.9343 - }, - "threshold": 0.98 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 45, - "recall": 1.0, - "total": 1485, - "trueNegative": 1440, - "truePositive": 45, - "wilsonPrecisionLower95": 0.9213 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9195, - "falseNegative": 5, - "falsePositive": 2, - "precision": 0.9524, - "predictedPositive": 42, - "recall": 0.8889, - "total": 1485, - "trueNegative": 1438, - "truePositive": 40, - "wilsonPrecisionLower95": 0.8421 - }, - "threshold": 0.945 - } - }, - "metrics": { - "f1": 0.9667, - "falseNegative": 3, - "falsePositive": 3, - "precision": 0.9667, - "predictedPositive": 90, - "recall": 0.9667, - "total": 2970, - "trueNegative": 2877, - "truePositive": 87, - "wilsonPrecisionLower95": 0.9065 - }, - "threshold": 0.83 - }, - "task": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "trainingRecords": 16801, - "validation": { - "intentsMeeting95Precision": 7, - "macroF1": 0.4224, - "macroPrecision": 0.9791, - "macroRecall": 0.2773, - "minimumPrecision": 0.9215, - "perIntent": { - "blessing": { - "f1": 0.4483, - "falseNegative": 64, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 26, - "recall": 0.2889, - "total": 2970, - "trueNegative": 2880, - "truePositive": 26, - "wilsonPrecisionLower95": 0.8713 - }, - "complaint": { - "f1": 0.4648, - "falseNegative": 251, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 109, - "recall": 0.3028, - "total": 2970, - "trueNegative": 2610, - "truePositive": 109, - "wilsonPrecisionLower95": 0.966 - }, - "confirmationDecision": { - "f1": 0.4348, - "falseNegative": 130, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 50, - "recall": 0.2778, - "total": 2970, - "trueNegative": 2790, - "truePositive": 50, - "wilsonPrecisionLower95": 0.9287 - }, - "followUpReminder": { - "f1": 0.2045, - "falseNegative": 319, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 41, - "recall": 0.1139, - "total": 2970, - "trueNegative": 2610, - "truePositive": 41, - "wilsonPrecisionLower95": 0.9143 - }, - "invitation": { - "f1": 0.3019, - "falseNegative": 148, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 32, - "recall": 0.1778, - "total": 2970, - "trueNegative": 2790, - "truePositive": 32, - "wilsonPrecisionLower95": 0.8928 - }, - "question": { - "f1": 0.4287, - "falseNegative": 454, - "falsePositive": 15, - "precision": 0.9215, - "predictedPositive": 191, - "recall": 0.2794, - "total": 2970, - "trueNegative": 2325, - "truePositive": 176, - "wilsonPrecisionLower95": 0.8745 - }, - "replyableMessage": { - "f1": 0.6347, - "falseNegative": 990, - "falsePositive": 46, - "precision": 0.9514, - "predictedPositive": 946, - "recall": 0.4762, - "total": 2970, - "trueNegative": 1034, - "truePositive": 900, - "wilsonPrecisionLower95": 0.9357 - }, - "scheduleNegotiation": { - "f1": 0.5041, - "falseNegative": 59, - "falsePositive": 2, - "precision": 0.9394, - "predictedPositive": 33, - "recall": 0.3444, - "total": 2970, - "trueNegative": 2878, - "truePositive": 31, - "wilsonPrecisionLower95": 0.8039 - }, - "task": { - "f1": 0.3802, - "falseNegative": 551, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 169, - "recall": 0.2347, - "total": 2970, - "trueNegative": 2250, - "truePositive": 169, - "wilsonPrecisionLower95": 0.9778 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 1e-06, - "augmentation": "numeric", - "char_max": 8, - "char_min": 2, - "external_weight": 0.9500000000000001, - "hard_example_weight": 3.0, - "l1_ratio": 0.25, - "max_char_features": 100000, - "max_word_features": 50000, - "min_df": 1, - "pseudo_weight": 0.44999999999999996, - "round": 20, - "word_max": 3 - }, - "featureCount": 150000, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 1 - }, - "complaint": { - "negative": 250, - "positive": 2 - }, - "confirmationDecision": { - "negative": 250, - "positive": 25 - }, - "followUpReminder": { - "negative": 250, - "positive": 0 - }, - "invitation": { - "negative": 250, - "positive": 1 - }, - "question": { - "negative": 250, - "positive": 2 - }, - "replyableMessage": { - "negative": 104, - "positive": 32 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 1 - }, - "task": { - "negative": 250, - "positive": 0 - } - }, - "round": 20, - "selectionObjective": 0.6339, - "syntheticRegression": { - "intentsMeeting95Precision": 8, - "macroF1": 0.471, - "macroPrecision": 0.9861, - "macroRecall": 0.3411, - "minimumPrecision": 0.9231, - "perIntent": { - "blessing": { - "f1": 0.2353, - "falseNegative": 78, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 12, - "recall": 0.1333, - "total": 2970, - "trueNegative": 2880, - "truePositive": 12, - "wilsonPrecisionLower95": 0.7575 - }, - "complaint": { - "f1": 0.2816, - "falseNegative": 301, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 59, - "recall": 0.1639, - "total": 2970, - "trueNegative": 2610, - "truePositive": 59, - "wilsonPrecisionLower95": 0.9389 - }, - "confirmationDecision": { - "f1": 0.6199, - "falseNegative": 96, - "falsePositive": 7, - "precision": 0.9231, - "predictedPositive": 91, - "recall": 0.4667, - "total": 2970, - "trueNegative": 2783, - "truePositive": 84, - "wilsonPrecisionLower95": 0.8496 - }, - "followUpReminder": { - "f1": 0.2353, - "falseNegative": 312, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 48, - "recall": 0.1333, - "total": 2970, - "trueNegative": 2610, - "truePositive": 48, - "wilsonPrecisionLower95": 0.9259 - }, - "invitation": { - "f1": 0.3636, - "falseNegative": 140, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 40, - "recall": 0.2222, - "total": 2970, - "trueNegative": 2790, - "truePositive": 40, - "wilsonPrecisionLower95": 0.9124 - }, - "question": { - "f1": 0.5211, - "falseNegative": 408, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 222, - "recall": 0.3524, - "total": 2970, - "trueNegative": 2340, - "truePositive": 222, - "wilsonPrecisionLower95": 0.983 - }, - "replyableMessage": { - "f1": 0.7688, - "falseNegative": 671, - "falsePositive": 62, - "precision": 0.9516, - "predictedPositive": 1281, - "recall": 0.645, - "total": 2970, - "trueNegative": 1018, - "truePositive": 1219, - "wilsonPrecisionLower95": 0.9384 - }, - "scheduleNegotiation": { - "f1": 0.8387, - "falseNegative": 25, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 65, - "recall": 0.7222, - "total": 2970, - "trueNegative": 2880, - "truePositive": 65, - "wilsonPrecisionLower95": 0.9442 - }, - "task": { - "f1": 0.3747, - "falseNegative": 554, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 166, - "recall": 0.2306, - "total": 2970, - "trueNegative": 2250, - "truePositive": 166, - "wilsonPrecisionLower95": 0.9774 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "confirmationDecision": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.5954, - "falseNegative": 51, - "falsePositive": 2, - "precision": 0.9512, - "predictedPositive": 41, - "recall": 0.4333, - "total": 1485, - "trueNegative": 1393, - "truePositive": 39, - "wilsonPrecisionLower95": 0.8386 - }, - "threshold": 0.995 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "followUpReminder": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8117, - "falseNegative": 279, - "falsePositive": 30, - "precision": 0.9569, - "predictedPositive": 696, - "recall": 0.7048, - "total": 1485, - "trueNegative": 510, - "truePositive": 666, - "wilsonPrecisionLower95": 0.9391 - }, - "threshold": 0.995 - } - }, - "metrics": { - "f1": 0.8736, - "falseNegative": 363, - "falsePositive": 79, - "precision": 0.9508, - "predictedPositive": 1606, - "recall": 0.8079, - "total": 2970, - "trueNegative": 1001, - "truePositive": 1527, - "wilsonPrecisionLower95": 0.9391 - }, - "threshold": 0.995 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 45, - "recall": 1.0, - "total": 1485, - "trueNegative": 1440, - "truePositive": 45, - "wilsonPrecisionLower95": 0.9213 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.967, - "falseNegative": 1, - "falsePositive": 2, - "precision": 0.9565, - "predictedPositive": 46, - "recall": 0.9778, - "total": 1485, - "trueNegative": 1438, - "truePositive": 44, - "wilsonPrecisionLower95": 0.8547 - }, - "threshold": 0.83 - } - }, - "metrics": { - "f1": 0.9834, - "falseNegative": 1, - "falsePositive": 2, - "precision": 0.978, - "predictedPositive": 91, - "recall": 0.9889, - "total": 2970, - "trueNegative": 2878, - "truePositive": 89, - "wilsonPrecisionLower95": 0.9234 - }, - "threshold": 0.83 - }, - "task": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "trainingRecords": 16755, - "validation": { - "intentsMeeting95Precision": 8, - "macroF1": 0.4669, - "macroPrecision": 0.9855, - "macroRecall": 0.3215, - "minimumPrecision": 0.9333, - "perIntent": { - "blessing": { - "f1": 0.3929, - "falseNegative": 68, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 22, - "recall": 0.2444, - "total": 2970, - "trueNegative": 2880, - "truePositive": 22, - "wilsonPrecisionLower95": 0.8513 - }, - "complaint": { - "f1": 0.3929, - "falseNegative": 272, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 88, - "recall": 0.2444, - "total": 2970, - "trueNegative": 2610, - "truePositive": 88, - "wilsonPrecisionLower95": 0.9582 - }, - "confirmationDecision": { - "f1": 0.5366, - "falseNegative": 114, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 66, - "recall": 0.3667, - "total": 2970, - "trueNegative": 2790, - "truePositive": 66, - "wilsonPrecisionLower95": 0.945 - }, - "followUpReminder": { - "f1": 0.2074, - "falseNegative": 318, - "falsePositive": 3, - "precision": 0.9333, - "predictedPositive": 45, - "recall": 0.1167, - "total": 2970, - "trueNegative": 2607, - "truePositive": 42, - "wilsonPrecisionLower95": 0.8214 - }, - "invitation": { - "f1": 0.3178, - "falseNegative": 146, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 34, - "recall": 0.1889, - "total": 2970, - "trueNegative": 2790, - "truePositive": 34, - "wilsonPrecisionLower95": 0.8985 - }, - "question": { - "f1": 0.5568, - "falseNegative": 385, - "falsePositive": 5, - "precision": 0.98, - "predictedPositive": 250, - "recall": 0.3889, - "total": 2970, - "trueNegative": 2335, - "truePositive": 245, - "wilsonPrecisionLower95": 0.954 - }, - "replyableMessage": { - "f1": 0.7934, - "falseNegative": 621, - "falsePositive": 40, - "precision": 0.9694, - "predictedPositive": 1309, - "recall": 0.6714, - "total": 2970, - "trueNegative": 1040, - "truePositive": 1269, - "wilsonPrecisionLower95": 0.9587 - }, - "scheduleNegotiation": { - "f1": 0.5246, - "falseNegative": 58, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 32, - "recall": 0.3556, - "total": 2970, - "trueNegative": 2880, - "truePositive": 32, - "wilsonPrecisionLower95": 0.8928 - }, - "task": { - "f1": 0.4795, - "falseNegative": 492, - "falsePositive": 3, - "precision": 0.987, - "predictedPositive": 231, - "recall": 0.3167, - "total": 2970, - "trueNegative": 2247, - "truePositive": 228, - "wilsonPrecisionLower95": 0.9625 - } - }, - "recordCount": 2970 - } - } -] diff --git a/ModelTraining/ClipboardSemantics/IterativeResearch/PerIntentFineTune/final-report.json b/ModelTraining/ClipboardSemantics/IterativeResearch/PerIntentFineTune/final-report.json deleted file mode 100644 index 9fcd3c2..0000000 --- a/ModelTraining/ClipboardSemantics/IterativeResearch/PerIntentFineTune/final-report.json +++ /dev/null @@ -1,8422 +0,0 @@ -{ - "baseTrainingRecords": 11500, - "deploymentDecision": "retain-current-deployed-models", - "elapsedSeconds": 188.7732, - "finalEvaluations": { - "golden": { - "byLanguage": { - "en": { - "intentsMeeting95Precision": 1, - "macroF1": 0.847, - "macroPrecision": 0.9225, - "macroRecall": 0.8029, - "minimumPrecision": 0.7692, - "perIntent": { - "blessing": { - "f1": 0.8235, - "falseNegative": 3, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 7, - "recall": 0.7, - "total": 126, - "trueNegative": 116, - "truePositive": 7, - "wilsonPrecisionLower95": 0.6457 - }, - "complaint": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 6, - "recall": 1.0, - "total": 126, - "trueNegative": 120, - "truePositive": 6, - "wilsonPrecisionLower95": 0.6097 - }, - "confirmationDecision": { - "f1": 0.8696, - "falseNegative": 0, - "falsePositive": 3, - "precision": 0.7692, - "predictedPositive": 13, - "recall": 1.0, - "total": 126, - "trueNegative": 113, - "truePositive": 10, - "wilsonPrecisionLower95": 0.4974 - }, - "followUpReminder": { - "f1": 0.8696, - "falseNegative": 3, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 10, - "recall": 0.7692, - "total": 126, - "trueNegative": 113, - "truePositive": 10, - "wilsonPrecisionLower95": 0.7225 - }, - "invitation": { - "f1": 0.8571, - "falseNegative": 2, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 6, - "recall": 0.75, - "total": 126, - "trueNegative": 118, - "truePositive": 6, - "wilsonPrecisionLower95": 0.6097 - }, - "question": { - "f1": 0.8636, - "falseNegative": 4, - "falsePositive": 2, - "precision": 0.9048, - "predictedPositive": 21, - "recall": 0.8261, - "total": 126, - "trueNegative": 101, - "truePositive": 19, - "wilsonPrecisionLower95": 0.7109 - }, - "replyableMessage": { - "f1": 0.8889, - "falseNegative": 6, - "falsePositive": 10, - "precision": 0.8649, - "predictedPositive": 74, - "recall": 0.9143, - "total": 126, - "trueNegative": 46, - "truePositive": 64, - "wilsonPrecisionLower95": 0.7688 - }, - "scheduleNegotiation": { - "f1": 0.8421, - "falseNegative": 2, - "falsePositive": 1, - "precision": 0.8889, - "predictedPositive": 9, - "recall": 0.8, - "total": 126, - "trueNegative": 115, - "truePositive": 8, - "wilsonPrecisionLower95": 0.565 - }, - "task": { - "f1": 0.6087, - "falseNegative": 8, - "falsePositive": 1, - "precision": 0.875, - "predictedPositive": 8, - "recall": 0.4667, - "total": 126, - "trueNegative": 110, - "truePositive": 7, - "wilsonPrecisionLower95": 0.5291 - } - }, - "recordCount": 126 - }, - "zh-Hans": { - "intentsMeeting95Precision": 1, - "macroF1": 0.652, - "macroPrecision": 0.9656, - "macroRecall": 0.5609, - "minimumPrecision": 0.8182, - "perIntent": { - "blessing": { - "f1": 0.3333, - "falseNegative": 8, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 2, - "recall": 0.2, - "total": 126, - "trueNegative": 116, - "truePositive": 2, - "wilsonPrecisionLower95": 0.3424 - }, - "complaint": { - "f1": 0.6667, - "falseNegative": 3, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 3, - "recall": 0.5, - "total": 126, - "trueNegative": 120, - "truePositive": 3, - "wilsonPrecisionLower95": 0.4385 - }, - "confirmationDecision": { - "f1": 0.8571, - "falseNegative": 1, - "falsePositive": 2, - "precision": 0.8182, - "predictedPositive": 11, - "recall": 0.9, - "total": 126, - "trueNegative": 114, - "truePositive": 9, - "wilsonPrecisionLower95": 0.523 - }, - "followUpReminder": { - "f1": 0.8182, - "falseNegative": 4, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 9, - "recall": 0.6923, - "total": 126, - "trueNegative": 113, - "truePositive": 9, - "wilsonPrecisionLower95": 0.7009 - }, - "invitation": { - "f1": 0.5455, - "falseNegative": 5, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 3, - "recall": 0.375, - "total": 126, - "trueNegative": 118, - "truePositive": 3, - "wilsonPrecisionLower95": 0.4385 - }, - "question": { - "f1": 0.9787, - "falseNegative": 0, - "falsePositive": 1, - "precision": 0.9583, - "predictedPositive": 24, - "recall": 1.0, - "total": 126, - "trueNegative": 102, - "truePositive": 23, - "wilsonPrecisionLower95": 0.7976 - }, - "replyableMessage": { - "f1": 0.9143, - "falseNegative": 6, - "falsePositive": 6, - "precision": 0.9143, - "predictedPositive": 70, - "recall": 0.9143, - "total": 126, - "trueNegative": 50, - "truePositive": 64, - "wilsonPrecisionLower95": 0.8253 - }, - "scheduleNegotiation": { - "f1": 0.3333, - "falseNegative": 8, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 2, - "recall": 0.2, - "total": 126, - "trueNegative": 116, - "truePositive": 2, - "wilsonPrecisionLower95": 0.3424 - }, - "task": { - "f1": 0.4211, - "falseNegative": 11, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 4, - "recall": 0.2667, - "total": 126, - "trueNegative": 111, - "truePositive": 4, - "wilsonPrecisionLower95": 0.5101 - } - }, - "recordCount": 126 - } - }, - "bySource": { - "acknowledgment": { - "intentsMeeting95Precision": 0, - "macroF1": 0.0, - "macroPrecision": 0.0, - "macroRecall": 0.0, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 1, - "precision": 0.0, - "predictedPositive": 1, - "recall": 0.0, - "total": 8, - "trueNegative": 7, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 8 - }, - "blessing_boundary": { - "intentsMeeting95Precision": 0, - "macroF1": 0.1482, - "macroPrecision": 0.1667, - "macroRecall": 0.1667, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.6667, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 1, - "recall": 0.5, - "total": 20, - "trueNegative": 18, - "truePositive": 1, - "wilsonPrecisionLower95": 0.2065 - }, - "replyableMessage": { - "f1": 0.6667, - "falseNegative": 0, - "falsePositive": 4, - "precision": 0.5, - "predictedPositive": 8, - "recall": 1.0, - "total": 20, - "trueNegative": 12, - "truePositive": 4, - "wilsonPrecisionLower95": 0.2152 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 2, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 18, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 20 - }, - "blessing_golden": { - "intentsMeeting95Precision": 1, - "macroF1": 0.1772, - "macroPrecision": 0.2222, - "macroRecall": 0.1556, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.6207, - "falseNegative": 11, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 9, - "recall": 0.45, - "total": 20, - "trueNegative": 0, - "truePositive": 9, - "wilsonPrecisionLower95": 0.7009 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.9744, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 19, - "recall": 0.95, - "total": 20, - "trueNegative": 0, - "truePositive": 19, - "wilsonPrecisionLower95": 0.8318 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 20 - }, - "complaint_request": { - "intentsMeeting95Precision": 0, - "macroF1": 0.2703, - "macroPrecision": 0.3333, - "macroRecall": 0.2315, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.6667, - "falseNegative": 3, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 3, - "recall": 0.5, - "total": 6, - "trueNegative": 0, - "truePositive": 3, - "wilsonPrecisionLower95": 0.4385 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.8571, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 3, - "recall": 0.75, - "total": 6, - "trueNegative": 2, - "truePositive": 3, - "wilsonPrecisionLower95": 0.4385 - }, - "replyableMessage": { - "f1": 0.9091, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 5, - "recall": 0.8333, - "total": 6, - "trueNegative": 0, - "truePositive": 5, - "wilsonPrecisionLower95": 0.5655 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 6 - }, - "complaint_statement": { - "intentsMeeting95Precision": 0, - "macroF1": 0.2222, - "macroPrecision": 0.2222, - "macroRecall": 0.2222, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 6, - "recall": 1.0, - "total": 6, - "trueNegative": 0, - "truePositive": 6, - "wilsonPrecisionLower95": 0.6097 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 6, - "recall": 1.0, - "total": 6, - "trueNegative": 0, - "truePositive": 6, - "wilsonPrecisionLower95": 0.6097 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 6 - }, - "confirmation_decision_boundary": { - "intentsMeeting95Precision": 0, - "macroF1": 0.0, - "macroPrecision": 0.0, - "macroRecall": 0.0, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 2, - "precision": 0.0, - "predictedPositive": 2, - "recall": 0.0, - "total": 20, - "trueNegative": 18, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 3, - "precision": 0.0, - "predictedPositive": 3, - "recall": 0.0, - "total": 20, - "trueNegative": 17, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 20 - }, - "confirmation_decision_golden": { - "intentsMeeting95Precision": 2, - "macroF1": 0.2194, - "macroPrecision": 0.2222, - "macroRecall": 0.2167, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.9744, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 19, - "recall": 0.95, - "total": 20, - "trueNegative": 0, - "truePositive": 19, - "wilsonPrecisionLower95": 0.8318 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 20, - "recall": 1.0, - "total": 20, - "trueNegative": 0, - "truePositive": 20, - "wilsonPrecisionLower95": 0.8389 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 20 - }, - "conversational_message": { - "intentsMeeting95Precision": 1, - "macroF1": 0.1111, - "macroPrecision": 0.1111, - "macroRecall": 0.1111, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 10, - "trueNegative": 10, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 10, - "trueNegative": 10, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 10, - "trueNegative": 10, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 10, - "trueNegative": 10, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 10, - "trueNegative": 10, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 10, - "trueNegative": 10, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 10, - "recall": 1.0, - "total": 10, - "trueNegative": 0, - "truePositive": 10, - "wilsonPrecisionLower95": 0.7225 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 10, - "trueNegative": 10, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 10, - "trueNegative": 10, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 10 - }, - "event_statement": { - "intentsMeeting95Precision": 0, - "macroF1": 0.0, - "macroPrecision": 0.0, - "macroRecall": 0.0, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 4 - }, - "follow_up_reminder_boundary": { - "intentsMeeting95Precision": 0, - "macroF1": 0.0, - "macroPrecision": 0.0, - "macroRecall": 0.0, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 1, - "precision": 0.0, - "predictedPositive": 1, - "recall": 0.0, - "total": 20, - "trueNegative": 19, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 3, - "precision": 0.0, - "predictedPositive": 3, - "recall": 0.0, - "total": 20, - "trueNegative": 17, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 20 - }, - "follow_up_reminder_golden": { - "intentsMeeting95Precision": 1, - "macroF1": 0.2187, - "macroPrecision": 0.3333, - "macroRecall": 0.1706, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.8571, - "falseNegative": 5, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 15, - "recall": 0.75, - "total": 20, - "trueNegative": 0, - "truePositive": 15, - "wilsonPrecisionLower95": 0.7961 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.6667, - "falseNegative": 7, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 7, - "recall": 0.5, - "total": 20, - "trueNegative": 6, - "truePositive": 7, - "wilsonPrecisionLower95": 0.6457 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.4444, - "falseNegative": 10, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 4, - "recall": 0.2857, - "total": 20, - "trueNegative": 6, - "truePositive": 4, - "wilsonPrecisionLower95": 0.5101 - } - }, - "recordCount": 20 - }, - "information_question": { - "intentsMeeting95Precision": 0, - "macroF1": 0.1693, - "macroPrecision": 0.2222, - "macroRecall": 0.1389, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.6667, - "falseNegative": 2, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 2, - "recall": 0.5, - "total": 4, - "trueNegative": 0, - "truePositive": 2, - "wilsonPrecisionLower95": 0.3424 - }, - "replyableMessage": { - "f1": 0.8571, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 3, - "recall": 0.75, - "total": 4, - "trueNegative": 0, - "truePositive": 3, - "wilsonPrecisionLower95": 0.4385 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 4 - }, - "invitation_question": { - "intentsMeeting95Precision": 0, - "macroF1": 0.3175, - "macroPrecision": 0.3333, - "macroRecall": 0.3056, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.8571, - "falseNegative": 2, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 6, - "recall": 0.75, - "total": 8, - "trueNegative": 0, - "truePositive": 6, - "wilsonPrecisionLower95": 0.6097 - }, - "question": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 8, - "recall": 1.0, - "total": 8, - "trueNegative": 0, - "truePositive": 8, - "wilsonPrecisionLower95": 0.6756 - }, - "replyableMessage": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 8, - "recall": 1.0, - "total": 8, - "trueNegative": 0, - "truePositive": 8, - "wilsonPrecisionLower95": 0.6756 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 8 - }, - "negative_news": { - "intentsMeeting95Precision": 0, - "macroF1": 0.0, - "macroPrecision": 0.0, - "macroRecall": 0.0, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 1, - "precision": 0.0, - "predictedPositive": 1, - "recall": 0.0, - "total": 6, - "trueNegative": 5, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 6 - }, - "neutral_fact": { - "intentsMeeting95Precision": 0, - "macroF1": 0.0, - "macroPrecision": 0.0, - "macroRecall": 0.0, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 10, - "trueNegative": 10, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 10, - "trueNegative": 10, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 10, - "trueNegative": 10, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 10, - "trueNegative": 10, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 10, - "trueNegative": 10, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 10, - "trueNegative": 10, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 2, - "precision": 0.0, - "predictedPositive": 2, - "recall": 0.0, - "total": 10, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 10, - "trueNegative": 10, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 10, - "trueNegative": 10, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 10 - }, - "positive_feedback": { - "intentsMeeting95Precision": 0, - "macroF1": 0.1111, - "macroPrecision": 0.1111, - "macroRecall": 0.1111, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 6, - "recall": 1.0, - "total": 6, - "trueNegative": 0, - "truePositive": 6, - "wilsonPrecisionLower95": 0.6097 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 6 - }, - "quoted_question": { - "intentsMeeting95Precision": 0, - "macroF1": 0.0, - "macroPrecision": 0.0, - "macroRecall": 0.0, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 1, - "precision": 0.0, - "predictedPositive": 1, - "recall": 0.0, - "total": 4, - "trueNegative": 3, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - } - }, - "recordCount": 4 - }, - "schedule_negotiation_boundary": { - "intentsMeeting95Precision": 0, - "macroF1": 0.2541, - "macroPrecision": 0.284, - "macroRecall": 0.2639, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.5455, - "falseNegative": 5, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 3, - "recall": 0.375, - "total": 20, - "trueNegative": 12, - "truePositive": 3, - "wilsonPrecisionLower95": 0.4385 - }, - "question": { - "f1": 0.8, - "falseNegative": 0, - "falsePositive": 2, - "precision": 0.6667, - "predictedPositive": 6, - "recall": 1.0, - "total": 20, - "trueNegative": 14, - "truePositive": 4, - "wilsonPrecisionLower95": 0.3 - }, - "replyableMessage": { - "f1": 0.9412, - "falseNegative": 0, - "falsePositive": 1, - "precision": 0.8889, - "predictedPositive": 9, - "recall": 1.0, - "total": 20, - "trueNegative": 11, - "truePositive": 8, - "wilsonPrecisionLower95": 0.565 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 1, - "precision": 0.0, - "predictedPositive": 1, - "recall": 0.0, - "total": 20, - "trueNegative": 19, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 20 - }, - "schedule_negotiation_golden": { - "intentsMeeting95Precision": 2, - "macroF1": 0.2905, - "macroPrecision": 0.3275, - "macroRecall": 0.2722, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 1, - "precision": 0.0, - "predictedPositive": 1, - "recall": 0.0, - "total": 20, - "trueNegative": 19, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.973, - "falseNegative": 0, - "falsePositive": 1, - "precision": 0.9474, - "predictedPositive": 19, - "recall": 1.0, - "total": 20, - "trueNegative": 1, - "truePositive": 18, - "wilsonPrecisionLower95": 0.7536 - }, - "replyableMessage": { - "f1": 0.9744, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 19, - "recall": 0.95, - "total": 20, - "trueNegative": 0, - "truePositive": 19, - "wilsonPrecisionLower95": 0.8318 - }, - "scheduleNegotiation": { - "f1": 0.6667, - "falseNegative": 10, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 10, - "recall": 0.5, - "total": 20, - "trueNegative": 0, - "truePositive": 10, - "wilsonPrecisionLower95": 0.7225 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 20 - }, - "self_plan": { - "intentsMeeting95Precision": 0, - "macroF1": 0.1111, - "macroPrecision": 0.1111, - "macroRecall": 0.1111, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 2, - "recall": 1.0, - "total": 6, - "trueNegative": 4, - "truePositive": 2, - "wilsonPrecisionLower95": 0.3424 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 2, - "precision": 0.0, - "predictedPositive": 2, - "recall": 0.0, - "total": 6, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 6 - }, - "task_question": { - "intentsMeeting95Precision": 0, - "macroF1": 0.2963, - "macroPrecision": 0.3333, - "macroRecall": 0.2778, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 6, - "recall": 1.0, - "total": 6, - "trueNegative": 0, - "truePositive": 6, - "wilsonPrecisionLower95": 0.6097 - }, - "replyableMessage": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 6, - "recall": 1.0, - "total": 6, - "trueNegative": 0, - "truePositive": 6, - "wilsonPrecisionLower95": 0.6097 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.6667, - "falseNegative": 3, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 3, - "recall": 0.5, - "total": 6, - "trueNegative": 0, - "truePositive": 3, - "wilsonPrecisionLower95": 0.4385 - } - }, - "recordCount": 6 - }, - "task_statement": { - "intentsMeeting95Precision": 0, - "macroF1": 0.2519, - "macroPrecision": 0.3333, - "macroRecall": 0.2083, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.6667, - "falseNegative": 2, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 2, - "recall": 0.5, - "total": 8, - "trueNegative": 4, - "truePositive": 2, - "wilsonPrecisionLower95": 0.3424 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.9333, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 7, - "recall": 0.875, - "total": 8, - "trueNegative": 0, - "truePositive": 7, - "wilsonPrecisionLower95": 0.6457 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.6667, - "falseNegative": 4, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 4, - "recall": 0.5, - "total": 8, - "trueNegative": 0, - "truePositive": 4, - "wilsonPrecisionLower95": 0.5101 - } - }, - "recordCount": 8 - } - }, - "intentsMeeting95Precision": 1, - "macroF1": 0.7666, - "macroPrecision": 0.9377, - "macroRecall": 0.6819, - "minimumPrecision": 0.7917, - "perIntent": { - "blessing": { - "f1": 0.6207, - "falseNegative": 11, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 9, - "recall": 0.45, - "total": 252, - "trueNegative": 232, - "truePositive": 9, - "wilsonPrecisionLower95": 0.7009 - }, - "complaint": { - "f1": 0.8571, - "falseNegative": 3, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 9, - "recall": 0.75, - "total": 252, - "trueNegative": 240, - "truePositive": 9, - "wilsonPrecisionLower95": 0.7009 - }, - "confirmationDecision": { - "f1": 0.8636, - "falseNegative": 1, - "falsePositive": 5, - "precision": 0.7917, - "predictedPositive": 24, - "recall": 0.95, - "total": 252, - "trueNegative": 227, - "truePositive": 19, - "wilsonPrecisionLower95": 0.5953 - }, - "followUpReminder": { - "f1": 0.8444, - "falseNegative": 7, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 19, - "recall": 0.7308, - "total": 252, - "trueNegative": 226, - "truePositive": 19, - "wilsonPrecisionLower95": 0.8318 - }, - "invitation": { - "f1": 0.72, - "falseNegative": 7, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 9, - "recall": 0.5625, - "total": 252, - "trueNegative": 236, - "truePositive": 9, - "wilsonPrecisionLower95": 0.7009 - }, - "question": { - "f1": 0.9231, - "falseNegative": 4, - "falsePositive": 3, - "precision": 0.9333, - "predictedPositive": 45, - "recall": 0.913, - "total": 252, - "trueNegative": 203, - "truePositive": 42, - "wilsonPrecisionLower95": 0.8214 - }, - "replyableMessage": { - "f1": 0.9014, - "falseNegative": 12, - "falsePositive": 16, - "precision": 0.8889, - "predictedPositive": 144, - "recall": 0.9143, - "total": 252, - "trueNegative": 96, - "truePositive": 128, - "wilsonPrecisionLower95": 0.8271 - }, - "scheduleNegotiation": { - "f1": 0.6452, - "falseNegative": 10, - "falsePositive": 1, - "precision": 0.9091, - "predictedPositive": 11, - "recall": 0.5, - "total": 252, - "trueNegative": 231, - "truePositive": 10, - "wilsonPrecisionLower95": 0.6226 - }, - "task": { - "f1": 0.5238, - "falseNegative": 19, - "falsePositive": 1, - "precision": 0.9167, - "predictedPositive": 12, - "recall": 0.3667, - "total": 252, - "trueNegative": 221, - "truePositive": 11, - "wilsonPrecisionLower95": 0.6461 - } - }, - "recordCount": 252 - }, - "randomHoldout": { - "byLanguage": { - "en": { - "intentsMeeting95Precision": 5, - "macroF1": 0.6248, - "macroPrecision": 0.8629, - "macroRecall": 0.5355, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 340, - "trueNegative": 340, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.3333, - "falseNegative": 16, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 4, - "recall": 0.2, - "total": 340, - "trueNegative": 320, - "truePositive": 4, - "wilsonPrecisionLower95": 0.5101 - }, - "confirmationDecision": { - "f1": 0.9091, - "falseNegative": 0, - "falsePositive": 4, - "precision": 0.8333, - "predictedPositive": 24, - "recall": 1.0, - "total": 340, - "trueNegative": 316, - "truePositive": 20, - "wilsonPrecisionLower95": 0.6415 - }, - "followUpReminder": { - "f1": 0.9027, - "falseNegative": 9, - "falsePositive": 2, - "precision": 0.9623, - "predictedPositive": 53, - "recall": 0.85, - "total": 340, - "trueNegative": 278, - "truePositive": 51, - "wilsonPrecisionLower95": 0.8725 - }, - "invitation": { - "f1": 0.5714, - "falseNegative": 12, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 8, - "recall": 0.4, - "total": 340, - "trueNegative": 320, - "truePositive": 8, - "wilsonPrecisionLower95": 0.6756 - }, - "question": { - "f1": 0.6087, - "falseNegative": 45, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 35, - "recall": 0.4375, - "total": 340, - "trueNegative": 260, - "truePositive": 35, - "wilsonPrecisionLower95": 0.9011 - }, - "replyableMessage": { - "f1": 0.8859, - "falseNegative": 37, - "falsePositive": 5, - "precision": 0.9702, - "predictedPositive": 168, - "recall": 0.815, - "total": 340, - "trueNegative": 135, - "truePositive": 163, - "wilsonPrecisionLower95": 0.9322 - }, - "scheduleNegotiation": { - "f1": 0.8235, - "falseNegative": 6, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 14, - "recall": 0.7, - "total": 340, - "trueNegative": 320, - "truePositive": 14, - "wilsonPrecisionLower95": 0.7847 - }, - "task": { - "f1": 0.5882, - "falseNegative": 35, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 25, - "recall": 0.4167, - "total": 340, - "trueNegative": 280, - "truePositive": 25, - "wilsonPrecisionLower95": 0.8668 - } - }, - "recordCount": 340 - }, - "zh-Hans": { - "intentsMeeting95Precision": 3, - "macroF1": 0.497, - "macroPrecision": 0.7719, - "macroRecall": 0.4055, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 340, - "trueNegative": 340, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.3333, - "falseNegative": 16, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 4, - "recall": 0.2, - "total": 340, - "trueNegative": 320, - "truePositive": 4, - "wilsonPrecisionLower95": 0.5101 - }, - "confirmationDecision": { - "f1": 0.8235, - "falseNegative": 6, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 14, - "recall": 0.7, - "total": 340, - "trueNegative": 320, - "truePositive": 14, - "wilsonPrecisionLower95": 0.7847 - }, - "followUpReminder": { - "f1": 0.7879, - "falseNegative": 21, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 39, - "recall": 0.65, - "total": 340, - "trueNegative": 280, - "truePositive": 39, - "wilsonPrecisionLower95": 0.9103 - }, - "invitation": { - "f1": 0.6207, - "falseNegative": 11, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 9, - "recall": 0.45, - "total": 340, - "trueNegative": 320, - "truePositive": 9, - "wilsonPrecisionLower95": 0.7009 - }, - "question": { - "f1": 0.7786, - "falseNegative": 29, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 51, - "recall": 0.6375, - "total": 340, - "trueNegative": 260, - "truePositive": 51, - "wilsonPrecisionLower95": 0.93 - }, - "replyableMessage": { - "f1": 0.9203, - "falseNegative": 21, - "falsePositive": 10, - "precision": 0.9471, - "predictedPositive": 189, - "recall": 0.895, - "total": 340, - "trueNegative": 130, - "truePositive": 179, - "wilsonPrecisionLower95": 0.9054 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 20, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 340, - "trueNegative": 320, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.209, - "falseNegative": 53, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 7, - "recall": 0.1167, - "total": 340, - "trueNegative": 280, - "truePositive": 7, - "wilsonPrecisionLower95": 0.6457 - } - }, - "recordCount": 340 - } - }, - "bySource": { - "acknowledgment_boundary": { - "intentsMeeting95Precision": 0, - "macroF1": 0.0, - "macroPrecision": 0.0, - "macroRecall": 0.0, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 40 - }, - "complaint_support": { - "intentsMeeting95Precision": 1, - "macroF1": 0.1783, - "macroPrecision": 0.3333, - "macroRecall": 0.1389, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.3333, - "falseNegative": 32, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 8, - "recall": 0.2, - "total": 40, - "trueNegative": 0, - "truePositive": 8, - "wilsonPrecisionLower95": 0.6756 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.3673, - "falseNegative": 31, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 9, - "recall": 0.225, - "total": 40, - "trueNegative": 0, - "truePositive": 9, - "wilsonPrecisionLower95": 0.7009 - }, - "replyableMessage": { - "f1": 0.9041, - "falseNegative": 7, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 33, - "recall": 0.825, - "total": 40, - "trueNegative": 0, - "truePositive": 33, - "wilsonPrecisionLower95": 0.8957 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 40 - }, - "confirmation_decision": { - "intentsMeeting95Precision": 2, - "macroF1": 0.2026, - "macroPrecision": 0.2222, - "macroRecall": 0.1861, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.9189, - "falseNegative": 6, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 34, - "recall": 0.85, - "total": 40, - "trueNegative": 0, - "truePositive": 34, - "wilsonPrecisionLower95": 0.8985 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.9041, - "falseNegative": 7, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 33, - "recall": 0.825, - "total": 40, - "trueNegative": 0, - "truePositive": 33, - "wilsonPrecisionLower95": 0.8957 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 40 - }, - "conversation": { - "intentsMeeting95Precision": 1, - "macroF1": 0.1053, - "macroPrecision": 0.1111, - "macroRecall": 0.1, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.9474, - "falseNegative": 4, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 36, - "recall": 0.9, - "total": 40, - "trueNegative": 0, - "truePositive": 36, - "wilsonPrecisionLower95": 0.9036 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 40 - }, - "event_boundary": { - "intentsMeeting95Precision": 0, - "macroF1": 0.0, - "macroPrecision": 0.0, - "macroRecall": 0.0, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 2, - "precision": 0.0, - "predictedPositive": 2, - "recall": 0.0, - "total": 40, - "trueNegative": 38, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 40 - }, - "fixed_invitation": { - "intentsMeeting95Precision": 3, - "macroF1": 0.2797, - "macroPrecision": 0.3333, - "macroRecall": 0.2528, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.5965, - "falseNegative": 23, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 17, - "recall": 0.425, - "total": 40, - "trueNegative": 0, - "truePositive": 17, - "wilsonPrecisionLower95": 0.8157 - }, - "question": { - "f1": 0.9333, - "falseNegative": 5, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 35, - "recall": 0.875, - "total": 40, - "trueNegative": 0, - "truePositive": 35, - "wilsonPrecisionLower95": 0.9011 - }, - "replyableMessage": { - "f1": 0.9873, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 39, - "recall": 0.975, - "total": 40, - "trueNegative": 0, - "truePositive": 39, - "wilsonPrecisionLower95": 0.9103 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 40 - }, - "follow_up_instruction": { - "intentsMeeting95Precision": 3, - "macroF1": 0.2225, - "macroPrecision": 0.3333, - "macroRecall": 0.175, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.7302, - "falseNegative": 17, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 23, - "recall": 0.575, - "total": 40, - "trueNegative": 0, - "truePositive": 23, - "wilsonPrecisionLower95": 0.8569 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.8406, - "falseNegative": 11, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 29, - "recall": 0.725, - "total": 40, - "trueNegative": 0, - "truePositive": 29, - "wilsonPrecisionLower95": 0.883 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.4314, - "falseNegative": 29, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 11, - "recall": 0.275, - "total": 40, - "trueNegative": 0, - "truePositive": 11, - "wilsonPrecisionLower95": 0.7412 - } - }, - "recordCount": 40 - }, - "follow_up_personal_reminder": { - "intentsMeeting95Precision": 1, - "macroF1": 0.097, - "macroPrecision": 0.1111, - "macroRecall": 0.0861, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.8732, - "falseNegative": 9, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 31, - "recall": 0.775, - "total": 40, - "trueNegative": 0, - "truePositive": 31, - "wilsonPrecisionLower95": 0.8897 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 40 - }, - "information_question": { - "intentsMeeting95Precision": 2, - "macroF1": 0.1421, - "macroPrecision": 0.2222, - "macroRecall": 0.1083, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 2, - "precision": 0.0, - "predictedPositive": 2, - "recall": 0.0, - "total": 40, - "trueNegative": 38, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 2, - "precision": 0.0, - "predictedPositive": 2, - "recall": 0.0, - "total": 40, - "trueNegative": 38, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.4906, - "falseNegative": 27, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 13, - "recall": 0.325, - "total": 40, - "trueNegative": 0, - "truePositive": 13, - "wilsonPrecisionLower95": 0.7719 - }, - "replyableMessage": { - "f1": 0.7879, - "falseNegative": 14, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 26, - "recall": 0.65, - "total": 40, - "trueNegative": 0, - "truePositive": 26, - "wilsonPrecisionLower95": 0.8713 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 40 - }, - "negative_news": { - "intentsMeeting95Precision": 0, - "macroF1": 0.0, - "macroPrecision": 0.0, - "macroRecall": 0.0, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 3, - "precision": 0.0, - "predictedPositive": 3, - "recall": 0.0, - "total": 40, - "trueNegative": 37, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 40 - }, - "neutral_fact": { - "intentsMeeting95Precision": 0, - "macroF1": 0.0, - "macroPrecision": 0.0, - "macroRecall": 0.0, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 5, - "precision": 0.0, - "predictedPositive": 5, - "recall": 0.0, - "total": 40, - "trueNegative": 35, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 40 - }, - "positive_feedback": { - "intentsMeeting95Precision": 1, - "macroF1": 0.1068, - "macroPrecision": 0.1111, - "macroRecall": 0.1028, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.961, - "falseNegative": 3, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 37, - "recall": 0.925, - "total": 40, - "trueNegative": 0, - "truePositive": 37, - "wilsonPrecisionLower95": 0.9059 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 40 - }, - "quoted_question_boundary": { - "intentsMeeting95Precision": 0, - "macroF1": 0.0, - "macroPrecision": 0.0, - "macroRecall": 0.0, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 1, - "precision": 0.0, - "predictedPositive": 1, - "recall": 0.0, - "total": 40, - "trueNegative": 39, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 40 - }, - "schedule_negotiation": { - "intentsMeeting95Precision": 3, - "macroF1": 0.2621, - "macroPrecision": 0.3333, - "macroRecall": 0.2306, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.8406, - "falseNegative": 11, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 29, - "recall": 0.725, - "total": 40, - "trueNegative": 0, - "truePositive": 29, - "wilsonPrecisionLower95": 0.883 - }, - "replyableMessage": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 40, - "recall": 1.0, - "total": 40, - "trueNegative": 0, - "truePositive": 40, - "wilsonPrecisionLower95": 0.9124 - }, - "scheduleNegotiation": { - "f1": 0.5185, - "falseNegative": 26, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 14, - "recall": 0.35, - "total": 40, - "trueNegative": 0, - "truePositive": 14, - "wilsonPrecisionLower95": 0.7847 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 40 - }, - "task_next_action": { - "intentsMeeting95Precision": 2, - "macroF1": 0.2304, - "macroPrecision": 0.3333, - "macroRecall": 0.2056, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 1, - "precision": 0.0, - "predictedPositive": 1, - "recall": 0.0, - "total": 40, - "trueNegative": 39, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "followUpReminder": { - "f1": 0.9474, - "falseNegative": 4, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 36, - "recall": 0.9, - "total": 40, - "trueNegative": 0, - "truePositive": 36, - "wilsonPrecisionLower95": 0.9036 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.9041, - "falseNegative": 7, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 33, - "recall": 0.825, - "total": 40, - "trueNegative": 0, - "truePositive": 33, - "wilsonPrecisionLower95": 0.8957 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.2222, - "falseNegative": 35, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 5, - "recall": 0.125, - "total": 40, - "trueNegative": 0, - "truePositive": 5, - "wilsonPrecisionLower95": 0.5655 - } - }, - "recordCount": 40 - }, - "task_request": { - "intentsMeeting95Precision": 2, - "macroF1": 0.1688, - "macroPrecision": 0.2222, - "macroRecall": 0.1444, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 1, - "precision": 0.0, - "predictedPositive": 1, - "recall": 0.0, - "total": 40, - "trueNegative": 39, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.9474, - "falseNegative": 4, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 36, - "recall": 0.9, - "total": 40, - "trueNegative": 0, - "truePositive": 36, - "wilsonPrecisionLower95": 0.9036 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.5714, - "falseNegative": 24, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 16, - "recall": 0.4, - "total": 40, - "trueNegative": 0, - "truePositive": 16, - "wilsonPrecisionLower95": 0.8064 - } - }, - "recordCount": 40 - }, - "vague_future_boundary": { - "intentsMeeting95Precision": 0, - "macroF1": 0.0, - "macroPrecision": 0.0, - "macroRecall": 0.0, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 4, - "precision": 0.0, - "predictedPositive": 4, - "recall": 0.0, - "total": 40, - "trueNegative": 36, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 40 - } - }, - "intentsMeeting95Precision": 6, - "macroF1": 0.577, - "macroPrecision": 0.8701, - "macroRecall": 0.4705, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 680, - "trueNegative": 680, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.3333, - "falseNegative": 32, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 8, - "recall": 0.2, - "total": 680, - "trueNegative": 640, - "truePositive": 8, - "wilsonPrecisionLower95": 0.6756 - }, - "confirmationDecision": { - "f1": 0.8718, - "falseNegative": 6, - "falsePositive": 4, - "precision": 0.8947, - "predictedPositive": 38, - "recall": 0.85, - "total": 680, - "trueNegative": 636, - "truePositive": 34, - "wilsonPrecisionLower95": 0.7587 - }, - "followUpReminder": { - "f1": 0.8491, - "falseNegative": 30, - "falsePositive": 2, - "precision": 0.9783, - "predictedPositive": 92, - "recall": 0.75, - "total": 680, - "trueNegative": 558, - "truePositive": 90, - "wilsonPrecisionLower95": 0.9242 - }, - "invitation": { - "f1": 0.5965, - "falseNegative": 23, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 17, - "recall": 0.425, - "total": 680, - "trueNegative": 640, - "truePositive": 17, - "wilsonPrecisionLower95": 0.8157 - }, - "question": { - "f1": 0.6992, - "falseNegative": 74, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 86, - "recall": 0.5375, - "total": 680, - "trueNegative": 520, - "truePositive": 86, - "wilsonPrecisionLower95": 0.9572 - }, - "replyableMessage": { - "f1": 0.9036, - "falseNegative": 58, - "falsePositive": 15, - "precision": 0.958, - "predictedPositive": 357, - "recall": 0.855, - "total": 680, - "trueNegative": 265, - "truePositive": 342, - "wilsonPrecisionLower95": 0.9318 - }, - "scheduleNegotiation": { - "f1": 0.5185, - "falseNegative": 26, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 14, - "recall": 0.35, - "total": 680, - "trueNegative": 640, - "truePositive": 14, - "wilsonPrecisionLower95": 0.7847 - }, - "task": { - "f1": 0.4211, - "falseNegative": 88, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 32, - "recall": 0.2667, - "total": 680, - "trueNegative": 560, - "truePositive": 32, - "wilsonPrecisionLower95": 0.8928 - } - }, - "recordCount": 680 - }, - "researchOnlyComprehensive": { - "byLanguage": { - "en": { - "intentsMeeting95Precision": 1, - "macroF1": 0.2884, - "macroPrecision": 0.3871, - "macroRecall": 0.2731, - "minimumPrecision": 0.0006, - "perIntent": { - "blessing": { - "f1": 0.4667, - "falseNegative": 48, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 21, - "recall": 0.3043, - "total": 13176, - "trueNegative": 13107, - "truePositive": 21, - "wilsonPrecisionLower95": 0.8454 - }, - "complaint": { - "f1": 0.048, - "falseNegative": 421, - "falsePositive": 95, - "precision": 0.1204, - "predictedPositive": 108, - "recall": 0.03, - "total": 13176, - "trueNegative": 12647, - "truePositive": 13, - "wilsonPrecisionLower95": 0.0717 - }, - "confirmationDecision": { - "f1": 0.0012, - "falseNegative": 11, - "falsePositive": 1649, - "precision": 0.0006, - "predictedPositive": 1650, - "recall": 0.0833, - "total": 13176, - "trueNegative": 11515, - "truePositive": 1, - "wilsonPrecisionLower95": 0.0001 - }, - "followUpReminder": { - "f1": 0.1134, - "falseNegative": 114, - "falsePositive": 58, - "precision": 0.1594, - "predictedPositive": 69, - "recall": 0.088, - "total": 13176, - "trueNegative": 12993, - "truePositive": 11, - "wilsonPrecisionLower95": 0.0914 - }, - "invitation": { - "f1": 0.336, - "falseNegative": 40, - "falsePositive": 43, - "precision": 0.3281, - "predictedPositive": 64, - "recall": 0.3443, - "total": 13176, - "trueNegative": 13072, - "truePositive": 21, - "wilsonPrecisionLower95": 0.2257 - }, - "question": { - "f1": 0.4288, - "falseNegative": 1828, - "falsePositive": 378, - "precision": 0.6866, - "predictedPositive": 1206, - "recall": 0.3117, - "total": 13176, - "trueNegative": 10142, - "truePositive": 828, - "wilsonPrecisionLower95": 0.6598 - }, - "replyableMessage": { - "f1": 0.7642, - "falseNegative": 2967, - "falsePositive": 1917, - "precision": 0.805, - "predictedPositive": 9831, - "recall": 0.7273, - "total": 13176, - "trueNegative": 378, - "truePositive": 7914, - "wilsonPrecisionLower95": 0.7971 - }, - "scheduleNegotiation": { - "f1": 0.1333, - "falseNegative": 50, - "falsePositive": 171, - "precision": 0.0904, - "predictedPositive": 188, - "recall": 0.2537, - "total": 13176, - "trueNegative": 12938, - "truePositive": 17, - "wilsonPrecisionLower95": 0.0572 - }, - "task": { - "f1": 0.3039, - "falseNegative": 766, - "falsePositive": 851, - "precision": 0.2932, - "predictedPositive": 1204, - "recall": 0.3155, - "total": 13176, - "trueNegative": 11206, - "truePositive": 353, - "wilsonPrecisionLower95": 0.2682 - } - }, - "recordCount": 13176 - }, - "zh-Hans": { - "intentsMeeting95Precision": 0, - "macroF1": 0.1992, - "macroPrecision": 0.4046, - "macroRecall": 0.1613, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.2105, - "falseNegative": 15, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 2, - "recall": 0.1176, - "total": 28019, - "trueNegative": 28002, - "truePositive": 2, - "wilsonPrecisionLower95": 0.3424 - }, - "complaint": { - "f1": 0.4077, - "falseNegative": 154, - "falsePositive": 183, - "precision": 0.388, - "predictedPositive": 299, - "recall": 0.4296, - "total": 28019, - "trueNegative": 27566, - "truePositive": 116, - "wilsonPrecisionLower95": 0.3345 - }, - "confirmationDecision": { - "f1": 0.0594, - "falseNegative": 54, - "falsePositive": 41, - "precision": 0.0682, - "predictedPositive": 44, - "recall": 0.0526, - "total": 28019, - "trueNegative": 27921, - "truePositive": 3, - "wilsonPrecisionLower95": 0.0235 - }, - "followUpReminder": { - "f1": 0.1856, - "falseNegative": 234, - "falsePositive": 38, - "precision": 0.4493, - "predictedPositive": 69, - "recall": 0.117, - "total": 28019, - "trueNegative": 27716, - "truePositive": 31, - "wilsonPrecisionLower95": 0.3377 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 73, - "falsePositive": 9, - "precision": 0.0, - "predictedPositive": 9, - "recall": 0.0, - "total": 28019, - "trueNegative": 27937, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "question": { - "f1": 0.1419, - "falseNegative": 6619, - "falsePositive": 216, - "precision": 0.7234, - "predictedPositive": 781, - "recall": 0.0786, - "total": 28019, - "trueNegative": 20619, - "truePositive": 565, - "wilsonPrecisionLower95": 0.691 - }, - "replyableMessage": { - "f1": 0.7622, - "falseNegative": 9480, - "falsePositive": 1096, - "precision": 0.9393, - "predictedPositive": 18042, - "recall": 0.6413, - "total": 28019, - "trueNegative": 497, - "truePositive": 16946, - "wilsonPrecisionLower95": 0.9357 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 4, - "falsePositive": 5, - "precision": 0.0, - "predictedPositive": 5, - "recall": 0.0, - "total": 28019, - "trueNegative": 28010, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "task": { - "f1": 0.0251, - "falseNegative": 649, - "falsePositive": 127, - "precision": 0.073, - "predictedPositive": 137, - "recall": 0.0152, - "total": 28019, - "trueNegative": 27233, - "truePositive": 10, - "wilsonPrecisionLower95": 0.0401 - } - }, - "recordCount": 28019 - } - }, - "bySource": { - "AmazonScience/MASSIVE en-US test": { - "intentsMeeting95Precision": 2, - "macroF1": 0.1663, - "macroPrecision": 0.2646, - "macroRecall": 0.1307, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 2967, - "trueNegative": 2967, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 17, - "precision": 0.0, - "predictedPositive": 17, - "recall": 0.0, - "total": 2967, - "trueNegative": 2950, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 441, - "precision": 0.0, - "predictedPositive": 441, - "recall": 0.0, - "total": 2967, - "trueNegative": 2526, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "followUpReminder": { - "f1": 0.1447, - "falseNegative": 86, - "falsePositive": 44, - "precision": 0.2, - "predictedPositive": 55, - "recall": 0.1134, - "total": 2967, - "trueNegative": 2826, - "truePositive": 11, - "wilsonPrecisionLower95": 0.1155 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 7, - "precision": 0.0, - "predictedPositive": 7, - "recall": 0.0, - "total": 2967, - "trueNegative": 2960, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "question": { - "f1": 0.3532, - "falseNegative": 1266, - "falsePositive": 16, - "precision": 0.9563, - "predictedPositive": 366, - "recall": 0.2166, - "total": 2967, - "trueNegative": 1335, - "truePositive": 350, - "wilsonPrecisionLower95": 0.9302 - }, - "replyableMessage": { - "f1": 0.7951, - "falseNegative": 1009, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 1958, - "recall": 0.6599, - "total": 2967, - "trueNegative": 0, - "truePositive": 1958, - "wilsonPrecisionLower95": 0.998 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 69, - "precision": 0.0, - "predictedPositive": 69, - "recall": 0.0, - "total": 2967, - "trueNegative": 2898, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "task": { - "f1": 0.204, - "falseNegative": 440, - "falsePositive": 348, - "precision": 0.2249, - "predictedPositive": 449, - "recall": 0.1867, - "total": 2967, - "trueNegative": 2078, - "truePositive": 101, - "wilsonPrecisionLower95": 0.1887 - } - }, - "recordCount": 2967 - }, - "AmazonScience/MASSIVE zh-CN test": { - "intentsMeeting95Precision": 1, - "macroF1": 0.1386, - "macroPrecision": 0.3031, - "macroRecall": 0.1008, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 2850, - "trueNegative": 2850, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 2, - "precision": 0.0, - "predictedPositive": 2, - "recall": 0.0, - "total": 2850, - "trueNegative": 2848, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 5, - "precision": 0.0, - "predictedPositive": 5, - "recall": 0.0, - "total": 2850, - "trueNegative": 2845, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "followUpReminder": { - "f1": 0.2537, - "falseNegative": 140, - "falsePositive": 13, - "precision": 0.6667, - "predictedPositive": 39, - "recall": 0.1566, - "total": 2850, - "trueNegative": 2671, - "truePositive": 26, - "wilsonPrecisionLower95": 0.5098 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 7, - "precision": 0.0, - "predictedPositive": 7, - "recall": 0.0, - "total": 2850, - "trueNegative": 2843, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "question": { - "f1": 0.2054, - "falseNegative": 1381, - "falsePositive": 19, - "precision": 0.905, - "predictedPositive": 200, - "recall": 0.1159, - "total": 2850, - "trueNegative": 1269, - "truePositive": 181, - "wilsonPrecisionLower95": 0.8564 - }, - "replyableMessage": { - "f1": 0.7684, - "falseNegative": 1072, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 1778, - "recall": 0.6239, - "total": 2850, - "trueNegative": 0, - "truePositive": 1778, - "wilsonPrecisionLower95": 0.9978 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 4, - "precision": 0.0, - "predictedPositive": 4, - "recall": 0.0, - "total": 2850, - "trueNegative": 2846, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "task": { - "f1": 0.0196, - "falseNegative": 473, - "falsePositive": 27, - "precision": 0.1562, - "predictedPositive": 32, - "recall": 0.0105, - "total": 2850, - "trueNegative": 2345, - "truePositive": 5, - "wilsonPrecisionLower95": 0.0686 - } - }, - "recordCount": 2850 - }, - "CFPB via hpe-ai/customer-complaints": { - "intentsMeeting95Precision": 1, - "macroF1": 0.0968, - "macroPrecision": 0.2685, - "macroRecall": 0.0714, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 250, - "trueNegative": 250, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0159, - "falseNegative": 248, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 2, - "recall": 0.008, - "total": 250, - "trueNegative": 0, - "truePositive": 2, - "wilsonPrecisionLower95": 0.3424 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 4, - "precision": 0.0, - "predictedPositive": 4, - "recall": 0.0, - "total": 250, - "trueNegative": 246, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 250, - "trueNegative": 250, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 250, - "trueNegative": 250, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 17, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 250, - "trueNegative": 233, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.7013, - "falseNegative": 115, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 135, - "recall": 0.54, - "total": 250, - "trueNegative": 0, - "truePositive": 135, - "wilsonPrecisionLower95": 0.9723 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 250, - "trueNegative": 250, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.1538, - "falseNegative": 48, - "falsePositive": 7, - "precision": 0.4167, - "predictedPositive": 12, - "recall": 0.0943, - "total": 250, - "trueNegative": 190, - "truePositive": 5, - "wilsonPrecisionLower95": 0.1933 - } - }, - "recordCount": 250 - }, - "CPED test split": { - "intentsMeeting95Precision": 0, - "macroF1": 0.1434, - "macroPrecision": 0.3223, - "macroRecall": 0.1091, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.2105, - "falseNegative": 15, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 2, - "recall": 0.1176, - "total": 25049, - "trueNegative": 25032, - "truePositive": 2, - "wilsonPrecisionLower95": 0.3424 - }, - "complaint": { - "f1": 0.018, - "falseNegative": 147, - "falsePositive": 181, - "precision": 0.0163, - "predictedPositive": 184, - "recall": 0.02, - "total": 25049, - "trueNegative": 24718, - "truePositive": 3, - "wilsonPrecisionLower95": 0.0056 - }, - "confirmationDecision": { - "f1": 0.0625, - "falseNegative": 54, - "falsePositive": 36, - "precision": 0.0769, - "predictedPositive": 39, - "recall": 0.0526, - "total": 25049, - "trueNegative": 24956, - "truePositive": 3, - "wilsonPrecisionLower95": 0.0265 - }, - "followUpReminder": { - "f1": 0.0775, - "falseNegative": 94, - "falsePositive": 25, - "precision": 0.1667, - "predictedPositive": 30, - "recall": 0.0505, - "total": 25049, - "trueNegative": 24925, - "truePositive": 5, - "wilsonPrecisionLower95": 0.0734 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 73, - "falsePositive": 2, - "precision": 0.0, - "predictedPositive": 2, - "recall": 0.0, - "total": 25049, - "trueNegative": 24974, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "question": { - "f1": 0.1245, - "falseNegative": 5202, - "falsePositive": 197, - "precision": 0.6609, - "predictedPositive": 581, - "recall": 0.0687, - "total": 25049, - "trueNegative": 19266, - "truePositive": 384, - "wilsonPrecisionLower95": 0.6215 - }, - "replyableMessage": { - "f1": 0.7609, - "falseNegative": 8378, - "falsePositive": 1096, - "precision": 0.9322, - "predictedPositive": 16174, - "recall": 0.6428, - "total": 25049, - "trueNegative": 497, - "truePositive": 15078, - "wilsonPrecisionLower95": 0.9283 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 4, - "falsePositive": 1, - "precision": 0.0, - "predictedPositive": 1, - "recall": 0.0, - "total": 25049, - "trueNegative": 25044, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "task": { - "f1": 0.0366, - "falseNegative": 163, - "falsePositive": 100, - "precision": 0.0476, - "predictedPositive": 105, - "recall": 0.0298, - "total": 25049, - "trueNegative": 24781, - "truePositive": 5, - "wilsonPrecisionLower95": 0.0205 - } - }, - "recordCount": 25049 - }, - "Google Research GoEmotions test": { - "intentsMeeting95Precision": 2, - "macroF1": 0.2218, - "macroPrecision": 0.3073, - "macroRecall": 0.1928, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.4651, - "falseNegative": 46, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 20, - "recall": 0.303, - "total": 5418, - "trueNegative": 5352, - "truePositive": 20, - "wilsonPrecisionLower95": 0.8389 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 20, - "falsePositive": 27, - "precision": 0.0, - "predictedPositive": 27, - "recall": 0.0, - "total": 5418, - "trueNegative": 5371, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 742, - "precision": 0.0, - "predictedPositive": 742, - "recall": 0.0, - "total": 5418, - "trueNegative": 4676, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 6, - "precision": 0.0, - "predictedPositive": 6, - "recall": 0.0, - "total": 5418, - "trueNegative": 5412, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 8, - "precision": 0.0, - "predictedPositive": 8, - "recall": 0.0, - "total": 5418, - "trueNegative": 5410, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "question": { - "f1": 0.3991, - "falseNegative": 363, - "falsePositive": 146, - "precision": 0.5365, - "predictedPositive": 315, - "recall": 0.3177, - "total": 5418, - "trueNegative": 4740, - "truePositive": 169, - "wilsonPrecisionLower95": 0.4813 - }, - "replyableMessage": { - "f1": 0.8467, - "falseNegative": 1440, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 3978, - "recall": 0.7342, - "total": 5418, - "trueNegative": 0, - "truePositive": 3978, - "wilsonPrecisionLower95": 0.999 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 23, - "precision": 0.0, - "predictedPositive": 23, - "recall": 0.0, - "total": 5418, - "trueNegative": 5395, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "task": { - "f1": 0.2857, - "falseNegative": 31, - "falsePositive": 64, - "precision": 0.2289, - "predictedPositive": 83, - "recall": 0.38, - "total": 5418, - "trueNegative": 5304, - "truePositive": 19, - "wilsonPrecisionLower95": 0.1517 - } - }, - "recordCount": 5418 - }, - "Meituan-Dianping/ASAP test": { - "intentsMeeting95Precision": 2, - "macroF1": 0.203, - "macroPrecision": 0.2222, - "macroRecall": 0.188, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 120, - "trueNegative": 120, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.97, - "falseNegative": 7, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 113, - "recall": 0.9417, - "total": 120, - "trueNegative": 0, - "truePositive": 113, - "wilsonPrecisionLower95": 0.9671 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 120, - "trueNegative": 120, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 120, - "trueNegative": 120, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 120, - "trueNegative": 120, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 36, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 120, - "trueNegative": 84, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.8571, - "falseNegative": 30, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 90, - "recall": 0.75, - "total": 120, - "trueNegative": 0, - "truePositive": 90, - "wilsonPrecisionLower95": 0.9591 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 120, - "trueNegative": 120, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 13, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 120, - "trueNegative": 107, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 120 - }, - "Sulak2020 GitHub issues multi-repo dataset": { - "intentsMeeting95Precision": 1, - "macroF1": 0.1573, - "macroPrecision": 0.2376, - "macroRecall": 0.1229, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 2, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 1004, - "trueNegative": 1002, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0995, - "falseNegative": 153, - "falsePositive": 46, - "precision": 0.193, - "predictedPositive": 57, - "recall": 0.0671, - "total": 1004, - "trueNegative": 794, - "truePositive": 11, - "wilsonPrecisionLower95": 0.1113 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 2, - "falsePositive": 84, - "precision": 0.0, - "predictedPositive": 84, - "recall": 0.0, - "total": 1004, - "trueNegative": 918, - "truePositive": 0, - "wilsonPrecisionLower95": -0.0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 2, - "falsePositive": 1, - "precision": 0.0, - "predictedPositive": 1, - "recall": 0.0, - "total": 1004, - "trueNegative": 1001, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 2, - "precision": 0.0, - "predictedPositive": 2, - "recall": 0.0, - "total": 1004, - "trueNegative": 1002, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "question": { - "f1": 0.4474, - "falseNegative": 39, - "falsePositive": 3, - "precision": 0.85, - "predictedPositive": 20, - "recall": 0.3036, - "total": 1004, - "trueNegative": 945, - "truePositive": 17, - "wilsonPrecisionLower95": 0.6396 - }, - "replyableMessage": { - "f1": 0.8326, - "falseNegative": 288, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 716, - "recall": 0.7131, - "total": 1004, - "trueNegative": 0, - "truePositive": 716, - "wilsonPrecisionLower95": 0.9947 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 25, - "falsePositive": 4, - "precision": 0.0, - "predictedPositive": 4, - "recall": 0.0, - "total": 1004, - "trueNegative": 975, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "task": { - "f1": 0.036, - "falseNegative": 88, - "falsePositive": 19, - "precision": 0.0952, - "predictedPositive": 21, - "recall": 0.0222, - "total": 1004, - "trueNegative": 895, - "truePositive": 2, - "wilsonPrecisionLower95": 0.0265 - } - }, - "recordCount": 1004 - }, - "Verified Enron Intent Dataset": { - "intentsMeeting95Precision": 0, - "macroF1": 0.3596, - "macroPrecision": 0.3263, - "macroRecall": 0.4461, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 1, - "recall": 1.0, - "total": 3537, - "trueNegative": 3536, - "truePositive": 1, - "wilsonPrecisionLower95": 0.2065 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 5, - "precision": 0.0, - "predictedPositive": 5, - "recall": 0.0, - "total": 3537, - "trueNegative": 3532, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "confirmationDecision": { - "f1": 0.0051, - "falseNegative": 9, - "falsePositive": 378, - "precision": 0.0026, - "predictedPositive": 379, - "recall": 0.1, - "total": 3537, - "trueNegative": 3149, - "truePositive": 1, - "wilsonPrecisionLower95": 0.0005 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 26, - "falsePositive": 7, - "precision": 0.0, - "predictedPositive": 7, - "recall": 0.0, - "total": 3537, - "trueNegative": 3504, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "invitation": { - "f1": 0.3889, - "falseNegative": 40, - "falsePositive": 26, - "precision": 0.4468, - "predictedPositive": 47, - "recall": 0.3443, - "total": 3537, - "trueNegative": 3450, - "truePositive": 21, - "wilsonPrecisionLower95": 0.3141 - }, - "question": { - "f1": 0.6213, - "falseNegative": 143, - "falsePositive": 213, - "precision": 0.5782, - "predictedPositive": 505, - "recall": 0.6713, - "total": 3537, - "trueNegative": 2889, - "truePositive": 292, - "wilsonPrecisionLower95": 0.5347 - }, - "replyableMessage": { - "f1": 0.5259, - "falseNegative": 115, - "falsePositive": 1917, - "precision": 0.3702, - "predictedPositive": 3044, - "recall": 0.9074, - "total": 3537, - "trueNegative": 378, - "truePositive": 1127, - "wilsonPrecisionLower95": 0.3533 - }, - "scheduleNegotiation": { - "f1": 0.2537, - "falseNegative": 25, - "falsePositive": 75, - "precision": 0.1848, - "predictedPositive": 92, - "recall": 0.4048, - "total": 3537, - "trueNegative": 3420, - "truePositive": 17, - "wilsonPrecisionLower95": 0.1187 - }, - "task": { - "f1": 0.4414, - "falseNegative": 159, - "falsePositive": 413, - "precision": 0.3537, - "predictedPositive": 639, - "recall": 0.587, - "total": 3537, - "trueNegative": 2739, - "truePositive": 226, - "wilsonPrecisionLower95": 0.3176 - } - }, - "recordCount": 3537 - } - }, - "intentsMeeting95Precision": 1, - "macroF1": 0.2645, - "macroPrecision": 0.4292, - "macroRecall": 0.225, - "minimumPrecision": 0.0024, - "perIntent": { - "blessing": { - "f1": 0.422, - "falseNegative": 63, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 23, - "recall": 0.2674, - "total": 41195, - "trueNegative": 41109, - "truePositive": 23, - "wilsonPrecisionLower95": 0.8569 - }, - "complaint": { - "f1": 0.2322, - "falseNegative": 575, - "falsePositive": 278, - "precision": 0.317, - "predictedPositive": 407, - "recall": 0.1832, - "total": 41195, - "trueNegative": 40213, - "truePositive": 129, - "wilsonPrecisionLower95": 0.2736 - }, - "confirmationDecision": { - "f1": 0.0045, - "falseNegative": 65, - "falsePositive": 1690, - "precision": 0.0024, - "predictedPositive": 1694, - "recall": 0.058, - "total": 41195, - "trueNegative": 39436, - "truePositive": 4, - "wilsonPrecisionLower95": 0.0009 - }, - "followUpReminder": { - "f1": 0.1591, - "falseNegative": 348, - "falsePositive": 96, - "precision": 0.3043, - "predictedPositive": 138, - "recall": 0.1077, - "total": 41195, - "trueNegative": 40709, - "truePositive": 42, - "wilsonPrecisionLower95": 0.2337 - }, - "invitation": { - "f1": 0.2029, - "falseNegative": 113, - "falsePositive": 52, - "precision": 0.2877, - "predictedPositive": 73, - "recall": 0.1567, - "total": 41195, - "trueNegative": 41009, - "truePositive": 21, - "wilsonPrecisionLower95": 0.1965 - }, - "question": { - "f1": 0.2356, - "falseNegative": 8447, - "falsePositive": 594, - "precision": 0.7011, - "predictedPositive": 1987, - "recall": 0.1416, - "total": 41195, - "trueNegative": 30761, - "truePositive": 1393, - "wilsonPrecisionLower95": 0.6806 - }, - "replyableMessage": { - "f1": 0.7628, - "falseNegative": 12447, - "falsePositive": 3013, - "precision": 0.8919, - "predictedPositive": 27873, - "recall": 0.6664, - "total": 41195, - "trueNegative": 875, - "truePositive": 24860, - "wilsonPrecisionLower95": 0.8882 - }, - "scheduleNegotiation": { - "f1": 0.1288, - "falseNegative": 54, - "falsePositive": 176, - "precision": 0.0881, - "predictedPositive": 193, - "recall": 0.2394, - "total": 41195, - "trueNegative": 40948, - "truePositive": 17, - "wilsonPrecisionLower95": 0.0557 - }, - "task": { - "f1": 0.2328, - "falseNegative": 1415, - "falsePositive": 978, - "precision": 0.2707, - "predictedPositive": 1341, - "recall": 0.2042, - "total": 41195, - "trueNegative": 38439, - "truePositive": 363, - "wilsonPrecisionLower95": 0.2476 - } - }, - "recordCount": 41195 - }, - "targetedRelease": { - "byLanguage": { - "en": { - "intentsMeeting95Precision": 4, - "macroF1": 0.4252, - "macroPrecision": 0.5437, - "macroRecall": 0.3679, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 320, - "trueNegative": 320, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.6957, - "falseNegative": 56, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 64, - "recall": 0.5333, - "total": 320, - "trueNegative": 200, - "truePositive": 64, - "wilsonPrecisionLower95": 0.9434 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 8, - "precision": 0.0, - "predictedPositive": 8, - "recall": 0.0, - "total": 320, - "trueNegative": 312, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "followUpReminder": { - "f1": 0.961, - "falseNegative": 3, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 37, - "recall": 0.925, - "total": 320, - "trueNegative": 280, - "truePositive": 37, - "wilsonPrecisionLower95": 0.9059 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 320, - "trueNegative": 320, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.4762, - "falseNegative": 55, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 25, - "recall": 0.3125, - "total": 320, - "trueNegative": 240, - "truePositive": 25, - "wilsonPrecisionLower95": 0.8668 - }, - "replyableMessage": { - "f1": 0.8819, - "falseNegative": 32, - "falsePositive": 13, - "precision": 0.9282, - "predictedPositive": 181, - "recall": 0.84, - "total": 320, - "trueNegative": 107, - "truePositive": 168, - "wilsonPrecisionLower95": 0.881 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 320, - "trueNegative": 320, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.8116, - "falseNegative": 24, - "falsePositive": 2, - "precision": 0.9655, - "predictedPositive": 58, - "recall": 0.7, - "total": 320, - "trueNegative": 238, - "truePositive": 56, - "wilsonPrecisionLower95": 0.8827 - } - }, - "recordCount": 320 - }, - "zh-Hans": { - "intentsMeeting95Precision": 4, - "macroF1": 0.3643, - "macroPrecision": 0.5501, - "macroRecall": 0.3071, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 320, - "trueNegative": 320, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.4516, - "falseNegative": 85, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 35, - "recall": 0.2917, - "total": 320, - "trueNegative": 200, - "truePositive": 35, - "wilsonPrecisionLower95": 0.9011 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 320, - "trueNegative": 320, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.2609, - "falseNegative": 34, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 6, - "recall": 0.15, - "total": 320, - "trueNegative": 280, - "truePositive": 6, - "wilsonPrecisionLower95": 0.6097 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 320, - "trueNegative": 320, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.8811, - "falseNegative": 17, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 63, - "recall": 0.7875, - "total": 320, - "trueNegative": 240, - "truePositive": 63, - "wilsonPrecisionLower95": 0.9425 - }, - "replyableMessage": { - "f1": 0.9552, - "falseNegative": 8, - "falsePositive": 10, - "precision": 0.9505, - "predictedPositive": 202, - "recall": 0.96, - "total": 320, - "trueNegative": 110, - "truePositive": 192, - "wilsonPrecisionLower95": 0.9113 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 320, - "trueNegative": 320, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.7302, - "falseNegative": 34, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 46, - "recall": 0.575, - "total": 320, - "trueNegative": 240, - "truePositive": 46, - "wilsonPrecisionLower95": 0.9229 - } - }, - "recordCount": 320 - } - }, - "bySource": { - "release_complaint_brief": { - "intentsMeeting95Precision": 2, - "macroF1": 0.1893, - "macroPrecision": 0.2222, - "macroRecall": 0.1708, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.7097, - "falseNegative": 36, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 44, - "recall": 0.55, - "total": 80, - "trueNegative": 0, - "truePositive": 44, - "wilsonPrecisionLower95": 0.9197 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.9937, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 79, - "recall": 0.9875, - "total": 80, - "trueNegative": 0, - "truePositive": 79, - "wilsonPrecisionLower95": 0.9536 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 80 - }, - "release_complaint_question": { - "intentsMeeting95Precision": 3, - "macroF1": 0.2152, - "macroPrecision": 0.3333, - "macroRecall": 0.1667, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.5586, - "falseNegative": 49, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 31, - "recall": 0.3875, - "total": 80, - "trueNegative": 0, - "truePositive": 31, - "wilsonPrecisionLower95": 0.8897 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.5047, - "falseNegative": 53, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 27, - "recall": 0.3375, - "total": 80, - "trueNegative": 0, - "truePositive": 27, - "wilsonPrecisionLower95": 0.8754 - }, - "replyableMessage": { - "f1": 0.8732, - "falseNegative": 18, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 62, - "recall": 0.775, - "total": 80, - "trueNegative": 0, - "truePositive": 62, - "wilsonPrecisionLower95": 0.9417 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 80 - }, - "release_complaint_unspoken": { - "intentsMeeting95Precision": 2, - "macroF1": 0.1565, - "macroPrecision": 0.2222, - "macroRecall": 0.1333, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.4615, - "falseNegative": 56, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 24, - "recall": 0.3, - "total": 80, - "trueNegative": 0, - "truePositive": 24, - "wilsonPrecisionLower95": 0.862 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.9474, - "falseNegative": 8, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 72, - "recall": 0.9, - "total": 80, - "trueNegative": 0, - "truePositive": 72, - "wilsonPrecisionLower95": 0.9493 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 80 - }, - "release_negative_boundary": { - "intentsMeeting95Precision": 0, - "macroF1": 0.0, - "macroPrecision": 0.0, - "macroRecall": 0.0, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 80 - }, - "release_personal_boundary": { - "intentsMeeting95Precision": 1, - "macroF1": 0.0777, - "macroPrecision": 0.1111, - "macroRecall": 0.0597, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.6992, - "falseNegative": 37, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 43, - "recall": 0.5375, - "total": 80, - "trueNegative": 0, - "truePositive": 43, - "wilsonPrecisionLower95": 0.918 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 4, - "precision": 0.0, - "predictedPositive": 4, - "recall": 0.0, - "total": 80, - "trueNegative": 76, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 80 - }, - "release_resolved_boundary": { - "intentsMeeting95Precision": 0, - "macroF1": 0.0, - "macroPrecision": 0.0, - "macroRecall": 0.0, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 19, - "precision": 0.0, - "predictedPositive": 19, - "recall": 0.0, - "total": 80, - "trueNegative": 61, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 2, - "precision": 0.0, - "predictedPositive": 2, - "recall": 0.0, - "total": 80, - "trueNegative": 78, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - } - }, - "recordCount": 80 - }, - "release_task_owner": { - "intentsMeeting95Precision": 2, - "macroF1": 0.2, - "macroPrecision": 0.2222, - "macroRecall": 0.1847, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 8, - "precision": 0.0, - "predictedPositive": 8, - "recall": 0.0, - "total": 80, - "trueNegative": 72, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.9937, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 79, - "recall": 0.9875, - "total": 80, - "trueNegative": 0, - "truePositive": 79, - "wilsonPrecisionLower95": 0.9536 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.806, - "falseNegative": 26, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 54, - "recall": 0.675, - "total": 80, - "trueNegative": 0, - "truePositive": 54, - "wilsonPrecisionLower95": 0.9336 - } - }, - "recordCount": 80 - }, - "release_task_polite_request": { - "intentsMeeting95Precision": 3, - "macroF1": 0.2816, - "macroPrecision": 0.3333, - "macroRecall": 0.2458, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.8652, - "falseNegative": 19, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 61, - "recall": 0.7625, - "total": 80, - "trueNegative": 0, - "truePositive": 61, - "wilsonPrecisionLower95": 0.9408 - }, - "replyableMessage": { - "f1": 0.9189, - "falseNegative": 12, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 68, - "recall": 0.85, - "total": 80, - "trueNegative": 0, - "truePositive": 68, - "wilsonPrecisionLower95": 0.9465 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.75, - "falseNegative": 32, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 48, - "recall": 0.6, - "total": 80, - "trueNegative": 0, - "truePositive": 48, - "wilsonPrecisionLower95": 0.9259 - } - }, - "recordCount": 80 - } - }, - "intentsMeeting95Precision": 4, - "macroF1": 0.4095, - "macroPrecision": 0.5467, - "macroRecall": 0.3375, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 640, - "trueNegative": 640, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.5841, - "falseNegative": 141, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 99, - "recall": 0.4125, - "total": 640, - "trueNegative": 400, - "truePositive": 99, - "wilsonPrecisionLower95": 0.9626 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 8, - "precision": 0.0, - "predictedPositive": 8, - "recall": 0.0, - "total": 640, - "trueNegative": 632, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "followUpReminder": { - "f1": 0.6992, - "falseNegative": 37, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 43, - "recall": 0.5375, - "total": 640, - "trueNegative": 560, - "truePositive": 43, - "wilsonPrecisionLower95": 0.918 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 640, - "trueNegative": 640, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.7097, - "falseNegative": 72, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 88, - "recall": 0.55, - "total": 640, - "trueNegative": 480, - "truePositive": 88, - "wilsonPrecisionLower95": 0.9582 - }, - "replyableMessage": { - "f1": 0.9195, - "falseNegative": 40, - "falsePositive": 23, - "precision": 0.9399, - "predictedPositive": 383, - "recall": 0.9, - "total": 640, - "trueNegative": 217, - "truePositive": 360, - "wilsonPrecisionLower95": 0.9115 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 640, - "trueNegative": 640, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.7727, - "falseNegative": 58, - "falsePositive": 2, - "precision": 0.9808, - "predictedPositive": 104, - "recall": 0.6375, - "total": 640, - "trueNegative": 478, - "truePositive": 102, - "wilsonPrecisionLower95": 0.9326 - } - }, - "recordCount": 640 - } - }, - "generatedAtUnix": 1787825777, - "openTrainingRecords": 3845, - "overlapChecks": { - "ModelTraining/ClipboardSemantics/comprehensive-online-holdout-corpus.jsonl": 0, - "ModelTraining/ClipboardSemantics/random-holdout-corpus.jsonl": 0, - "ModelTraining/ClipboardSemantics/targeted-release-holdout-corpus.jsonl": 0 - }, - "platformLimitation": "Linux surrogate research only. Apple Create ML/Core ML artifacts must be retrained and tested on macOS before deployment.", - "releaseGatePassed": false, - "roundCount": 20, - "schemaVersion": 1, - "seed": 20260827, - "selectedConfiguration": { - "alpha": 2e-05, - "augmentation": "none", - "char_max": 5, - "char_min": 2, - "external_weight": 0.5700000000000001, - "hard_example_weight": 1.0, - "l1_ratio": 0.0, - "max_char_features": 45000, - "max_word_features": 22000, - "min_df": 2, - "pseudo_weight": 0.16499999999999998, - "round": 1, - "word_max": 2 - }, - "selectedRound": 1, - "selectedRoundsByIntent": { - "blessing": 19, - "complaint": 1, - "confirmationDecision": 3, - "followUpReminder": 1, - "invitation": 1, - "question": 1, - "replyableMessage": 1, - "scheduleNegotiation": 7, - "task": 15 - }, - "selectedThresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 180, - "recall": 1.0, - "total": 1485, - "trueNegative": 1305, - "truePositive": 180, - "wilsonPrecisionLower95": 0.9791 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9526, - "falseNegative": 9, - "falsePositive": 8, - "precision": 0.9553, - "predictedPositive": 179, - "recall": 0.95, - "total": 1485, - "trueNegative": 1297, - "truePositive": 171, - "wilsonPrecisionLower95": 0.9143 - }, - "threshold": 0.54 - } - }, - "metrics": { - "f1": 0.9764, - "falseNegative": 9, - "falsePositive": 8, - "precision": 0.9777, - "predictedPositive": 359, - "recall": 0.975, - "total": 2970, - "trueNegative": 2602, - "truePositive": 351, - "wilsonPrecisionLower95": 0.9567 - }, - "threshold": 0.54 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9727, - "falseNegative": 1, - "falsePositive": 4, - "precision": 0.957, - "predictedPositive": 93, - "recall": 0.9889, - "total": 1485, - "trueNegative": 1391, - "truePositive": 89, - "wilsonPrecisionLower95": 0.8946 - }, - "threshold": 0.585 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7815, - "falseNegative": 31, - "falsePositive": 2, - "precision": 0.9672, - "predictedPositive": 61, - "recall": 0.6556, - "total": 1485, - "trueNegative": 1393, - "truePositive": 59, - "wilsonPrecisionLower95": 0.8881 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.8701, - "falseNegative": 36, - "falsePositive": 7, - "precision": 0.9536, - "predictedPositive": 151, - "recall": 0.8, - "total": 2970, - "trueNegative": 2783, - "truePositive": 144, - "wilsonPrecisionLower95": 0.9074 - }, - "threshold": 0.555 - }, - "followUpReminder": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8554, - "falseNegative": 41, - "falsePositive": 6, - "precision": 0.9586, - "predictedPositive": 145, - "recall": 0.7722, - "total": 1485, - "trueNegative": 1299, - "truePositive": 139, - "wilsonPrecisionLower95": 0.9127 - }, - "threshold": 0.83 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8354, - "falseNegative": 48, - "falsePositive": 4, - "precision": 0.9706, - "predictedPositive": 136, - "recall": 0.7333, - "total": 1485, - "trueNegative": 1301, - "truePositive": 132, - "wilsonPrecisionLower95": 0.9268 - }, - "threshold": 0.505 - } - }, - "metrics": { - "f1": 0.7994, - "falseNegative": 113, - "falsePositive": 11, - "precision": 0.9574, - "predictedPositive": 258, - "recall": 0.6861, - "total": 2970, - "trueNegative": 2599, - "truePositive": 247, - "wilsonPrecisionLower95": 0.9253 - }, - "threshold": 0.815 - }, - "invitation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8302, - "falseNegative": 24, - "falsePositive": 3, - "precision": 0.9565, - "predictedPositive": 69, - "recall": 0.7333, - "total": 1485, - "trueNegative": 1392, - "truePositive": 66, - "wilsonPrecisionLower95": 0.8798 - }, - "threshold": 0.935 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6765, - "falseNegative": 44, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 46, - "recall": 0.5111, - "total": 1485, - "trueNegative": 1395, - "truePositive": 46, - "wilsonPrecisionLower95": 0.9229 - }, - "threshold": 0.505 - } - }, - "metrics": { - "f1": 0.5476, - "falseNegative": 111, - "falsePositive": 3, - "precision": 0.9583, - "predictedPositive": 72, - "recall": 0.3833, - "total": 2970, - "trueNegative": 2787, - "truePositive": 69, - "wilsonPrecisionLower95": 0.8845 - }, - "threshold": 0.935 - }, - "question": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8445, - "falseNegative": 76, - "falsePositive": 12, - "precision": 0.9522, - "predictedPositive": 251, - "recall": 0.7587, - "total": 1485, - "trueNegative": 1158, - "truePositive": 239, - "wilsonPrecisionLower95": 0.9183 - }, - "threshold": 0.725 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8536, - "falseNegative": 76, - "falsePositive": 6, - "precision": 0.9755, - "predictedPositive": 245, - "recall": 0.7587, - "total": 1485, - "trueNegative": 1164, - "truePositive": 239, - "wilsonPrecisionLower95": 0.9476 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.8729, - "falseNegative": 122, - "falsePositive": 26, - "precision": 0.9513, - "predictedPositive": 534, - "recall": 0.8063, - "total": 2970, - "trueNegative": 2314, - "truePositive": 508, - "wilsonPrecisionLower95": 0.9296 - }, - "threshold": 0.515 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.948, - "falseNegative": 52, - "falsePositive": 46, - "precision": 0.951, - "predictedPositive": 939, - "recall": 0.945, - "total": 1485, - "trueNegative": 494, - "truePositive": 893, - "wilsonPrecisionLower95": 0.9353 - }, - "threshold": 0.7 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9424, - "falseNegative": 62, - "falsePositive": 46, - "precision": 0.9505, - "predictedPositive": 929, - "recall": 0.9344, - "total": 1485, - "trueNegative": 494, - "truePositive": 883, - "wilsonPrecisionLower95": 0.9346 - }, - "threshold": 0.695 - } - }, - "metrics": { - "f1": 0.9454, - "falseNegative": 115, - "falsePositive": 90, - "precision": 0.9517, - "predictedPositive": 1865, - "recall": 0.9392, - "total": 2970, - "trueNegative": 990, - "truePositive": 1775, - "wilsonPrecisionLower95": 0.9411 - }, - "threshold": 0.7 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 45, - "recall": 1.0, - "total": 1485, - "trueNegative": 1440, - "truePositive": 45, - "wilsonPrecisionLower95": 0.9213 - }, - "threshold": 0.66 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 45, - "recall": 1.0, - "total": 1485, - "trueNegative": 1440, - "truePositive": 45, - "wilsonPrecisionLower95": 0.9213 - }, - "threshold": 0.55 - } - }, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 90, - "recall": 1.0, - "total": 2970, - "trueNegative": 2880, - "truePositive": 90, - "wilsonPrecisionLower95": 0.9591 - }, - "threshold": 0.66 - }, - "task": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8593, - "falseNegative": 79, - "falsePositive": 13, - "precision": 0.9558, - "predictedPositive": 294, - "recall": 0.7806, - "total": 1485, - "trueNegative": 1112, - "truePositive": 281, - "wilsonPrecisionLower95": 0.9258 - }, - "threshold": 0.98 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8166, - "falseNegative": 104, - "falsePositive": 11, - "precision": 0.9588, - "predictedPositive": 267, - "recall": 0.7111, - "total": 1485, - "trueNegative": 1114, - "truePositive": 256, - "wilsonPrecisionLower95": 0.9277 - }, - "threshold": 0.96 - } - }, - "metrics": { - "f1": 0.8434, - "falseNegative": 176, - "falsePositive": 26, - "precision": 0.9544, - "predictedPositive": 570, - "recall": 0.7556, - "total": 2970, - "trueNegative": 2224, - "truePositive": 544, - "wilsonPrecisionLower95": 0.934 - }, - "threshold": 0.965 - } - }, - "selectionObjective": 0.8669, - "trainingPolicy": "No user clipboard data. Official partial labels plus deterministic label-preserving augmentation and three-round high-confidence self-training consensus. Frozen holdouts never enter training or threshold calibration." -} diff --git a/ModelTraining/ClipboardSemantics/IterativeResearch/PerIntentFineTune/rounds.json b/ModelTraining/ClipboardSemantics/IterativeResearch/PerIntentFineTune/rounds.json deleted file mode 100644 index f64dde3..0000000 --- a/ModelTraining/ClipboardSemantics/IterativeResearch/PerIntentFineTune/rounds.json +++ /dev/null @@ -1,14831 +0,0 @@ -[ - { - "configuration": { - "alpha": 2e-05, - "augmentation": "none", - "char_max": 5, - "char_min": 2, - "external_weight": 0.5700000000000001, - "hard_example_weight": 1.0, - "l1_ratio": 0.0, - "max_char_features": 45000, - "max_word_features": 22000, - "min_df": 2, - "pseudo_weight": 0.16499999999999998, - "round": 1, - "word_max": 2 - }, - "featureCount": 56451, - "pseudoLabelCounts": {}, - "round": 1, - "selectionObjective": 0.8669, - "syntheticRegression": { - "intentsMeeting95Precision": 6, - "macroF1": 0.8088, - "macroPrecision": 0.9661, - "macroRecall": 0.7541, - "minimumPrecision": 0.8589, - "perIntent": { - "blessing": { - "f1": 0.125, - "falseNegative": 84, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 6, - "recall": 0.0667, - "total": 2970, - "trueNegative": 2880, - "truePositive": 6, - "wilsonPrecisionLower95": 0.6097 - }, - "complaint": { - "f1": 0.9773, - "falseNegative": 16, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 344, - "recall": 0.9556, - "total": 2970, - "trueNegative": 2610, - "truePositive": 344, - "wilsonPrecisionLower95": 0.989 - }, - "confirmationDecision": { - "f1": 0.8163, - "falseNegative": 40, - "falsePositive": 23, - "precision": 0.8589, - "predictedPositive": 163, - "recall": 0.7778, - "total": 2970, - "trueNegative": 2767, - "truePositive": 140, - "wilsonPrecisionLower95": 0.7972 - }, - "followUpReminder": { - "f1": 0.8705, - "falseNegative": 81, - "falsePositive": 2, - "precision": 0.9929, - "predictedPositive": 281, - "recall": 0.775, - "total": 2970, - "trueNegative": 2608, - "truePositive": 279, - "wilsonPrecisionLower95": 0.9744 - }, - "invitation": { - "f1": 0.8545, - "falseNegative": 42, - "falsePositive": 5, - "precision": 0.965, - "predictedPositive": 143, - "recall": 0.7667, - "total": 2970, - "trueNegative": 2785, - "truePositive": 138, - "wilsonPrecisionLower95": 0.9208 - }, - "question": { - "f1": 0.9092, - "falseNegative": 104, - "falsePositive": 1, - "precision": 0.9981, - "predictedPositive": 527, - "recall": 0.8349, - "total": 2970, - "trueNegative": 2339, - "truePositive": 526, - "wilsonPrecisionLower95": 0.9893 - }, - "replyableMessage": { - "f1": 0.9465, - "falseNegative": 156, - "falsePositive": 40, - "precision": 0.9775, - "predictedPositive": 1774, - "recall": 0.9175, - "total": 2970, - "trueNegative": 1040, - "truePositive": 1734, - "wilsonPrecisionLower95": 0.9694 - }, - "scheduleNegotiation": { - "f1": 0.9727, - "falseNegative": 1, - "falsePositive": 4, - "precision": 0.957, - "predictedPositive": 93, - "recall": 0.9889, - "total": 2970, - "trueNegative": 2876, - "truePositive": 89, - "wilsonPrecisionLower95": 0.8946 - }, - "task": { - "f1": 0.8073, - "falseNegative": 213, - "falsePositive": 29, - "precision": 0.9459, - "predictedPositive": 536, - "recall": 0.7042, - "total": 2970, - "trueNegative": 2221, - "truePositive": 507, - "wilsonPrecisionLower95": 0.9234 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9318, - "falseNegative": 4, - "falsePositive": 2, - "precision": 0.9535, - "predictedPositive": 43, - "recall": 0.9111, - "total": 1485, - "trueNegative": 1438, - "truePositive": 41, - "wilsonPrecisionLower95": 0.8454 - }, - "threshold": 0.915 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7568, - "falseNegative": 17, - "falsePositive": 1, - "precision": 0.9655, - "predictedPositive": 29, - "recall": 0.6222, - "total": 1485, - "trueNegative": 1439, - "truePositive": 28, - "wilsonPrecisionLower95": 0.8282 - }, - "threshold": 0.9 - } - }, - "metrics": { - "f1": 0.8375, - "falseNegative": 23, - "falsePositive": 3, - "precision": 0.9571, - "predictedPositive": 70, - "recall": 0.7444, - "total": 2970, - "trueNegative": 2877, - "truePositive": 67, - "wilsonPrecisionLower95": 0.8814 - }, - "threshold": 0.915 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 180, - "recall": 1.0, - "total": 1485, - "trueNegative": 1305, - "truePositive": 180, - "wilsonPrecisionLower95": 0.9791 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9526, - "falseNegative": 9, - "falsePositive": 8, - "precision": 0.9553, - "predictedPositive": 179, - "recall": 0.95, - "total": 1485, - "trueNegative": 1297, - "truePositive": 171, - "wilsonPrecisionLower95": 0.9143 - }, - "threshold": 0.54 - } - }, - "metrics": { - "f1": 0.9764, - "falseNegative": 9, - "falsePositive": 8, - "precision": 0.9777, - "predictedPositive": 359, - "recall": 0.975, - "total": 2970, - "trueNegative": 2602, - "truePositive": 351, - "wilsonPrecisionLower95": 0.9567 - }, - "threshold": 0.54 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8848, - "falseNegative": 17, - "falsePositive": 2, - "precision": 0.9733, - "predictedPositive": 75, - "recall": 0.8111, - "total": 1485, - "trueNegative": 1393, - "truePositive": 73, - "wilsonPrecisionLower95": 0.9079 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7947, - "falseNegative": 30, - "falsePositive": 1, - "precision": 0.9836, - "predictedPositive": 61, - "recall": 0.6667, - "total": 1485, - "trueNegative": 1394, - "truePositive": 60, - "wilsonPrecisionLower95": 0.9128 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.8418, - "falseNegative": 47, - "falsePositive": 3, - "precision": 0.9779, - "predictedPositive": 136, - "recall": 0.7389, - "total": 2970, - "trueNegative": 2787, - "truePositive": 133, - "wilsonPrecisionLower95": 0.9372 - }, - "threshold": 0.5 - }, - "followUpReminder": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8554, - "falseNegative": 41, - "falsePositive": 6, - "precision": 0.9586, - "predictedPositive": 145, - "recall": 0.7722, - "total": 1485, - "trueNegative": 1299, - "truePositive": 139, - "wilsonPrecisionLower95": 0.9127 - }, - "threshold": 0.83 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8354, - "falseNegative": 48, - "falsePositive": 4, - "precision": 0.9706, - "predictedPositive": 136, - "recall": 0.7333, - "total": 1485, - "trueNegative": 1301, - "truePositive": 132, - "wilsonPrecisionLower95": 0.9268 - }, - "threshold": 0.505 - } - }, - "metrics": { - "f1": 0.7994, - "falseNegative": 113, - "falsePositive": 11, - "precision": 0.9574, - "predictedPositive": 258, - "recall": 0.6861, - "total": 2970, - "trueNegative": 2599, - "truePositive": 247, - "wilsonPrecisionLower95": 0.9253 - }, - "threshold": 0.815 - }, - "invitation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8302, - "falseNegative": 24, - "falsePositive": 3, - "precision": 0.9565, - "predictedPositive": 69, - "recall": 0.7333, - "total": 1485, - "trueNegative": 1392, - "truePositive": 66, - "wilsonPrecisionLower95": 0.8798 - }, - "threshold": 0.935 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6765, - "falseNegative": 44, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 46, - "recall": 0.5111, - "total": 1485, - "trueNegative": 1395, - "truePositive": 46, - "wilsonPrecisionLower95": 0.9229 - }, - "threshold": 0.505 - } - }, - "metrics": { - "f1": 0.5476, - "falseNegative": 111, - "falsePositive": 3, - "precision": 0.9583, - "predictedPositive": 72, - "recall": 0.3833, - "total": 2970, - "trueNegative": 2787, - "truePositive": 69, - "wilsonPrecisionLower95": 0.8845 - }, - "threshold": 0.935 - }, - "question": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8445, - "falseNegative": 76, - "falsePositive": 12, - "precision": 0.9522, - "predictedPositive": 251, - "recall": 0.7587, - "total": 1485, - "trueNegative": 1158, - "truePositive": 239, - "wilsonPrecisionLower95": 0.9183 - }, - "threshold": 0.725 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8536, - "falseNegative": 76, - "falsePositive": 6, - "precision": 0.9755, - "predictedPositive": 245, - "recall": 0.7587, - "total": 1485, - "trueNegative": 1164, - "truePositive": 239, - "wilsonPrecisionLower95": 0.9476 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.8729, - "falseNegative": 122, - "falsePositive": 26, - "precision": 0.9513, - "predictedPositive": 534, - "recall": 0.8063, - "total": 2970, - "trueNegative": 2314, - "truePositive": 508, - "wilsonPrecisionLower95": 0.9296 - }, - "threshold": 0.515 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.948, - "falseNegative": 52, - "falsePositive": 46, - "precision": 0.951, - "predictedPositive": 939, - "recall": 0.945, - "total": 1485, - "trueNegative": 494, - "truePositive": 893, - "wilsonPrecisionLower95": 0.9353 - }, - "threshold": 0.7 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9424, - "falseNegative": 62, - "falsePositive": 46, - "precision": 0.9505, - "predictedPositive": 929, - "recall": 0.9344, - "total": 1485, - "trueNegative": 494, - "truePositive": 883, - "wilsonPrecisionLower95": 0.9346 - }, - "threshold": 0.695 - } - }, - "metrics": { - "f1": 0.9454, - "falseNegative": 115, - "falsePositive": 90, - "precision": 0.9517, - "predictedPositive": 1865, - "recall": 0.9392, - "total": 2970, - "trueNegative": 990, - "truePositive": 1775, - "wilsonPrecisionLower95": 0.9411 - }, - "threshold": 0.7 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 45, - "recall": 1.0, - "total": 1485, - "trueNegative": 1440, - "truePositive": 45, - "wilsonPrecisionLower95": 0.9213 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.967, - "falseNegative": 1, - "falsePositive": 2, - "precision": 0.9565, - "predictedPositive": 46, - "recall": 0.9778, - "total": 1485, - "trueNegative": 1438, - "truePositive": 44, - "wilsonPrecisionLower95": 0.8547 - }, - "threshold": 0.93 - } - }, - "metrics": { - "f1": 0.978, - "falseNegative": 1, - "falsePositive": 3, - "precision": 0.9674, - "predictedPositive": 92, - "recall": 0.9889, - "total": 2970, - "trueNegative": 2877, - "truePositive": 89, - "wilsonPrecisionLower95": 0.9085 - }, - "threshold": 0.91 - }, - "task": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8403, - "falseNegative": 89, - "falsePositive": 14, - "precision": 0.9509, - "predictedPositive": 285, - "recall": 0.7528, - "total": 1485, - "trueNegative": 1111, - "truePositive": 271, - "wilsonPrecisionLower95": 0.9192 - }, - "threshold": 0.835 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8278, - "falseNegative": 98, - "falsePositive": 11, - "precision": 0.9597, - "predictedPositive": 273, - "recall": 0.7278, - "total": 1485, - "trueNegative": 1114, - "truePositive": 262, - "wilsonPrecisionLower95": 0.9293 - }, - "threshold": 0.745 - } - }, - "metrics": { - "f1": 0.8637, - "falseNegative": 153, - "falsePositive": 26, - "precision": 0.9562, - "predictedPositive": 593, - "recall": 0.7875, - "total": 2970, - "trueNegative": 2224, - "truePositive": 567, - "wilsonPrecisionLower95": 0.9365 - }, - "threshold": 0.755 - } - }, - "trainingRecords": 15345, - "validation": { - "intentsMeeting95Precision": 9, - "macroF1": 0.822, - "macroPrecision": 0.9713, - "macroRecall": 0.7487, - "minimumPrecision": 0.9507, - "perIntent": { - "blessing": { - "f1": 0.3636, - "falseNegative": 70, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 20, - "recall": 0.2222, - "total": 2970, - "trueNegative": 2880, - "truePositive": 20, - "wilsonPrecisionLower95": 0.8389 - }, - "complaint": { - "f1": 0.9764, - "falseNegative": 9, - "falsePositive": 8, - "precision": 0.9777, - "predictedPositive": 359, - "recall": 0.975, - "total": 2970, - "trueNegative": 2602, - "truePositive": 351, - "wilsonPrecisionLower95": 0.9567 - }, - "confirmationDecision": { - "f1": 0.8418, - "falseNegative": 47, - "falsePositive": 3, - "precision": 0.9779, - "predictedPositive": 136, - "recall": 0.7389, - "total": 2970, - "trueNegative": 2787, - "truePositive": 133, - "wilsonPrecisionLower95": 0.9372 - }, - "followUpReminder": { - "f1": 0.8456, - "falseNegative": 89, - "falsePositive": 10, - "precision": 0.9644, - "predictedPositive": 281, - "recall": 0.7528, - "total": 2970, - "trueNegative": 2600, - "truePositive": 271, - "wilsonPrecisionLower95": 0.9357 - }, - "invitation": { - "f1": 0.7593, - "falseNegative": 68, - "falsePositive": 3, - "precision": 0.9739, - "predictedPositive": 115, - "recall": 0.6222, - "total": 2970, - "trueNegative": 2787, - "truePositive": 112, - "wilsonPrecisionLower95": 0.9261 - }, - "question": { - "f1": 0.849, - "falseNegative": 152, - "falsePositive": 18, - "precision": 0.9637, - "predictedPositive": 496, - "recall": 0.7587, - "total": 2970, - "trueNegative": 2322, - "truePositive": 478, - "wilsonPrecisionLower95": 0.9434 - }, - "replyableMessage": { - "f1": 0.9452, - "falseNegative": 114, - "falsePositive": 92, - "precision": 0.9507, - "predictedPositive": 1868, - "recall": 0.9397, - "total": 2970, - "trueNegative": 988, - "truePositive": 1776, - "wilsonPrecisionLower95": 0.94 - }, - "scheduleNegotiation": { - "f1": 0.9834, - "falseNegative": 1, - "falsePositive": 2, - "precision": 0.978, - "predictedPositive": 91, - "recall": 0.9889, - "total": 2970, - "trueNegative": 2878, - "truePositive": 89, - "wilsonPrecisionLower95": 0.9234 - }, - "task": { - "f1": 0.8341, - "falseNegative": 187, - "falsePositive": 25, - "precision": 0.9552, - "predictedPositive": 558, - "recall": 0.7403, - "total": 2970, - "trueNegative": 2225, - "truePositive": 533, - "wilsonPrecisionLower95": 0.9347 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 1.5e-05, - "augmentation": "none", - "char_max": 5, - "char_min": 3, - "external_weight": 0.5900000000000001, - "hard_example_weight": 1.0, - "l1_ratio": 0.0, - "max_char_features": 50000, - "max_word_features": 25000, - "min_df": 2, - "pseudo_weight": 0.18, - "round": 2, - "word_max": 2 - }, - "featureCount": 61451, - "pseudoLabelCounts": {}, - "round": 2, - "selectionObjective": 0.8228, - "syntheticRegression": { - "intentsMeeting95Precision": 7, - "macroF1": 0.751, - "macroPrecision": 0.9667, - "macroRecall": 0.6602, - "minimumPrecision": 0.8606, - "perIntent": { - "blessing": { - "f1": 0.2178, - "falseNegative": 79, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 11, - "recall": 0.1222, - "total": 2970, - "trueNegative": 2880, - "truePositive": 11, - "wilsonPrecisionLower95": 0.7412 - }, - "complaint": { - "f1": 0.9595, - "falseNegative": 28, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 332, - "recall": 0.9222, - "total": 2970, - "trueNegative": 2610, - "truePositive": 332, - "wilsonPrecisionLower95": 0.9886 - }, - "confirmationDecision": { - "f1": 0.8232, - "falseNegative": 38, - "falsePositive": 23, - "precision": 0.8606, - "predictedPositive": 165, - "recall": 0.7889, - "total": 2970, - "trueNegative": 2767, - "truePositive": 142, - "wilsonPrecisionLower95": 0.7995 - }, - "followUpReminder": { - "f1": 0.7565, - "falseNegative": 141, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 219, - "recall": 0.6083, - "total": 2970, - "trueNegative": 2610, - "truePositive": 219, - "wilsonPrecisionLower95": 0.9828 - }, - "invitation": { - "f1": 0.6394, - "falseNegative": 94, - "falsePositive": 3, - "precision": 0.9663, - "predictedPositive": 89, - "recall": 0.4778, - "total": 2970, - "trueNegative": 2787, - "truePositive": 86, - "wilsonPrecisionLower95": 0.9055 - }, - "question": { - "f1": 0.86, - "falseNegative": 151, - "falsePositive": 5, - "precision": 0.9897, - "predictedPositive": 484, - "recall": 0.7603, - "total": 2970, - "trueNegative": 2335, - "truePositive": 479, - "wilsonPrecisionLower95": 0.976 - }, - "replyableMessage": { - "f1": 0.9038, - "falseNegative": 293, - "falsePositive": 47, - "precision": 0.9714, - "predictedPositive": 1644, - "recall": 0.845, - "total": 2970, - "trueNegative": 1033, - "truePositive": 1597, - "wilsonPrecisionLower95": 0.9622 - }, - "scheduleNegotiation": { - "f1": 0.9425, - "falseNegative": 8, - "falsePositive": 2, - "precision": 0.9762, - "predictedPositive": 84, - "recall": 0.9111, - "total": 2970, - "trueNegative": 2878, - "truePositive": 82, - "wilsonPrecisionLower95": 0.9173 - }, - "task": { - "f1": 0.6564, - "falseNegative": 356, - "falsePositive": 25, - "precision": 0.9357, - "predictedPositive": 389, - "recall": 0.5056, - "total": 2970, - "trueNegative": 2225, - "truePositive": 364, - "wilsonPrecisionLower95": 0.9069 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9048, - "falseNegative": 7, - "falsePositive": 1, - "precision": 0.9744, - "predictedPositive": 39, - "recall": 0.8444, - "total": 1485, - "trueNegative": 1439, - "truePositive": 38, - "wilsonPrecisionLower95": 0.8682 - }, - "threshold": 0.59 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 180, - "recall": 1.0, - "total": 1485, - "trueNegative": 1305, - "truePositive": 180, - "wilsonPrecisionLower95": 0.9791 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9226, - "falseNegative": 19, - "falsePositive": 8, - "precision": 0.9527, - "predictedPositive": 169, - "recall": 0.8944, - "total": 1485, - "trueNegative": 1297, - "truePositive": 161, - "wilsonPrecisionLower95": 0.9094 - }, - "threshold": 0.555 - } - }, - "metrics": { - "f1": 0.9649, - "falseNegative": 16, - "falsePositive": 9, - "precision": 0.9745, - "predictedPositive": 353, - "recall": 0.9556, - "total": 2970, - "trueNegative": 2601, - "truePositive": 344, - "wilsonPrecisionLower95": 0.9523 - }, - "threshold": 0.5 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.954, - "falseNegative": 7, - "falsePositive": 1, - "precision": 0.9881, - "predictedPositive": 84, - "recall": 0.9222, - "total": 1485, - "trueNegative": 1394, - "truePositive": 83, - "wilsonPrecisionLower95": 0.9356 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7183, - "falseNegative": 39, - "falsePositive": 1, - "precision": 0.9808, - "predictedPositive": 52, - "recall": 0.5667, - "total": 1485, - "trueNegative": 1394, - "truePositive": 51, - "wilsonPrecisionLower95": 0.8988 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.8481, - "falseNegative": 46, - "falsePositive": 2, - "precision": 0.9853, - "predictedPositive": 136, - "recall": 0.7444, - "total": 2970, - "trueNegative": 2788, - "truePositive": 134, - "wilsonPrecisionLower95": 0.948 - }, - "threshold": 0.5 - }, - "followUpReminder": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8328, - "falseNegative": 48, - "falsePositive": 5, - "precision": 0.9635, - "predictedPositive": 137, - "recall": 0.7333, - "total": 1485, - "trueNegative": 1300, - "truePositive": 132, - "wilsonPrecisionLower95": 0.9174 - }, - "threshold": 0.91 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6879, - "falseNegative": 83, - "falsePositive": 5, - "precision": 0.951, - "predictedPositive": 102, - "recall": 0.5389, - "total": 1485, - "trueNegative": 1300, - "truePositive": 97, - "wilsonPrecisionLower95": 0.8903 - }, - "threshold": 0.765 - } - }, - "metrics": { - "f1": 0.6243, - "falseNegative": 193, - "falsePositive": 8, - "precision": 0.9543, - "predictedPositive": 175, - "recall": 0.4639, - "total": 2970, - "trueNegative": 2602, - "truePositive": 167, - "wilsonPrecisionLower95": 0.9124 - }, - "threshold": 0.905 - }, - "invitation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9059, - "falseNegative": 13, - "falsePositive": 3, - "precision": 0.9625, - "predictedPositive": 80, - "recall": 0.8556, - "total": 1485, - "trueNegative": 1392, - "truePositive": 77, - "wilsonPrecisionLower95": 0.8955 - }, - "threshold": 0.86 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.6113, - "falseNegative": 99, - "falsePositive": 4, - "precision": 0.9529, - "predictedPositive": 85, - "recall": 0.45, - "total": 2970, - "trueNegative": 2786, - "truePositive": 81, - "wilsonPrecisionLower95": 0.8852 - }, - "threshold": 0.855 - }, - "question": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8505, - "falseNegative": 76, - "falsePositive": 8, - "precision": 0.9676, - "predictedPositive": 247, - "recall": 0.7587, - "total": 1485, - "trueNegative": 1162, - "truePositive": 239, - "wilsonPrecisionLower95": 0.9374 - }, - "threshold": 0.705 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7308, - "falseNegative": 129, - "falsePositive": 8, - "precision": 0.9588, - "predictedPositive": 194, - "recall": 0.5905, - "total": 1485, - "trueNegative": 1162, - "truePositive": 186, - "wilsonPrecisionLower95": 0.9208 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.7813, - "falseNegative": 212, - "falsePositive": 22, - "precision": 0.95, - "predictedPositive": 440, - "recall": 0.6635, - "total": 2970, - "trueNegative": 2318, - "truePositive": 418, - "wilsonPrecisionLower95": 0.9255 - }, - "threshold": 0.57 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9629, - "falseNegative": 23, - "falsePositive": 48, - "precision": 0.9505, - "predictedPositive": 970, - "recall": 0.9757, - "total": 1485, - "trueNegative": 492, - "truePositive": 922, - "wilsonPrecisionLower95": 0.935 - }, - "threshold": 0.62 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8526, - "falseNegative": 216, - "falsePositive": 36, - "precision": 0.9529, - "predictedPositive": 765, - "recall": 0.7714, - "total": 1485, - "trueNegative": 504, - "truePositive": 729, - "wilsonPrecisionLower95": 0.9355 - }, - "threshold": 0.84 - } - }, - "metrics": { - "f1": 0.9224, - "falseNegative": 197, - "falsePositive": 88, - "precision": 0.9506, - "predictedPositive": 1781, - "recall": 0.8958, - "total": 2970, - "trueNegative": 992, - "truePositive": 1693, - "wilsonPrecisionLower95": 0.9395 - }, - "threshold": 0.775 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9783, - "falseNegative": 0, - "falsePositive": 2, - "precision": 0.9574, - "predictedPositive": 47, - "recall": 1.0, - "total": 1485, - "trueNegative": 1438, - "truePositive": 45, - "wilsonPrecisionLower95": 0.8575 - }, - "threshold": 0.765 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9778, - "falseNegative": 1, - "falsePositive": 1, - "precision": 0.9778, - "predictedPositive": 45, - "recall": 0.9778, - "total": 1485, - "trueNegative": 1439, - "truePositive": 44, - "wilsonPrecisionLower95": 0.8843 - }, - "threshold": 0.855 - } - }, - "metrics": { - "f1": 0.978, - "falseNegative": 1, - "falsePositive": 3, - "precision": 0.9674, - "predictedPositive": 92, - "recall": 0.9889, - "total": 2970, - "trueNegative": 2877, - "truePositive": 89, - "wilsonPrecisionLower95": 0.9085 - }, - "threshold": 0.855 - }, - "task": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8541, - "falseNegative": 82, - "falsePositive": 13, - "precision": 0.9553, - "predictedPositive": 291, - "recall": 0.7722, - "total": 1485, - "trueNegative": 1112, - "truePositive": 278, - "wilsonPrecisionLower95": 0.9251 - }, - "threshold": 0.84 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.5418, - "falseNegative": 224, - "falsePositive": 6, - "precision": 0.9577, - "predictedPositive": 142, - "recall": 0.3778, - "total": 1485, - "trueNegative": 1119, - "truePositive": 136, - "wilsonPrecisionLower95": 0.9109 - }, - "threshold": 0.91 - } - }, - "metrics": { - "f1": 0.7246, - "falseNegative": 299, - "falsePositive": 21, - "precision": 0.9525, - "predictedPositive": 442, - "recall": 0.5847, - "total": 2970, - "trueNegative": 2229, - "truePositive": 421, - "wilsonPrecisionLower95": 0.9285 - }, - "threshold": 0.88 - } - }, - "trainingRecords": 15345, - "validation": { - "intentsMeeting95Precision": 9, - "macroF1": 0.7578, - "macroPrecision": 0.9691, - "macroRecall": 0.668, - "minimumPrecision": 0.9516, - "perIntent": { - "blessing": { - "f1": 0.2524, - "falseNegative": 77, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 13, - "recall": 0.1444, - "total": 2970, - "trueNegative": 2880, - "truePositive": 13, - "wilsonPrecisionLower95": 0.7719 - }, - "complaint": { - "f1": 0.9619, - "falseNegative": 19, - "falsePositive": 8, - "precision": 0.9771, - "predictedPositive": 349, - "recall": 0.9472, - "total": 2970, - "trueNegative": 2602, - "truePositive": 341, - "wilsonPrecisionLower95": 0.9554 - }, - "confirmationDecision": { - "f1": 0.8481, - "falseNegative": 46, - "falsePositive": 2, - "precision": 0.9853, - "predictedPositive": 136, - "recall": 0.7444, - "total": 2970, - "trueNegative": 2788, - "truePositive": 134, - "wilsonPrecisionLower95": 0.948 - }, - "followUpReminder": { - "f1": 0.7646, - "falseNegative": 131, - "falsePositive": 10, - "precision": 0.9582, - "predictedPositive": 239, - "recall": 0.6361, - "total": 2970, - "trueNegative": 2600, - "truePositive": 229, - "wilsonPrecisionLower95": 0.9247 - }, - "invitation": { - "f1": 0.5923, - "falseNegative": 103, - "falsePositive": 3, - "precision": 0.9625, - "predictedPositive": 80, - "recall": 0.4278, - "total": 2970, - "trueNegative": 2787, - "truePositive": 77, - "wilsonPrecisionLower95": 0.8955 - }, - "question": { - "f1": 0.7937, - "falseNegative": 205, - "falsePositive": 16, - "precision": 0.9637, - "predictedPositive": 441, - "recall": 0.6746, - "total": 2970, - "trueNegative": 2324, - "truePositive": 425, - "wilsonPrecisionLower95": 0.9419 - }, - "replyableMessage": { - "f1": 0.9109, - "falseNegative": 239, - "falsePositive": 84, - "precision": 0.9516, - "predictedPositive": 1735, - "recall": 0.8735, - "total": 2970, - "trueNegative": 996, - "truePositive": 1651, - "wilsonPrecisionLower95": 0.9404 - }, - "scheduleNegotiation": { - "f1": 0.978, - "falseNegative": 1, - "falsePositive": 3, - "precision": 0.9674, - "predictedPositive": 92, - "recall": 0.9889, - "total": 2970, - "trueNegative": 2877, - "truePositive": 89, - "wilsonPrecisionLower95": 0.9085 - }, - "task": { - "f1": 0.7181, - "falseNegative": 306, - "falsePositive": 19, - "precision": 0.9561, - "predictedPositive": 433, - "recall": 0.575, - "total": 2970, - "trueNegative": 2231, - "truePositive": 414, - "wilsonPrecisionLower95": 0.9325 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 1e-05, - "augmentation": "none", - "char_max": 6, - "char_min": 2, - "external_weight": 0.6100000000000001, - "hard_example_weight": 1.2, - "l1_ratio": 0.05, - "max_char_features": 55000, - "max_word_features": 25000, - "min_df": 2, - "pseudo_weight": 0.195, - "round": 3, - "word_max": 2 - }, - "featureCount": 66451, - "pseudoLabelCounts": {}, - "round": 3, - "selectionObjective": 0.8348, - "syntheticRegression": { - "intentsMeeting95Precision": 4, - "macroF1": 0.7343, - "macroPrecision": 0.9403, - "macroRecall": 0.6672, - "minimumPrecision": 0.809, - "perIntent": { - "blessing": { - "f1": 0.0645, - "falseNegative": 87, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 3, - "recall": 0.0333, - "total": 2970, - "trueNegative": 2880, - "truePositive": 3, - "wilsonPrecisionLower95": 0.4385 - }, - "complaint": { - "f1": 0.9816, - "falseNegative": 13, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 347, - "recall": 0.9639, - "total": 2970, - "trueNegative": 2610, - "truePositive": 347, - "wilsonPrecisionLower95": 0.9891 - }, - "confirmationDecision": { - "f1": 0.8045, - "falseNegative": 36, - "falsePositive": 34, - "precision": 0.809, - "predictedPositive": 178, - "recall": 0.8, - "total": 2970, - "trueNegative": 2756, - "truePositive": 144, - "wilsonPrecisionLower95": 0.745 - }, - "followUpReminder": { - "f1": 0.7967, - "falseNegative": 121, - "falsePositive": 1, - "precision": 0.9958, - "predictedPositive": 240, - "recall": 0.6639, - "total": 2970, - "trueNegative": 2609, - "truePositive": 239, - "wilsonPrecisionLower95": 0.9768 - }, - "invitation": { - "f1": 0.5985, - "falseNegative": 101, - "falsePositive": 5, - "precision": 0.9405, - "predictedPositive": 84, - "recall": 0.4389, - "total": 2970, - "trueNegative": 2785, - "truePositive": 79, - "wilsonPrecisionLower95": 0.8681 - }, - "question": { - "f1": 0.8643, - "falseNegative": 149, - "falsePositive": 2, - "precision": 0.9959, - "predictedPositive": 483, - "recall": 0.7635, - "total": 2970, - "trueNegative": 2338, - "truePositive": 481, - "wilsonPrecisionLower95": 0.985 - }, - "replyableMessage": { - "f1": 0.931, - "falseNegative": 189, - "falsePositive": 63, - "precision": 0.9643, - "predictedPositive": 1764, - "recall": 0.9, - "total": 2970, - "trueNegative": 1017, - "truePositive": 1701, - "wilsonPrecisionLower95": 0.9546 - }, - "scheduleNegotiation": { - "f1": 0.8222, - "falseNegative": 16, - "falsePositive": 16, - "precision": 0.8222, - "predictedPositive": 90, - "recall": 0.8222, - "total": 2970, - "trueNegative": 2864, - "truePositive": 74, - "wilsonPrecisionLower95": 0.7306 - }, - "task": { - "f1": 0.7452, - "falseNegative": 274, - "falsePositive": 31, - "precision": 0.935, - "predictedPositive": 477, - "recall": 0.6194, - "total": 2970, - "trueNegative": 2219, - "truePositive": 446, - "wilsonPrecisionLower95": 0.9092 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 180, - "recall": 1.0, - "total": 1485, - "trueNegative": 1305, - "truePositive": 180, - "wilsonPrecisionLower95": 0.9791 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9371, - "falseNegative": 16, - "falsePositive": 6, - "precision": 0.9647, - "predictedPositive": 170, - "recall": 0.9111, - "total": 1485, - "trueNegative": 1299, - "truePositive": 164, - "wilsonPrecisionLower95": 0.9251 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.969, - "falseNegative": 16, - "falsePositive": 6, - "precision": 0.9829, - "predictedPositive": 350, - "recall": 0.9556, - "total": 2970, - "trueNegative": 2604, - "truePositive": 344, - "wilsonPrecisionLower95": 0.9631 - }, - "threshold": 0.5 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9727, - "falseNegative": 1, - "falsePositive": 4, - "precision": 0.957, - "predictedPositive": 93, - "recall": 0.9889, - "total": 1485, - "trueNegative": 1391, - "truePositive": 89, - "wilsonPrecisionLower95": 0.8946 - }, - "threshold": 0.585 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7815, - "falseNegative": 31, - "falsePositive": 2, - "precision": 0.9672, - "predictedPositive": 61, - "recall": 0.6556, - "total": 1485, - "trueNegative": 1393, - "truePositive": 59, - "wilsonPrecisionLower95": 0.8881 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.8701, - "falseNegative": 36, - "falsePositive": 7, - "precision": 0.9536, - "predictedPositive": 151, - "recall": 0.8, - "total": 2970, - "trueNegative": 2783, - "truePositive": 144, - "wilsonPrecisionLower95": 0.9074 - }, - "threshold": 0.555 - }, - "followUpReminder": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8454, - "falseNegative": 46, - "falsePositive": 3, - "precision": 0.9781, - "predictedPositive": 137, - "recall": 0.7444, - "total": 1485, - "trueNegative": 1302, - "truePositive": 134, - "wilsonPrecisionLower95": 0.9376 - }, - "threshold": 0.97 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7525, - "falseNegative": 69, - "falsePositive": 4, - "precision": 0.9652, - "predictedPositive": 115, - "recall": 0.6167, - "total": 1485, - "trueNegative": 1301, - "truePositive": 111, - "wilsonPrecisionLower95": 0.914 - }, - "threshold": 0.785 - } - }, - "metrics": { - "f1": 0.703, - "falseNegative": 160, - "falsePositive": 9, - "precision": 0.9569, - "predictedPositive": 209, - "recall": 0.5556, - "total": 2970, - "trueNegative": 2601, - "truePositive": 200, - "wilsonPrecisionLower95": 0.9202 - }, - "threshold": 0.955 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7974, - "falseNegative": 29, - "falsePositive": 2, - "precision": 0.9683, - "predictedPositive": 63, - "recall": 0.6778, - "total": 1485, - "trueNegative": 1393, - "truePositive": 61, - "wilsonPrecisionLower95": 0.8914 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8259, - "falseNegative": 85, - "falsePositive": 12, - "precision": 0.9504, - "predictedPositive": 242, - "recall": 0.7302, - "total": 1485, - "trueNegative": 1158, - "truePositive": 230, - "wilsonPrecisionLower95": 0.9153 - }, - "threshold": 0.865 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8194, - "falseNegative": 95, - "falsePositive": 2, - "precision": 0.991, - "predictedPositive": 222, - "recall": 0.6984, - "total": 1485, - "trueNegative": 1168, - "truePositive": 220, - "wilsonPrecisionLower95": 0.9678 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.7959, - "falseNegative": 199, - "falsePositive": 22, - "precision": 0.9514, - "predictedPositive": 453, - "recall": 0.6841, - "total": 2970, - "trueNegative": 2318, - "truePositive": 431, - "wilsonPrecisionLower95": 0.9276 - }, - "threshold": 0.74 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9496, - "falseNegative": 50, - "falsePositive": 45, - "precision": 0.9521, - "predictedPositive": 940, - "recall": 0.9471, - "total": 1485, - "trueNegative": 495, - "truePositive": 895, - "wilsonPrecisionLower95": 0.9365 - }, - "threshold": 0.705 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9209, - "falseNegative": 101, - "falsePositive": 44, - "precision": 0.9505, - "predictedPositive": 888, - "recall": 0.8931, - "total": 1485, - "trueNegative": 496, - "truePositive": 844, - "wilsonPrecisionLower95": 0.9341 - }, - "threshold": 0.78 - } - }, - "metrics": { - "f1": 0.9318, - "falseNegative": 163, - "falsePositive": 90, - "precision": 0.9505, - "predictedPositive": 1817, - "recall": 0.9138, - "total": 2970, - "trueNegative": 990, - "truePositive": 1727, - "wilsonPrecisionLower95": 0.9395 - }, - "threshold": 0.77 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 45, - "recall": 1.0, - "total": 1485, - "trueNegative": 1440, - "truePositive": 45, - "wilsonPrecisionLower95": 0.9213 - }, - "threshold": 0.745 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9783, - "falseNegative": 0, - "falsePositive": 2, - "precision": 0.9574, - "predictedPositive": 47, - "recall": 1.0, - "total": 1485, - "trueNegative": 1438, - "truePositive": 45, - "wilsonPrecisionLower95": 0.8575 - }, - "threshold": 0.93 - } - }, - "metrics": { - "f1": 0.9778, - "falseNegative": 2, - "falsePositive": 2, - "precision": 0.9778, - "predictedPositive": 90, - "recall": 0.9778, - "total": 2970, - "trueNegative": 2878, - "truePositive": 88, - "wilsonPrecisionLower95": 0.9226 - }, - "threshold": 0.93 - }, - "task": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8488, - "falseNegative": 85, - "falsePositive": 13, - "precision": 0.9549, - "predictedPositive": 288, - "recall": 0.7639, - "total": 1485, - "trueNegative": 1112, - "truePositive": 275, - "wilsonPrecisionLower95": 0.9243 - }, - "threshold": 0.865 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8006, - "falseNegative": 111, - "falsePositive": 13, - "precision": 0.9504, - "predictedPositive": 262, - "recall": 0.6917, - "total": 1485, - "trueNegative": 1112, - "truePositive": 249, - "wilsonPrecisionLower95": 0.917 - }, - "threshold": 0.8 - } - }, - "metrics": { - "f1": 0.8301, - "falseNegative": 190, - "falsePositive": 27, - "precision": 0.9515, - "predictedPositive": 557, - "recall": 0.7361, - "total": 2970, - "trueNegative": 2223, - "truePositive": 530, - "wilsonPrecisionLower95": 0.9304 - }, - "threshold": 0.82 - } - }, - "trainingRecords": 15345, - "validation": { - "intentsMeeting95Precision": 8, - "macroF1": 0.7723, - "macroPrecision": 0.9713, - "macroRecall": 0.7011, - "minimumPrecision": 0.9513, - "perIntent": { - "blessing": { - "f1": 0.1443, - "falseNegative": 83, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 7, - "recall": 0.0778, - "total": 2970, - "trueNegative": 2880, - "truePositive": 7, - "wilsonPrecisionLower95": 0.6457 - }, - "complaint": { - "f1": 0.969, - "falseNegative": 16, - "falsePositive": 6, - "precision": 0.9829, - "predictedPositive": 350, - "recall": 0.9556, - "total": 2970, - "trueNegative": 2604, - "truePositive": 344, - "wilsonPrecisionLower95": 0.9631 - }, - "confirmationDecision": { - "f1": 0.8862, - "falseNegative": 32, - "falsePositive": 6, - "precision": 0.961, - "predictedPositive": 154, - "recall": 0.8222, - "total": 2970, - "trueNegative": 2784, - "truePositive": 148, - "wilsonPrecisionLower95": 0.9176 - }, - "followUpReminder": { - "f1": 0.8007, - "falseNegative": 115, - "falsePositive": 7, - "precision": 0.9722, - "predictedPositive": 252, - "recall": 0.6806, - "total": 2970, - "trueNegative": 2603, - "truePositive": 245, - "wilsonPrecisionLower95": 0.9438 - }, - "invitation": { - "f1": 0.5781, - "falseNegative": 106, - "falsePositive": 2, - "precision": 0.9737, - "predictedPositive": 76, - "recall": 0.4111, - "total": 2970, - "trueNegative": 2788, - "truePositive": 74, - "wilsonPrecisionLower95": 0.909 - }, - "question": { - "f1": 0.8227, - "falseNegative": 180, - "falsePositive": 14, - "precision": 0.9698, - "predictedPositive": 464, - "recall": 0.7143, - "total": 2970, - "trueNegative": 2326, - "truePositive": 450, - "wilsonPrecisionLower95": 0.95 - }, - "replyableMessage": { - "f1": 0.9354, - "falseNegative": 151, - "falsePositive": 89, - "precision": 0.9513, - "predictedPositive": 1828, - "recall": 0.9201, - "total": 2970, - "trueNegative": 991, - "truePositive": 1739, - "wilsonPrecisionLower95": 0.9405 - }, - "scheduleNegotiation": { - "f1": 0.989, - "falseNegative": 0, - "falsePositive": 2, - "precision": 0.9783, - "predictedPositive": 92, - "recall": 1.0, - "total": 2970, - "trueNegative": 2878, - "truePositive": 90, - "wilsonPrecisionLower95": 0.9242 - }, - "task": { - "f1": 0.8252, - "falseNegative": 196, - "falsePositive": 26, - "precision": 0.9527, - "predictedPositive": 550, - "recall": 0.7278, - "total": 2970, - "trueNegative": 2224, - "truePositive": 524, - "wilsonPrecisionLower95": 0.9316 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 8e-06, - "augmentation": "none", - "char_max": 6, - "char_min": 3, - "external_weight": 0.63, - "hard_example_weight": 1.2, - "l1_ratio": 0.05, - "max_char_features": 60000, - "max_word_features": 30000, - "min_df": 2, - "pseudo_weight": 0.21, - "round": 4, - "word_max": 3 - }, - "featureCount": 79458, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 0 - }, - "complaint": { - "negative": 97, - "positive": 0 - }, - "confirmationDecision": { - "negative": 51, - "positive": 2 - }, - "followUpReminder": { - "negative": 250, - "positive": 0 - }, - "invitation": { - "negative": 250, - "positive": 0 - }, - "question": { - "negative": 36, - "positive": 1 - }, - "replyableMessage": { - "negative": 3, - "positive": 130 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 0 - }, - "task": { - "negative": 85, - "positive": 0 - } - }, - "round": 4, - "selectionObjective": 0.796, - "syntheticRegression": { - "intentsMeeting95Precision": 6, - "macroF1": 0.6978, - "macroPrecision": 0.9611, - "macroRecall": 0.5994, - "minimumPrecision": 0.8074, - "perIntent": { - "blessing": { - "f1": 0.0645, - "falseNegative": 87, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 3, - "recall": 0.0333, - "total": 2970, - "trueNegative": 2880, - "truePositive": 3, - "wilsonPrecisionLower95": 0.4385 - }, - "complaint": { - "f1": 0.967, - "falseNegative": 23, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 337, - "recall": 0.9361, - "total": 2970, - "trueNegative": 2610, - "truePositive": 337, - "wilsonPrecisionLower95": 0.9887 - }, - "confirmationDecision": { - "f1": 0.6921, - "falseNegative": 71, - "falsePositive": 26, - "precision": 0.8074, - "predictedPositive": 135, - "recall": 0.6056, - "total": 2970, - "trueNegative": 2764, - "truePositive": 109, - "wilsonPrecisionLower95": 0.7328 - }, - "followUpReminder": { - "f1": 0.7734, - "falseNegative": 133, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 227, - "recall": 0.6306, - "total": 2970, - "trueNegative": 2610, - "truePositive": 227, - "wilsonPrecisionLower95": 0.9834 - }, - "invitation": { - "f1": 0.5837, - "falseNegative": 105, - "falsePositive": 2, - "precision": 0.974, - "predictedPositive": 77, - "recall": 0.4167, - "total": 2970, - "trueNegative": 2788, - "truePositive": 75, - "wilsonPrecisionLower95": 0.9102 - }, - "question": { - "f1": 0.7613, - "falseNegative": 241, - "falsePositive": 3, - "precision": 0.9923, - "predictedPositive": 392, - "recall": 0.6175, - "total": 2970, - "trueNegative": 2337, - "truePositive": 389, - "wilsonPrecisionLower95": 0.9777 - }, - "replyableMessage": { - "f1": 0.8816, - "falseNegative": 353, - "falsePositive": 60, - "precision": 0.9624, - "predictedPositive": 1597, - "recall": 0.8132, - "total": 2970, - "trueNegative": 1020, - "truePositive": 1537, - "wilsonPrecisionLower95": 0.9519 - }, - "scheduleNegotiation": { - "f1": 0.9222, - "falseNegative": 13, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 77, - "recall": 0.8556, - "total": 2970, - "trueNegative": 2880, - "truePositive": 77, - "wilsonPrecisionLower95": 0.9525 - }, - "task": { - "f1": 0.6346, - "falseNegative": 370, - "falsePositive": 33, - "precision": 0.9138, - "predictedPositive": 383, - "recall": 0.4861, - "total": 2970, - "trueNegative": 2217, - "truePositive": 350, - "wilsonPrecisionLower95": 0.8815 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9972, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 179, - "recall": 0.9944, - "total": 1485, - "trueNegative": 1305, - "truePositive": 179, - "wilsonPrecisionLower95": 0.979 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9253, - "falseNegative": 19, - "falsePositive": 7, - "precision": 0.9583, - "predictedPositive": 168, - "recall": 0.8944, - "total": 1485, - "trueNegative": 1298, - "truePositive": 161, - "wilsonPrecisionLower95": 0.9165 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.9618, - "falseNegative": 20, - "falsePositive": 7, - "precision": 0.9798, - "predictedPositive": 347, - "recall": 0.9444, - "total": 2970, - "trueNegative": 2603, - "truePositive": 340, - "wilsonPrecisionLower95": 0.959 - }, - "threshold": 0.5 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8765, - "falseNegative": 19, - "falsePositive": 1, - "precision": 0.9861, - "predictedPositive": 72, - "recall": 0.7889, - "total": 1485, - "trueNegative": 1394, - "truePositive": 71, - "wilsonPrecisionLower95": 0.9254 - }, - "threshold": 0.945 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6569, - "falseNegative": 45, - "falsePositive": 2, - "precision": 0.9574, - "predictedPositive": 47, - "recall": 0.5, - "total": 1485, - "trueNegative": 1393, - "truePositive": 45, - "wilsonPrecisionLower95": 0.8575 - }, - "threshold": 0.64 - } - }, - "metrics": { - "f1": 0.7108, - "falseNegative": 78, - "falsePositive": 5, - "precision": 0.9533, - "predictedPositive": 107, - "recall": 0.5667, - "total": 2970, - "trueNegative": 2785, - "truePositive": 102, - "wilsonPrecisionLower95": 0.8952 - }, - "threshold": 0.91 - }, - "followUpReminder": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8, - "falseNegative": 56, - "falsePositive": 6, - "precision": 0.9538, - "predictedPositive": 130, - "recall": 0.6889, - "total": 1485, - "trueNegative": 1299, - "truePositive": 124, - "wilsonPrecisionLower95": 0.903 - }, - "threshold": 0.98 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6165, - "falseNegative": 98, - "falsePositive": 4, - "precision": 0.9535, - "predictedPositive": 86, - "recall": 0.4556, - "total": 1485, - "trueNegative": 1301, - "truePositive": 82, - "wilsonPrecisionLower95": 0.8864 - }, - "threshold": 0.825 - } - }, - "metrics": { - "f1": 0.5912, - "falseNegative": 206, - "falsePositive": 7, - "precision": 0.9565, - "predictedPositive": 161, - "recall": 0.4278, - "total": 2970, - "trueNegative": 2603, - "truePositive": 154, - "wilsonPrecisionLower95": 0.913 - }, - "threshold": 0.98 - }, - "invitation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8302, - "falseNegative": 24, - "falsePositive": 3, - "precision": 0.9565, - "predictedPositive": 69, - "recall": 0.7333, - "total": 1485, - "trueNegative": 1392, - "truePositive": 66, - "wilsonPrecisionLower95": 0.8798 - }, - "threshold": 0.98 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.536, - "falseNegative": 113, - "falsePositive": 3, - "precision": 0.9571, - "predictedPositive": 70, - "recall": 0.3722, - "total": 2970, - "trueNegative": 2787, - "truePositive": 67, - "wilsonPrecisionLower95": 0.8814 - }, - "threshold": 0.995 - }, - "question": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.7813, - "falseNegative": 106, - "falsePositive": 11, - "precision": 0.95, - "predictedPositive": 220, - "recall": 0.6635, - "total": 1485, - "trueNegative": 1159, - "truePositive": 209, - "wilsonPrecisionLower95": 0.9127 - }, - "threshold": 0.865 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6844, - "falseNegative": 148, - "falsePositive": 6, - "precision": 0.9653, - "predictedPositive": 173, - "recall": 0.5302, - "total": 1485, - "trueNegative": 1164, - "truePositive": 167, - "wilsonPrecisionLower95": 0.9264 - }, - "threshold": 0.63 - } - }, - "metrics": { - "f1": 0.75, - "falseNegative": 240, - "falsePositive": 20, - "precision": 0.9512, - "predictedPositive": 410, - "recall": 0.619, - "total": 2970, - "trueNegative": 2320, - "truePositive": 390, - "wilsonPrecisionLower95": 0.9259 - }, - "threshold": 0.74 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9441, - "falseNegative": 59, - "falsePositive": 46, - "precision": 0.9506, - "predictedPositive": 932, - "recall": 0.9376, - "total": 1485, - "trueNegative": 494, - "truePositive": 886, - "wilsonPrecisionLower95": 0.9348 - }, - "threshold": 0.76 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8342, - "falseNegative": 243, - "falsePositive": 36, - "precision": 0.9512, - "predictedPositive": 738, - "recall": 0.7429, - "total": 1485, - "trueNegative": 504, - "truePositive": 702, - "wilsonPrecisionLower95": 0.9332 - }, - "threshold": 0.885 - } - }, - "metrics": { - "f1": 0.9056, - "falseNegative": 255, - "falsePositive": 86, - "precision": 0.95, - "predictedPositive": 1721, - "recall": 0.8651, - "total": 2970, - "trueNegative": 994, - "truePositive": 1635, - "wilsonPrecisionLower95": 0.9387 - }, - "threshold": 0.855 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 45, - "recall": 1.0, - "total": 1485, - "trueNegative": 1440, - "truePositive": 45, - "wilsonPrecisionLower95": 0.9213 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9318, - "falseNegative": 4, - "falsePositive": 2, - "precision": 0.9535, - "predictedPositive": 43, - "recall": 0.9111, - "total": 1485, - "trueNegative": 1438, - "truePositive": 41, - "wilsonPrecisionLower95": 0.8454 - }, - "threshold": 0.945 - } - }, - "metrics": { - "f1": 0.9727, - "falseNegative": 1, - "falsePositive": 4, - "precision": 0.957, - "predictedPositive": 93, - "recall": 0.9889, - "total": 2970, - "trueNegative": 2876, - "truePositive": 89, - "wilsonPrecisionLower95": 0.8946 - }, - "threshold": 0.62 - }, - "task": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.851, - "falseNegative": 83, - "falsePositive": 14, - "precision": 0.9519, - "predictedPositive": 291, - "recall": 0.7694, - "total": 1485, - "trueNegative": 1111, - "truePositive": 277, - "wilsonPrecisionLower95": 0.9209 - }, - "threshold": 0.87 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.4886, - "falseNegative": 242, - "falsePositive": 5, - "precision": 0.9593, - "predictedPositive": 123, - "recall": 0.3278, - "total": 1485, - "trueNegative": 1120, - "truePositive": 118, - "wilsonPrecisionLower95": 0.9084 - }, - "threshold": 0.935 - } - }, - "metrics": { - "f1": 0.6978, - "falseNegative": 324, - "falsePositive": 19, - "precision": 0.9542, - "predictedPositive": 415, - "recall": 0.55, - "total": 2970, - "trueNegative": 2231, - "truePositive": 396, - "wilsonPrecisionLower95": 0.9296 - }, - "threshold": 0.915 - } - }, - "trainingRecords": 15345, - "validation": { - "intentsMeeting95Precision": 9, - "macroF1": 0.7192, - "macroPrecision": 0.9671, - "macroRecall": 0.62, - "minimumPrecision": 0.9509, - "perIntent": { - "blessing": { - "f1": 0.2, - "falseNegative": 80, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 10, - "recall": 0.1111, - "total": 2970, - "trueNegative": 2880, - "truePositive": 10, - "wilsonPrecisionLower95": 0.7225 - }, - "complaint": { - "f1": 0.9618, - "falseNegative": 20, - "falsePositive": 7, - "precision": 0.9798, - "predictedPositive": 347, - "recall": 0.9444, - "total": 2970, - "trueNegative": 2603, - "truePositive": 340, - "wilsonPrecisionLower95": 0.959 - }, - "confirmationDecision": { - "f1": 0.7759, - "falseNegative": 64, - "falsePositive": 3, - "precision": 0.9748, - "predictedPositive": 119, - "recall": 0.6444, - "total": 2970, - "trueNegative": 2787, - "truePositive": 116, - "wilsonPrecisionLower95": 0.9285 - }, - "followUpReminder": { - "f1": 0.7153, - "falseNegative": 154, - "falsePositive": 10, - "precision": 0.9537, - "predictedPositive": 216, - "recall": 0.5722, - "total": 2970, - "trueNegative": 2600, - "truePositive": 206, - "wilsonPrecisionLower95": 0.9169 - }, - "invitation": { - "f1": 0.5301, - "falseNegative": 114, - "falsePositive": 3, - "precision": 0.9565, - "predictedPositive": 69, - "recall": 0.3667, - "total": 2970, - "trueNegative": 2787, - "truePositive": 66, - "wilsonPrecisionLower95": 0.8798 - }, - "question": { - "f1": 0.7351, - "falseNegative": 254, - "falsePositive": 17, - "precision": 0.9567, - "predictedPositive": 393, - "recall": 0.5968, - "total": 2970, - "trueNegative": 2323, - "truePositive": 376, - "wilsonPrecisionLower95": 0.9318 - }, - "replyableMessage": { - "f1": 0.8921, - "falseNegative": 302, - "falsePositive": 82, - "precision": 0.9509, - "predictedPositive": 1670, - "recall": 0.8402, - "total": 2970, - "trueNegative": 998, - "truePositive": 1588, - "wilsonPrecisionLower95": 0.9395 - }, - "scheduleNegotiation": { - "f1": 0.9663, - "falseNegative": 4, - "falsePositive": 2, - "precision": 0.9773, - "predictedPositive": 88, - "recall": 0.9556, - "total": 2970, - "trueNegative": 2878, - "truePositive": 86, - "wilsonPrecisionLower95": 0.9209 - }, - "task": { - "f1": 0.6966, - "falseNegative": 325, - "falsePositive": 19, - "precision": 0.9541, - "predictedPositive": 414, - "recall": 0.5486, - "total": 2970, - "trueNegative": 2231, - "truePositive": 395, - "wilsonPrecisionLower95": 0.9294 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 1.2e-05, - "augmentation": "none", - "char_max": 5, - "char_min": 2, - "external_weight": 0.65, - "hard_example_weight": 1.4, - "l1_ratio": 0.1, - "max_char_features": 60000, - "max_word_features": 32000, - "min_df": 1, - "pseudo_weight": 0.22499999999999998, - "round": 5, - "word_max": 3 - }, - "featureCount": 92000, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 0 - }, - "complaint": { - "negative": 88, - "positive": 0 - }, - "confirmationDecision": { - "negative": 96, - "positive": 3 - }, - "followUpReminder": { - "negative": 250, - "positive": 0 - }, - "invitation": { - "negative": 250, - "positive": 0 - }, - "question": { - "negative": 40, - "positive": 1 - }, - "replyableMessage": { - "negative": 3, - "positive": 150 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 0 - }, - "task": { - "negative": 97, - "positive": 0 - } - }, - "round": 5, - "selectionObjective": 0.7771, - "syntheticRegression": { - "intentsMeeting95Precision": 5, - "macroF1": 0.6709, - "macroPrecision": 0.9586, - "macroRecall": 0.5803, - "minimumPrecision": 0.8588, - "perIntent": { - "blessing": { - "f1": 0.022, - "falseNegative": 89, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 1, - "recall": 0.0111, - "total": 2970, - "trueNegative": 2880, - "truePositive": 1, - "wilsonPrecisionLower95": 0.2065 - }, - "complaint": { - "f1": 0.967, - "falseNegative": 23, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 337, - "recall": 0.9361, - "total": 2970, - "trueNegative": 2610, - "truePositive": 337, - "wilsonPrecisionLower95": 0.9887 - }, - "confirmationDecision": { - "f1": 0.8343, - "falseNegative": 34, - "falsePositive": 24, - "precision": 0.8588, - "predictedPositive": 170, - "recall": 0.8111, - "total": 2970, - "trueNegative": 2766, - "truePositive": 146, - "wilsonPrecisionLower95": 0.7985 - }, - "followUpReminder": { - "f1": 0.7173, - "falseNegative": 157, - "falsePositive": 3, - "precision": 0.9854, - "predictedPositive": 206, - "recall": 0.5639, - "total": 2970, - "trueNegative": 2607, - "truePositive": 203, - "wilsonPrecisionLower95": 0.9581 - }, - "invitation": { - "f1": 0.3612, - "falseNegative": 139, - "falsePositive": 6, - "precision": 0.8723, - "predictedPositive": 47, - "recall": 0.2278, - "total": 2970, - "trueNegative": 2784, - "truePositive": 41, - "wilsonPrecisionLower95": 0.7483 - }, - "question": { - "f1": 0.8336, - "falseNegative": 179, - "falsePositive": 1, - "precision": 0.9978, - "predictedPositive": 452, - "recall": 0.7159, - "total": 2970, - "trueNegative": 2339, - "truePositive": 451, - "wilsonPrecisionLower95": 0.9876 - }, - "replyableMessage": { - "f1": 0.912, - "falseNegative": 257, - "falsePositive": 58, - "precision": 0.9657, - "predictedPositive": 1691, - "recall": 0.864, - "total": 2970, - "trueNegative": 1022, - "truePositive": 1633, - "wilsonPrecisionLower95": 0.9559 - }, - "scheduleNegotiation": { - "f1": 0.6364, - "falseNegative": 48, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 42, - "recall": 0.4667, - "total": 2970, - "trueNegative": 2880, - "truePositive": 42, - "wilsonPrecisionLower95": 0.9162 - }, - "task": { - "f1": 0.7542, - "falseNegative": 269, - "falsePositive": 25, - "precision": 0.9475, - "predictedPositive": 476, - "recall": 0.6264, - "total": 2970, - "trueNegative": 2225, - "truePositive": 451, - "wilsonPrecisionLower95": 0.9236 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 180, - "recall": 1.0, - "total": 1485, - "trueNegative": 1305, - "truePositive": 180, - "wilsonPrecisionLower95": 0.9791 - }, - "threshold": 0.69 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9288, - "falseNegative": 17, - "falsePositive": 8, - "precision": 0.9532, - "predictedPositive": 171, - "recall": 0.9056, - "total": 1485, - "trueNegative": 1297, - "truePositive": 163, - "wilsonPrecisionLower95": 0.9104 - }, - "threshold": 0.51 - } - }, - "metrics": { - "f1": 0.9569, - "falseNegative": 16, - "falsePositive": 15, - "precision": 0.9582, - "predictedPositive": 359, - "recall": 0.9556, - "total": 2970, - "trueNegative": 2595, - "truePositive": 344, - "wilsonPrecisionLower95": 0.9322 - }, - "threshold": 0.5 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8589, - "falseNegative": 20, - "falsePositive": 3, - "precision": 0.9589, - "predictedPositive": 73, - "recall": 0.7778, - "total": 1485, - "trueNegative": 1392, - "truePositive": 70, - "wilsonPrecisionLower95": 0.886 - }, - "threshold": 0.57 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6763, - "falseNegative": 43, - "falsePositive": 2, - "precision": 0.9592, - "predictedPositive": 49, - "recall": 0.5222, - "total": 1485, - "trueNegative": 1393, - "truePositive": 47, - "wilsonPrecisionLower95": 0.8629 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.76, - "falseNegative": 66, - "falsePositive": 6, - "precision": 0.95, - "predictedPositive": 120, - "recall": 0.6333, - "total": 2970, - "trueNegative": 2784, - "truePositive": 114, - "wilsonPrecisionLower95": 0.8952 - }, - "threshold": 0.56 - }, - "followUpReminder": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8, - "falseNegative": 56, - "falsePositive": 6, - "precision": 0.9538, - "predictedPositive": 130, - "recall": 0.6889, - "total": 1485, - "trueNegative": 1299, - "truePositive": 124, - "wilsonPrecisionLower95": 0.903 - }, - "threshold": 0.96 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7708, - "falseNegative": 64, - "falsePositive": 5, - "precision": 0.9587, - "predictedPositive": 121, - "recall": 0.6444, - "total": 1485, - "trueNegative": 1300, - "truePositive": 116, - "wilsonPrecisionLower95": 0.9069 - }, - "threshold": 0.58 - } - }, - "metrics": { - "f1": 0.6892, - "falseNegative": 166, - "falsePositive": 9, - "precision": 0.9557, - "predictedPositive": 203, - "recall": 0.5389, - "total": 2970, - "trueNegative": 2601, - "truePositive": 194, - "wilsonPrecisionLower95": 0.9179 - }, - "threshold": 0.915 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.5203, - "falseNegative": 58, - "falsePositive": 1, - "precision": 0.9697, - "predictedPositive": 33, - "recall": 0.3556, - "total": 1485, - "trueNegative": 1394, - "truePositive": 32, - "wilsonPrecisionLower95": 0.8468 - }, - "threshold": 0.88 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8225, - "falseNegative": 88, - "falsePositive": 10, - "precision": 0.9578, - "predictedPositive": 237, - "recall": 0.7206, - "total": 1485, - "trueNegative": 1160, - "truePositive": 227, - "wilsonPrecisionLower95": 0.9241 - }, - "threshold": 0.84 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8388, - "falseNegative": 86, - "falsePositive": 2, - "precision": 0.9913, - "predictedPositive": 231, - "recall": 0.727, - "total": 1485, - "trueNegative": 1168, - "truePositive": 229, - "wilsonPrecisionLower95": 0.969 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.8591, - "falseNegative": 136, - "falsePositive": 26, - "precision": 0.95, - "predictedPositive": 520, - "recall": 0.7841, - "total": 2970, - "trueNegative": 2314, - "truePositive": 494, - "wilsonPrecisionLower95": 0.9277 - }, - "threshold": 0.505 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9541, - "falseNegative": 40, - "falsePositive": 47, - "precision": 0.9506, - "predictedPositive": 952, - "recall": 0.9577, - "total": 1485, - "trueNegative": 493, - "truePositive": 905, - "wilsonPrecisionLower95": 0.935 - }, - "threshold": 0.69 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8807, - "falseNegative": 170, - "falsePositive": 40, - "precision": 0.9509, - "predictedPositive": 815, - "recall": 0.8201, - "total": 1485, - "trueNegative": 500, - "truePositive": 775, - "wilsonPrecisionLower95": 0.9339 - }, - "threshold": 0.8 - } - }, - "metrics": { - "f1": 0.914, - "falseNegative": 227, - "falsePositive": 86, - "precision": 0.9508, - "predictedPositive": 1749, - "recall": 0.8799, - "total": 2970, - "trueNegative": 994, - "truePositive": 1663, - "wilsonPrecisionLower95": 0.9397 - }, - "threshold": 0.78 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9888, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 44, - "recall": 0.9778, - "total": 1485, - "trueNegative": 1440, - "truePositive": 44, - "wilsonPrecisionLower95": 0.9197 - }, - "threshold": 0.635 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.8129, - "falseNegative": 27, - "falsePositive": 2, - "precision": 0.9692, - "predictedPositive": 65, - "recall": 0.7, - "total": 2970, - "trueNegative": 2878, - "truePositive": 63, - "wilsonPrecisionLower95": 0.8946 - }, - "threshold": 0.995 - }, - "task": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8185, - "falseNegative": 103, - "falsePositive": 11, - "precision": 0.959, - "predictedPositive": 268, - "recall": 0.7139, - "total": 1485, - "trueNegative": 1114, - "truePositive": 257, - "wilsonPrecisionLower95": 0.928 - }, - "threshold": 0.83 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7702, - "falseNegative": 127, - "falsePositive": 12, - "precision": 0.951, - "predictedPositive": 245, - "recall": 0.6472, - "total": 1485, - "trueNegative": 1113, - "truePositive": 233, - "wilsonPrecisionLower95": 0.9164 - }, - "threshold": 0.765 - } - }, - "metrics": { - "f1": 0.7816, - "falseNegative": 244, - "falsePositive": 22, - "precision": 0.9558, - "predictedPositive": 498, - "recall": 0.6611, - "total": 2970, - "trueNegative": 2228, - "truePositive": 476, - "wilsonPrecisionLower95": 0.934 - }, - "threshold": 0.805 - } - }, - "trainingRecords": 15345, - "validation": { - "intentsMeeting95Precision": 8, - "macroF1": 0.6867, - "macroPrecision": 0.9721, - "macroRecall": 0.5915, - "minimumPrecision": 0.9508, - "perIntent": { - "blessing": { - "f1": 0.0851, - "falseNegative": 86, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 4, - "recall": 0.0444, - "total": 2970, - "trueNegative": 2880, - "truePositive": 4, - "wilsonPrecisionLower95": 0.5101 - }, - "complaint": { - "f1": 0.9648, - "falseNegative": 17, - "falsePositive": 8, - "precision": 0.9772, - "predictedPositive": 351, - "recall": 0.9528, - "total": 2970, - "trueNegative": 2602, - "truePositive": 343, - "wilsonPrecisionLower95": 0.9557 - }, - "confirmationDecision": { - "f1": 0.7748, - "falseNegative": 63, - "falsePositive": 5, - "precision": 0.959, - "predictedPositive": 122, - "recall": 0.65, - "total": 2970, - "trueNegative": 2785, - "truePositive": 117, - "wilsonPrecisionLower95": 0.9076 - }, - "followUpReminder": { - "f1": 0.7856, - "falseNegative": 120, - "falsePositive": 11, - "precision": 0.9562, - "predictedPositive": 251, - "recall": 0.6667, - "total": 2970, - "trueNegative": 2599, - "truePositive": 240, - "wilsonPrecisionLower95": 0.9232 - }, - "invitation": { - "f1": 0.3694, - "falseNegative": 139, - "falsePositive": 1, - "precision": 0.9762, - "predictedPositive": 42, - "recall": 0.2278, - "total": 2970, - "trueNegative": 2789, - "truePositive": 41, - "wilsonPrecisionLower95": 0.8768 - }, - "question": { - "f1": 0.8306, - "falseNegative": 174, - "falsePositive": 12, - "precision": 0.9744, - "predictedPositive": 468, - "recall": 0.7238, - "total": 2970, - "trueNegative": 2328, - "truePositive": 456, - "wilsonPrecisionLower95": 0.9557 - }, - "replyableMessage": { - "f1": 0.9188, - "falseNegative": 210, - "falsePositive": 87, - "precision": 0.9508, - "predictedPositive": 1767, - "recall": 0.8889, - "total": 2970, - "trueNegative": 993, - "truePositive": 1680, - "wilsonPrecisionLower95": 0.9397 - }, - "scheduleNegotiation": { - "f1": 0.6567, - "falseNegative": 46, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 44, - "recall": 0.4889, - "total": 2970, - "trueNegative": 2880, - "truePositive": 44, - "wilsonPrecisionLower95": 0.9197 - }, - "task": { - "f1": 0.7948, - "falseNegative": 230, - "falsePositive": 23, - "precision": 0.9552, - "predictedPositive": 513, - "recall": 0.6806, - "total": 2970, - "trueNegative": 2227, - "truePositive": 490, - "wilsonPrecisionLower95": 0.9336 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 8e-06, - "augmentation": "punctuation", - "char_max": 6, - "char_min": 2, - "external_weight": 0.67, - "hard_example_weight": 1.5, - "l1_ratio": 0.1, - "max_char_features": 65000, - "max_word_features": 28000, - "min_df": 2, - "pseudo_weight": 0.24, - "round": 6, - "word_max": 2 - }, - "featureCount": 78354, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 0 - }, - "complaint": { - "negative": 146, - "positive": 0 - }, - "confirmationDecision": { - "negative": 88, - "positive": 2 - }, - "followUpReminder": { - "negative": 250, - "positive": 0 - }, - "invitation": { - "negative": 250, - "positive": 0 - }, - "question": { - "negative": 36, - "positive": 5 - }, - "replyableMessage": { - "negative": 2, - "positive": 130 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 0 - }, - "task": { - "negative": 83, - "positive": 0 - } - }, - "round": 6, - "selectionObjective": 0.811, - "syntheticRegression": { - "intentsMeeting95Precision": 5, - "macroF1": 0.7064, - "macroPrecision": 0.9443, - "macroRecall": 0.623, - "minimumPrecision": 0.7975, - "perIntent": { - "blessing": { - "f1": 0.0435, - "falseNegative": 88, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 2, - "recall": 0.0222, - "total": 2970, - "trueNegative": 2880, - "truePositive": 2, - "wilsonPrecisionLower95": 0.3424 - }, - "complaint": { - "f1": 0.967, - "falseNegative": 23, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 337, - "recall": 0.9361, - "total": 2970, - "trueNegative": 2610, - "truePositive": 337, - "wilsonPrecisionLower95": 0.9887 - }, - "confirmationDecision": { - "f1": 0.7215, - "falseNegative": 66, - "falsePositive": 22, - "precision": 0.8382, - "predictedPositive": 136, - "recall": 0.6333, - "total": 2970, - "trueNegative": 2768, - "truePositive": 114, - "wilsonPrecisionLower95": 0.7672 - }, - "followUpReminder": { - "f1": 0.6847, - "falseNegative": 170, - "falsePositive": 5, - "precision": 0.9744, - "predictedPositive": 195, - "recall": 0.5278, - "total": 2970, - "trueNegative": 2605, - "truePositive": 190, - "wilsonPrecisionLower95": 0.9414 - }, - "invitation": { - "f1": 0.4865, - "falseNegative": 117, - "falsePositive": 16, - "precision": 0.7975, - "predictedPositive": 79, - "recall": 0.35, - "total": 2970, - "trueNegative": 2774, - "truePositive": 63, - "wilsonPrecisionLower95": 0.696 - }, - "question": { - "f1": 0.8556, - "falseNegative": 156, - "falsePositive": 4, - "precision": 0.9916, - "predictedPositive": 478, - "recall": 0.7524, - "total": 2970, - "trueNegative": 2336, - "truePositive": 474, - "wilsonPrecisionLower95": 0.9787 - }, - "replyableMessage": { - "f1": 0.8888, - "falseNegative": 335, - "falsePositive": 54, - "precision": 0.9664, - "predictedPositive": 1609, - "recall": 0.8228, - "total": 2970, - "trueNegative": 1026, - "truePositive": 1555, - "wilsonPrecisionLower95": 0.9565 - }, - "scheduleNegotiation": { - "f1": 0.9831, - "falseNegative": 3, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 87, - "recall": 0.9667, - "total": 2970, - "trueNegative": 2880, - "truePositive": 87, - "wilsonPrecisionLower95": 0.9577 - }, - "task": { - "f1": 0.7265, - "falseNegative": 291, - "falsePositive": 32, - "precision": 0.9306, - "predictedPositive": 461, - "recall": 0.5958, - "total": 2970, - "trueNegative": 2218, - "truePositive": 429, - "wilsonPrecisionLower95": 0.9037 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9834, - "falseNegative": 2, - "falsePositive": 4, - "precision": 0.978, - "predictedPositive": 182, - "recall": 0.9889, - "total": 1485, - "trueNegative": 1301, - "truePositive": 178, - "wilsonPrecisionLower95": 0.9449 - }, - "threshold": 0.555 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9494, - "falseNegative": 11, - "falsePositive": 7, - "precision": 0.9602, - "predictedPositive": 176, - "recall": 0.9389, - "total": 1485, - "trueNegative": 1298, - "truePositive": 169, - "wilsonPrecisionLower95": 0.9202 - }, - "threshold": 0.535 - } - }, - "metrics": { - "f1": 0.9652, - "falseNegative": 13, - "falsePositive": 12, - "precision": 0.9666, - "predictedPositive": 359, - "recall": 0.9639, - "total": 2970, - "trueNegative": 2598, - "truePositive": 347, - "wilsonPrecisionLower95": 0.9425 - }, - "threshold": 0.535 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8, - "falseNegative": 28, - "falsePositive": 3, - "precision": 0.9538, - "predictedPositive": 65, - "recall": 0.6889, - "total": 1485, - "trueNegative": 1392, - "truePositive": 62, - "wilsonPrecisionLower95": 0.8729 - }, - "threshold": 0.695 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.731, - "falseNegative": 37, - "falsePositive": 2, - "precision": 0.9636, - "predictedPositive": 55, - "recall": 0.5889, - "total": 1485, - "trueNegative": 1393, - "truePositive": 53, - "wilsonPrecisionLower95": 0.8768 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.7354, - "falseNegative": 73, - "falsePositive": 4, - "precision": 0.964, - "predictedPositive": 111, - "recall": 0.5944, - "total": 2970, - "trueNegative": 2786, - "truePositive": 107, - "wilsonPrecisionLower95": 0.911 - }, - "threshold": 0.695 - }, - "followUpReminder": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8632, - "falseNegative": 38, - "falsePositive": 7, - "precision": 0.953, - "predictedPositive": 149, - "recall": 0.7889, - "total": 1485, - "trueNegative": 1298, - "truePositive": 142, - "wilsonPrecisionLower95": 0.9062 - }, - "threshold": 0.9 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.4938, - "falseNegative": 120, - "falsePositive": 3, - "precision": 0.9524, - "predictedPositive": 63, - "recall": 0.3333, - "total": 1485, - "trueNegative": 1302, - "truePositive": 60, - "wilsonPrecisionLower95": 0.8691 - }, - "threshold": 0.845 - } - }, - "metrics": { - "f1": 0.7018, - "falseNegative": 160, - "falsePositive": 10, - "precision": 0.9524, - "predictedPositive": 210, - "recall": 0.5556, - "total": 2970, - "trueNegative": 2600, - "truePositive": 200, - "wilsonPrecisionLower95": 0.9146 - }, - "threshold": 0.885 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7222, - "falseNegative": 38, - "falsePositive": 2, - "precision": 0.963, - "predictedPositive": 54, - "recall": 0.5778, - "total": 1485, - "trueNegative": 1393, - "truePositive": 52, - "wilsonPrecisionLower95": 0.8746 - }, - "threshold": 0.665 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8167, - "falseNegative": 90, - "falsePositive": 11, - "precision": 0.9534, - "predictedPositive": 236, - "recall": 0.7143, - "total": 1485, - "trueNegative": 1159, - "truePositive": 225, - "wilsonPrecisionLower95": 0.9185 - }, - "threshold": 0.865 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8205, - "falseNegative": 91, - "falsePositive": 7, - "precision": 0.9697, - "predictedPositive": 231, - "recall": 0.7111, - "total": 1485, - "trueNegative": 1163, - "truePositive": 224, - "wilsonPrecisionLower95": 0.9388 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.804, - "falseNegative": 191, - "falsePositive": 23, - "precision": 0.9502, - "predictedPositive": 462, - "recall": 0.6968, - "total": 2970, - "trueNegative": 2317, - "truePositive": 439, - "wilsonPrecisionLower95": 0.9264 - }, - "threshold": 0.68 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9268, - "falseNegative": 90, - "falsePositive": 45, - "precision": 0.95, - "predictedPositive": 900, - "recall": 0.9048, - "total": 1485, - "trueNegative": 495, - "truePositive": 855, - "wilsonPrecisionLower95": 0.9338 - }, - "threshold": 0.86 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8789, - "falseNegative": 176, - "falsePositive": 36, - "precision": 0.9553, - "predictedPositive": 805, - "recall": 0.8138, - "total": 1485, - "trueNegative": 504, - "truePositive": 769, - "wilsonPrecisionLower95": 0.9387 - }, - "threshold": 0.85 - } - }, - "metrics": { - "f1": 0.9046, - "falseNegative": 259, - "falsePositive": 85, - "precision": 0.9505, - "predictedPositive": 1716, - "recall": 0.863, - "total": 2970, - "trueNegative": 995, - "truePositive": 1631, - "wilsonPrecisionLower95": 0.9392 - }, - "threshold": 0.85 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9773, - "falseNegative": 2, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 43, - "recall": 0.9556, - "total": 1485, - "trueNegative": 1440, - "truePositive": 43, - "wilsonPrecisionLower95": 0.918 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9888, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 44, - "recall": 0.9778, - "total": 1485, - "trueNegative": 1440, - "truePositive": 44, - "wilsonPrecisionLower95": 0.9197 - }, - "threshold": 0.97 - } - }, - "metrics": { - "f1": 0.9497, - "falseNegative": 5, - "falsePositive": 4, - "precision": 0.9551, - "predictedPositive": 89, - "recall": 0.9444, - "total": 2970, - "trueNegative": 2876, - "truePositive": 85, - "wilsonPrecisionLower95": 0.8901 - }, - "threshold": 0.91 - }, - "task": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.827, - "falseNegative": 97, - "falsePositive": 13, - "precision": 0.9529, - "predictedPositive": 276, - "recall": 0.7306, - "total": 1485, - "trueNegative": 1112, - "truePositive": 263, - "wilsonPrecisionLower95": 0.9211 - }, - "threshold": 0.89 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7902, - "falseNegative": 117, - "falsePositive": 12, - "precision": 0.9529, - "predictedPositive": 255, - "recall": 0.675, - "total": 1485, - "trueNegative": 1113, - "truePositive": 243, - "wilsonPrecisionLower95": 0.9196 - }, - "threshold": 0.785 - } - }, - "metrics": { - "f1": 0.7783, - "falseNegative": 246, - "falsePositive": 24, - "precision": 0.9518, - "predictedPositive": 498, - "recall": 0.6583, - "total": 2970, - "trueNegative": 2226, - "truePositive": 474, - "wilsonPrecisionLower95": 0.9293 - }, - "threshold": 0.85 - } - }, - "trainingRecords": 17767, - "validation": { - "intentsMeeting95Precision": 8, - "macroF1": 0.7395, - "macroPrecision": 0.9687, - "macroRecall": 0.6518, - "minimumPrecision": 0.9525, - "perIntent": { - "blessing": { - "f1": 0.1633, - "falseNegative": 82, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 8, - "recall": 0.0889, - "total": 2970, - "trueNegative": 2880, - "truePositive": 8, - "wilsonPrecisionLower95": 0.6756 - }, - "complaint": { - "f1": 0.9666, - "falseNegative": 13, - "falsePositive": 11, - "precision": 0.9693, - "predictedPositive": 358, - "recall": 0.9639, - "total": 2970, - "trueNegative": 2599, - "truePositive": 347, - "wilsonPrecisionLower95": 0.9458 - }, - "confirmationDecision": { - "f1": 0.7667, - "falseNegative": 65, - "falsePositive": 5, - "precision": 0.9583, - "predictedPositive": 120, - "recall": 0.6389, - "total": 2970, - "trueNegative": 2785, - "truePositive": 115, - "wilsonPrecisionLower95": 0.9062 - }, - "followUpReminder": { - "f1": 0.7063, - "falseNegative": 158, - "falsePositive": 10, - "precision": 0.9528, - "predictedPositive": 212, - "recall": 0.5611, - "total": 2970, - "trueNegative": 2600, - "truePositive": 202, - "wilsonPrecisionLower95": 0.9154 - }, - "invitation": { - "f1": 0.5382, - "falseNegative": 113, - "falsePositive": 2, - "precision": 0.971, - "predictedPositive": 69, - "recall": 0.3722, - "total": 2970, - "trueNegative": 2788, - "truePositive": 67, - "wilsonPrecisionLower95": 0.9003 - }, - "question": { - "f1": 0.8186, - "falseNegative": 181, - "falsePositive": 18, - "precision": 0.9615, - "predictedPositive": 467, - "recall": 0.7127, - "total": 2970, - "trueNegative": 2322, - "truePositive": 449, - "wilsonPrecisionLower95": 0.9399 - }, - "replyableMessage": { - "f1": 0.9035, - "falseNegative": 266, - "falsePositive": 81, - "precision": 0.9525, - "predictedPositive": 1705, - "recall": 0.8593, - "total": 2970, - "trueNegative": 999, - "truePositive": 1624, - "wilsonPrecisionLower95": 0.9413 - }, - "scheduleNegotiation": { - "f1": 0.9831, - "falseNegative": 3, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 87, - "recall": 0.9667, - "total": 2970, - "trueNegative": 2880, - "truePositive": 87, - "wilsonPrecisionLower95": 0.9577 - }, - "task": { - "f1": 0.809, - "falseNegative": 214, - "falsePositive": 25, - "precision": 0.9529, - "predictedPositive": 531, - "recall": 0.7028, - "total": 2970, - "trueNegative": 2225, - "truePositive": 506, - "wilsonPrecisionLower95": 0.9314 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 6e-06, - "augmentation": "punctuation", - "char_max": 6, - "char_min": 3, - "external_weight": 0.6900000000000001, - "hard_example_weight": 1.5, - "l1_ratio": 0.1, - "max_char_features": 65000, - "max_word_features": 32000, - "min_df": 2, - "pseudo_weight": 0.255, - "round": 7, - "word_max": 3 - }, - "featureCount": 87507, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 0 - }, - "complaint": { - "negative": 141, - "positive": 0 - }, - "confirmationDecision": { - "negative": 95, - "positive": 2 - }, - "followUpReminder": { - "negative": 250, - "positive": 0 - }, - "invitation": { - "negative": 250, - "positive": 0 - }, - "question": { - "negative": 36, - "positive": 5 - }, - "replyableMessage": { - "negative": 2, - "positive": 129 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 0 - }, - "task": { - "negative": 87, - "positive": 0 - } - }, - "round": 7, - "selectionObjective": 0.7501, - "syntheticRegression": { - "intentsMeeting95Precision": 5, - "macroF1": 0.587, - "macroPrecision": 0.9374, - "macroRecall": 0.4917, - "minimumPrecision": 0.7548, - "perIntent": { - "blessing": { - "f1": 0.0851, - "falseNegative": 86, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 4, - "recall": 0.0444, - "total": 2970, - "trueNegative": 2880, - "truePositive": 4, - "wilsonPrecisionLower95": 0.5101 - }, - "complaint": { - "f1": 0.9304, - "falseNegative": 39, - "falsePositive": 9, - "precision": 0.9727, - "predictedPositive": 330, - "recall": 0.8917, - "total": 2970, - "trueNegative": 2601, - "truePositive": 321, - "wilsonPrecisionLower95": 0.949 - }, - "confirmationDecision": { - "f1": 0.6985, - "falseNegative": 63, - "falsePositive": 38, - "precision": 0.7548, - "predictedPositive": 155, - "recall": 0.65, - "total": 2970, - "trueNegative": 2752, - "truePositive": 117, - "wilsonPrecisionLower95": 0.6815 - }, - "followUpReminder": { - "f1": 0.3974, - "falseNegative": 269, - "falsePositive": 7, - "precision": 0.9286, - "predictedPositive": 98, - "recall": 0.2528, - "total": 2970, - "trueNegative": 2603, - "truePositive": 91, - "wilsonPrecisionLower95": 0.8598 - }, - "invitation": { - "f1": 0.2609, - "falseNegative": 153, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 27, - "recall": 0.15, - "total": 2970, - "trueNegative": 2790, - "truePositive": 27, - "wilsonPrecisionLower95": 0.8754 - }, - "question": { - "f1": 0.697, - "falseNegative": 285, - "falsePositive": 15, - "precision": 0.9583, - "predictedPositive": 360, - "recall": 0.5476, - "total": 2970, - "trueNegative": 2325, - "truePositive": 345, - "wilsonPrecisionLower95": 0.9324 - }, - "replyableMessage": { - "f1": 0.8083, - "falseNegative": 572, - "falsePositive": 53, - "precision": 0.9613, - "predictedPositive": 1371, - "recall": 0.6974, - "total": 2970, - "trueNegative": 1027, - "truePositive": 1318, - "wilsonPrecisionLower95": 0.9498 - }, - "scheduleNegotiation": { - "f1": 0.8929, - "falseNegative": 15, - "falsePositive": 3, - "precision": 0.9615, - "predictedPositive": 78, - "recall": 0.8333, - "total": 2970, - "trueNegative": 2877, - "truePositive": 75, - "wilsonPrecisionLower95": 0.8929 - }, - "task": { - "f1": 0.5124, - "falseNegative": 462, - "falsePositive": 29, - "precision": 0.899, - "predictedPositive": 287, - "recall": 0.3583, - "total": 2970, - "trueNegative": 2221, - "truePositive": 258, - "wilsonPrecisionLower95": 0.8586 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9809, - "falseNegative": 0, - "falsePositive": 7, - "precision": 0.9626, - "predictedPositive": 187, - "recall": 1.0, - "total": 1485, - "trueNegative": 1298, - "truePositive": 180, - "wilsonPrecisionLower95": 0.9248 - }, - "threshold": 0.74 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8974, - "falseNegative": 27, - "falsePositive": 8, - "precision": 0.9503, - "predictedPositive": 161, - "recall": 0.85, - "total": 1485, - "trueNegative": 1297, - "truePositive": 153, - "wilsonPrecisionLower95": 0.905 - }, - "threshold": 0.665 - } - }, - "metrics": { - "f1": 0.9529, - "falseNegative": 16, - "falsePositive": 18, - "precision": 0.9503, - "predictedPositive": 362, - "recall": 0.9556, - "total": 2970, - "trueNegative": 2592, - "truePositive": 344, - "wilsonPrecisionLower95": 0.9228 - }, - "threshold": 0.51 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.7763, - "falseNegative": 31, - "falsePositive": 3, - "precision": 0.9516, - "predictedPositive": 62, - "recall": 0.6556, - "total": 1485, - "trueNegative": 1392, - "truePositive": 59, - "wilsonPrecisionLower95": 0.8671 - }, - "threshold": 0.895 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7568, - "falseNegative": 34, - "falsePositive": 2, - "precision": 0.9655, - "predictedPositive": 58, - "recall": 0.6222, - "total": 1485, - "trueNegative": 1393, - "truePositive": 56, - "wilsonPrecisionLower95": 0.8827 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.6569, - "falseNegative": 90, - "falsePositive": 4, - "precision": 0.9574, - "predictedPositive": 94, - "recall": 0.5, - "total": 2970, - "trueNegative": 2786, - "truePositive": 90, - "wilsonPrecisionLower95": 0.8956 - }, - "threshold": 0.895 - }, - "followUpReminder": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8421, - "falseNegative": 44, - "falsePositive": 7, - "precision": 0.951, - "predictedPositive": 143, - "recall": 0.7556, - "total": 1485, - "trueNegative": 1298, - "truePositive": 136, - "wilsonPrecisionLower95": 0.9024 - }, - "threshold": 0.985 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.5985, - "falseNegative": 205, - "falsePositive": 3, - "precision": 0.981, - "predictedPositive": 158, - "recall": 0.4306, - "total": 2970, - "trueNegative": 2607, - "truePositive": 155, - "wilsonPrecisionLower95": 0.9457 - }, - "threshold": 0.995 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.7143, - "falseNegative": 135, - "falsePositive": 9, - "precision": 0.9524, - "predictedPositive": 189, - "recall": 0.5714, - "total": 1485, - "trueNegative": 1161, - "truePositive": 180, - "wilsonPrecisionLower95": 0.912 - }, - "threshold": 0.965 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7168, - "falseNegative": 134, - "falsePositive": 9, - "precision": 0.9526, - "predictedPositive": 190, - "recall": 0.5746, - "total": 1485, - "trueNegative": 1161, - "truePositive": 181, - "wilsonPrecisionLower95": 0.9124 - }, - "threshold": 0.52 - } - }, - "metrics": { - "f1": 0.7149, - "falseNegative": 269, - "falsePositive": 19, - "precision": 0.95, - "predictedPositive": 380, - "recall": 0.573, - "total": 2970, - "trueNegative": 2321, - "truePositive": 361, - "wilsonPrecisionLower95": 0.9232 - }, - "threshold": 0.795 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8846, - "falseNegative": 163, - "falsePositive": 41, - "precision": 0.9502, - "predictedPositive": 823, - "recall": 0.8275, - "total": 1485, - "trueNegative": 499, - "truePositive": 782, - "wilsonPrecisionLower95": 0.9331 - }, - "threshold": 0.89 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7476, - "falseNegative": 363, - "falsePositive": 30, - "precision": 0.951, - "predictedPositive": 612, - "recall": 0.6159, - "total": 1485, - "trueNegative": 510, - "truePositive": 582, - "wilsonPrecisionLower95": 0.9309 - }, - "threshold": 0.92 - } - }, - "metrics": { - "f1": 0.828, - "falseNegative": 506, - "falsePositive": 69, - "precision": 0.9525, - "predictedPositive": 1453, - "recall": 0.7323, - "total": 2970, - "trueNegative": 1011, - "truePositive": 1384, - "wilsonPrecisionLower95": 0.9403 - }, - "threshold": 0.91 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 45, - "recall": 1.0, - "total": 1485, - "trueNegative": 1440, - "truePositive": 45, - "wilsonPrecisionLower95": 0.9213 - }, - "threshold": 0.66 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 45, - "recall": 1.0, - "total": 1485, - "trueNegative": 1440, - "truePositive": 45, - "wilsonPrecisionLower95": 0.9213 - }, - "threshold": 0.55 - } - }, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 90, - "recall": 1.0, - "total": 2970, - "trueNegative": 2880, - "truePositive": 90, - "wilsonPrecisionLower95": 0.9591 - }, - "threshold": 0.66 - }, - "task": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8615, - "falseNegative": 77, - "falsePositive": 14, - "precision": 0.9529, - "predictedPositive": 297, - "recall": 0.7861, - "total": 1485, - "trueNegative": 1111, - "truePositive": 283, - "wilsonPrecisionLower95": 0.9224 - }, - "threshold": 0.805 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.6868, - "falseNegative": 333, - "falsePositive": 20, - "precision": 0.9509, - "predictedPositive": 407, - "recall": 0.5375, - "total": 2970, - "trueNegative": 2230, - "truePositive": 387, - "wilsonPrecisionLower95": 0.9253 - }, - "threshold": 0.925 - } - }, - "trainingRecords": 17708, - "validation": { - "intentsMeeting95Precision": 8, - "macroF1": 0.6491, - "macroPrecision": 0.967, - "macroRecall": 0.5465, - "minimumPrecision": 0.9315, - "perIntent": { - "blessing": { - "f1": 0.2353, - "falseNegative": 78, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 12, - "recall": 0.1333, - "total": 2970, - "trueNegative": 2880, - "truePositive": 12, - "wilsonPrecisionLower95": 0.7575 - }, - "complaint": { - "f1": 0.9407, - "falseNegative": 27, - "falsePositive": 15, - "precision": 0.9569, - "predictedPositive": 348, - "recall": 0.925, - "total": 2970, - "trueNegative": 2595, - "truePositive": 333, - "wilsonPrecisionLower95": 0.9301 - }, - "confirmationDecision": { - "f1": 0.7667, - "falseNegative": 65, - "falsePositive": 5, - "precision": 0.9583, - "predictedPositive": 120, - "recall": 0.6389, - "total": 2970, - "trueNegative": 2785, - "truePositive": 115, - "wilsonPrecisionLower95": 0.9062 - }, - "followUpReminder": { - "f1": 0.5375, - "falseNegative": 224, - "falsePositive": 10, - "precision": 0.9315, - "predictedPositive": 146, - "recall": 0.3778, - "total": 2970, - "trueNegative": 2600, - "truePositive": 136, - "wilsonPrecisionLower95": 0.8785 - }, - "invitation": { - "f1": 0.2692, - "falseNegative": 152, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 28, - "recall": 0.1556, - "total": 2970, - "trueNegative": 2790, - "truePositive": 28, - "wilsonPrecisionLower95": 0.8794 - }, - "question": { - "f1": 0.7156, - "falseNegative": 269, - "falsePositive": 18, - "precision": 0.9525, - "predictedPositive": 379, - "recall": 0.573, - "total": 2970, - "trueNegative": 2322, - "truePositive": 361, - "wilsonPrecisionLower95": 0.9262 - }, - "replyableMessage": { - "f1": 0.8205, - "falseNegative": 526, - "falsePositive": 71, - "precision": 0.9505, - "predictedPositive": 1435, - "recall": 0.7217, - "total": 2970, - "trueNegative": 1009, - "truePositive": 1364, - "wilsonPrecisionLower95": 0.9381 - }, - "scheduleNegotiation": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 90, - "recall": 1.0, - "total": 2970, - "trueNegative": 2880, - "truePositive": 90, - "wilsonPrecisionLower95": 0.9591 - }, - "task": { - "f1": 0.5565, - "falseNegative": 437, - "falsePositive": 14, - "precision": 0.9529, - "predictedPositive": 297, - "recall": 0.3931, - "total": 2970, - "trueNegative": 2236, - "truePositive": 283, - "wilsonPrecisionLower95": 0.9224 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 5e-06, - "augmentation": "punctuation", - "char_max": 7, - "char_min": 2, - "external_weight": 0.7100000000000001, - "hard_example_weight": 1.7, - "l1_ratio": 0.15, - "max_char_features": 70000, - "max_word_features": 28000, - "min_df": 2, - "pseudo_weight": 0.27, - "round": 8, - "word_max": 2 - }, - "featureCount": 83313, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 0 - }, - "complaint": { - "negative": 187, - "positive": 0 - }, - "confirmationDecision": { - "negative": 107, - "positive": 2 - }, - "followUpReminder": { - "negative": 250, - "positive": 0 - }, - "invitation": { - "negative": 250, - "positive": 0 - }, - "question": { - "negative": 40, - "positive": 5 - }, - "replyableMessage": { - "negative": 2, - "positive": 135 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 0 - }, - "task": { - "negative": 94, - "positive": 0 - } - }, - "round": 8, - "selectionObjective": 0.7837, - "syntheticRegression": { - "intentsMeeting95Precision": 4, - "macroF1": 0.6947, - "macroPrecision": 0.9526, - "macroRecall": 0.6066, - "minimumPrecision": 0.8509, - "perIntent": { - "blessing": { - "f1": 0.125, - "falseNegative": 84, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 6, - "recall": 0.0667, - "total": 2970, - "trueNegative": 2880, - "truePositive": 6, - "wilsonPrecisionLower95": 0.6097 - }, - "complaint": { - "f1": 0.9744, - "falseNegative": 18, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 342, - "recall": 0.95, - "total": 2970, - "trueNegative": 2610, - "truePositive": 342, - "wilsonPrecisionLower95": 0.9889 - }, - "confirmationDecision": { - "f1": 0.8035, - "falseNegative": 43, - "falsePositive": 24, - "precision": 0.8509, - "predictedPositive": 161, - "recall": 0.7611, - "total": 2970, - "trueNegative": 2766, - "truePositive": 137, - "wilsonPrecisionLower95": 0.7878 - }, - "followUpReminder": { - "f1": 0.5422, - "falseNegative": 222, - "falsePositive": 11, - "precision": 0.9262, - "predictedPositive": 149, - "recall": 0.3833, - "total": 2970, - "trueNegative": 2599, - "truePositive": 138, - "wilsonPrecisionLower95": 0.8726 - }, - "invitation": { - "f1": 0.5062, - "falseNegative": 119, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 61, - "recall": 0.3389, - "total": 2970, - "trueNegative": 2790, - "truePositive": 61, - "wilsonPrecisionLower95": 0.9408 - }, - "question": { - "f1": 0.7719, - "falseNegative": 229, - "falsePositive": 8, - "precision": 0.9804, - "predictedPositive": 409, - "recall": 0.6365, - "total": 2970, - "trueNegative": 2332, - "truePositive": 401, - "wilsonPrecisionLower95": 0.9619 - }, - "replyableMessage": { - "f1": 0.8832, - "falseNegative": 366, - "falsePositive": 37, - "precision": 0.9763, - "predictedPositive": 1561, - "recall": 0.8063, - "total": 2970, - "trueNegative": 1043, - "truePositive": 1524, - "wilsonPrecisionLower95": 0.9675 - }, - "scheduleNegotiation": { - "f1": 0.9162, - "falseNegative": 8, - "falsePositive": 7, - "precision": 0.9213, - "predictedPositive": 89, - "recall": 0.9111, - "total": 2970, - "trueNegative": 2873, - "truePositive": 82, - "wilsonPrecisionLower95": 0.8464 - }, - "task": { - "f1": 0.7297, - "falseNegative": 284, - "falsePositive": 39, - "precision": 0.9179, - "predictedPositive": 475, - "recall": 0.6056, - "total": 2970, - "trueNegative": 2211, - "truePositive": 436, - "wilsonPrecisionLower95": 0.8897 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.989, - "falseNegative": 1, - "falsePositive": 3, - "precision": 0.9835, - "predictedPositive": 182, - "recall": 0.9944, - "total": 1485, - "trueNegative": 1302, - "truePositive": 179, - "wilsonPrecisionLower95": 0.9527 - }, - "threshold": 0.97 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9405, - "falseNegative": 14, - "falsePositive": 7, - "precision": 0.9595, - "predictedPositive": 173, - "recall": 0.9222, - "total": 1485, - "trueNegative": 1298, - "truePositive": 166, - "wilsonPrecisionLower95": 0.9188 - }, - "threshold": 0.54 - } - }, - "metrics": { - "f1": 0.9237, - "falseNegative": 39, - "falsePositive": 14, - "precision": 0.9582, - "predictedPositive": 335, - "recall": 0.8917, - "total": 2970, - "trueNegative": 2596, - "truePositive": 321, - "wilsonPrecisionLower95": 0.9311 - }, - "threshold": 0.925 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9613, - "falseNegative": 3, - "falsePositive": 4, - "precision": 0.956, - "predictedPositive": 91, - "recall": 0.9667, - "total": 1485, - "trueNegative": 1391, - "truePositive": 87, - "wilsonPrecisionLower95": 0.8924 - }, - "threshold": 0.72 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.5556, - "falseNegative": 55, - "falsePositive": 1, - "precision": 0.9722, - "predictedPositive": 36, - "recall": 0.3889, - "total": 1485, - "trueNegative": 1394, - "truePositive": 35, - "wilsonPrecisionLower95": 0.8583 - }, - "threshold": 0.905 - } - }, - "metrics": { - "f1": 0.7922, - "falseNegative": 58, - "falsePositive": 6, - "precision": 0.9531, - "predictedPositive": 128, - "recall": 0.6778, - "total": 2970, - "trueNegative": 2784, - "truePositive": 122, - "wilsonPrecisionLower95": 0.9015 - }, - "threshold": 0.88 - }, - "followUpReminder": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8437, - "falseNegative": 45, - "falsePositive": 5, - "precision": 0.9643, - "predictedPositive": 140, - "recall": 0.75, - "total": 1485, - "trueNegative": 1300, - "truePositive": 135, - "wilsonPrecisionLower95": 0.9191 - }, - "threshold": 0.96 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.7063, - "falseNegative": 158, - "falsePositive": 10, - "precision": 0.9528, - "predictedPositive": 212, - "recall": 0.5611, - "total": 2970, - "trueNegative": 2600, - "truePositive": 202, - "wilsonPrecisionLower95": 0.9154 - }, - "threshold": 0.96 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.7157, - "falseNegative": 135, - "falsePositive": 8, - "precision": 0.9574, - "predictedPositive": 188, - "recall": 0.5714, - "total": 1485, - "trueNegative": 1162, - "truePositive": 180, - "wilsonPrecisionLower95": 0.9183 - }, - "threshold": 0.975 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.822, - "falseNegative": 91, - "falsePositive": 6, - "precision": 0.9739, - "predictedPositive": 230, - "recall": 0.7111, - "total": 1485, - "trueNegative": 1164, - "truePositive": 224, - "wilsonPrecisionLower95": 0.9443 - }, - "threshold": 0.505 - } - }, - "metrics": { - "f1": 0.8095, - "falseNegative": 186, - "falsePositive": 23, - "precision": 0.9507, - "predictedPositive": 467, - "recall": 0.7048, - "total": 2970, - "trueNegative": 2317, - "truePositive": 444, - "wilsonPrecisionLower95": 0.9272 - }, - "threshold": 0.71 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8741, - "falseNegative": 181, - "falsePositive": 39, - "precision": 0.9514, - "predictedPositive": 803, - "recall": 0.8085, - "total": 1485, - "trueNegative": 501, - "truePositive": 764, - "wilsonPrecisionLower95": 0.9343 - }, - "threshold": 0.95 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8775, - "falseNegative": 175, - "falsePositive": 40, - "precision": 0.9506, - "predictedPositive": 810, - "recall": 0.8148, - "total": 1485, - "trueNegative": 500, - "truePositive": 770, - "wilsonPrecisionLower95": 0.9335 - }, - "threshold": 0.88 - } - }, - "metrics": { - "f1": 0.8762, - "falseNegative": 354, - "falsePositive": 80, - "precision": 0.9505, - "predictedPositive": 1616, - "recall": 0.8127, - "total": 2970, - "trueNegative": 1000, - "truePositive": 1536, - "wilsonPrecisionLower95": 0.9388 - }, - "threshold": 0.905 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 45, - "recall": 1.0, - "total": 1485, - "trueNegative": 1440, - "truePositive": 45, - "wilsonPrecisionLower95": 0.9213 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9556, - "falseNegative": 2, - "falsePositive": 2, - "precision": 0.9556, - "predictedPositive": 45, - "recall": 0.9556, - "total": 1485, - "trueNegative": 1438, - "truePositive": 43, - "wilsonPrecisionLower95": 0.8517 - }, - "threshold": 0.75 - } - }, - "metrics": { - "f1": 0.9724, - "falseNegative": 2, - "falsePositive": 3, - "precision": 0.967, - "predictedPositive": 91, - "recall": 0.9778, - "total": 2970, - "trueNegative": 2877, - "truePositive": 88, - "wilsonPrecisionLower95": 0.9075 - }, - "threshold": 0.505 - }, - "task": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8331, - "falseNegative": 93, - "falsePositive": 14, - "precision": 0.9502, - "predictedPositive": 281, - "recall": 0.7417, - "total": 1485, - "trueNegative": 1111, - "truePositive": 267, - "wilsonPrecisionLower95": 0.9181 - }, - "threshold": 0.95 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7445, - "falseNegative": 140, - "falsePositive": 11, - "precision": 0.9524, - "predictedPositive": 231, - "recall": 0.6111, - "total": 1485, - "trueNegative": 1114, - "truePositive": 220, - "wilsonPrecisionLower95": 0.9168 - }, - "threshold": 0.86 - } - }, - "metrics": { - "f1": 0.7672, - "falseNegative": 257, - "falsePositive": 24, - "precision": 0.9507, - "predictedPositive": 487, - "recall": 0.6431, - "total": 2970, - "trueNegative": 2226, - "truePositive": 463, - "wilsonPrecisionLower95": 0.9277 - }, - "threshold": 0.92 - } - }, - "trainingRecords": 17762, - "validation": { - "intentsMeeting95Precision": 8, - "macroF1": 0.698, - "macroPrecision": 0.9692, - "macroRecall": 0.6056, - "minimumPrecision": 0.9444, - "perIntent": { - "blessing": { - "f1": 0.2524, - "falseNegative": 77, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 13, - "recall": 0.1444, - "total": 2970, - "trueNegative": 2880, - "truePositive": 13, - "wilsonPrecisionLower95": 0.7719 - }, - "complaint": { - "f1": 0.965, - "falseNegative": 15, - "falsePositive": 10, - "precision": 0.9718, - "predictedPositive": 355, - "recall": 0.9583, - "total": 2970, - "trueNegative": 2600, - "truePositive": 345, - "wilsonPrecisionLower95": 0.9489 - }, - "confirmationDecision": { - "f1": 0.7948, - "falseNegative": 58, - "falsePositive": 5, - "precision": 0.9606, - "predictedPositive": 127, - "recall": 0.6778, - "total": 2970, - "trueNegative": 2785, - "truePositive": 122, - "wilsonPrecisionLower95": 0.9111 - }, - "followUpReminder": { - "f1": 0.5397, - "falseNegative": 224, - "falsePositive": 8, - "precision": 0.9444, - "predictedPositive": 144, - "recall": 0.3778, - "total": 2970, - "trueNegative": 2602, - "truePositive": 136, - "wilsonPrecisionLower95": 0.8942 - }, - "invitation": { - "f1": 0.3178, - "falseNegative": 146, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 34, - "recall": 0.1889, - "total": 2970, - "trueNegative": 2790, - "truePositive": 34, - "wilsonPrecisionLower95": 0.8985 - }, - "question": { - "f1": 0.771, - "falseNegative": 226, - "falsePositive": 14, - "precision": 0.9665, - "predictedPositive": 418, - "recall": 0.6413, - "total": 2970, - "trueNegative": 2326, - "truePositive": 404, - "wilsonPrecisionLower95": 0.9446 - }, - "replyableMessage": { - "f1": 0.8758, - "falseNegative": 356, - "falsePositive": 79, - "precision": 0.951, - "predictedPositive": 1613, - "recall": 0.8116, - "total": 2970, - "trueNegative": 1001, - "truePositive": 1534, - "wilsonPrecisionLower95": 0.9394 - }, - "scheduleNegotiation": { - "f1": 0.9778, - "falseNegative": 2, - "falsePositive": 2, - "precision": 0.9778, - "predictedPositive": 90, - "recall": 0.9778, - "total": 2970, - "trueNegative": 2878, - "truePositive": 88, - "wilsonPrecisionLower95": 0.9226 - }, - "task": { - "f1": 0.7876, - "falseNegative": 236, - "falsePositive": 25, - "precision": 0.9509, - "predictedPositive": 509, - "recall": 0.6722, - "total": 2970, - "trueNegative": 2225, - "truePositive": 484, - "wilsonPrecisionLower95": 0.9285 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 4e-06, - "augmentation": "punctuation", - "char_max": 7, - "char_min": 3, - "external_weight": 0.73, - "hard_example_weight": 1.7, - "l1_ratio": 0.15, - "max_char_features": 70000, - "max_word_features": 35000, - "min_df": 2, - "pseudo_weight": 0.28500000000000003, - "round": 9, - "word_max": 3 - }, - "featureCount": 92641, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 0 - }, - "complaint": { - "negative": 250, - "positive": 0 - }, - "confirmationDecision": { - "negative": 248, - "positive": 6 - }, - "followUpReminder": { - "negative": 250, - "positive": 1 - }, - "invitation": { - "negative": 250, - "positive": 0 - }, - "question": { - "negative": 146, - "positive": 8 - }, - "replyableMessage": { - "negative": 8, - "positive": 250 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 0 - }, - "task": { - "negative": 208, - "positive": 0 - } - }, - "round": 9, - "selectionObjective": 0.7717, - "syntheticRegression": { - "intentsMeeting95Precision": 4, - "macroF1": 0.6445, - "macroPrecision": 0.9357, - "macroRecall": 0.5386, - "minimumPrecision": 0.8583, - "perIntent": { - "blessing": { - "f1": 0.125, - "falseNegative": 84, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 6, - "recall": 0.0667, - "total": 2970, - "trueNegative": 2880, - "truePositive": 6, - "wilsonPrecisionLower95": 0.6097 - }, - "complaint": { - "f1": 0.949, - "falseNegative": 25, - "falsePositive": 11, - "precision": 0.9682, - "predictedPositive": 346, - "recall": 0.9306, - "total": 2970, - "trueNegative": 2599, - "truePositive": 335, - "wilsonPrecisionLower95": 0.944 - }, - "confirmationDecision": { - "f1": 0.7101, - "falseNegative": 71, - "falsePositive": 18, - "precision": 0.8583, - "predictedPositive": 127, - "recall": 0.6056, - "total": 2970, - "trueNegative": 2772, - "truePositive": 109, - "wilsonPrecisionLower95": 0.7871 - }, - "followUpReminder": { - "f1": 0.5244, - "falseNegative": 231, - "falsePositive": 3, - "precision": 0.9773, - "predictedPositive": 132, - "recall": 0.3583, - "total": 2970, - "trueNegative": 2607, - "truePositive": 129, - "wilsonPrecisionLower95": 0.9353 - }, - "invitation": { - "f1": 0.6286, - "falseNegative": 92, - "falsePositive": 12, - "precision": 0.88, - "predictedPositive": 100, - "recall": 0.4889, - "total": 2970, - "trueNegative": 2778, - "truePositive": 88, - "wilsonPrecisionLower95": 0.8019 - }, - "question": { - "f1": 0.6667, - "falseNegative": 310, - "falsePositive": 10, - "precision": 0.9697, - "predictedPositive": 330, - "recall": 0.5079, - "total": 2970, - "trueNegative": 2330, - "truePositive": 320, - "wilsonPrecisionLower95": 0.9451 - }, - "replyableMessage": { - "f1": 0.7919, - "falseNegative": 594, - "falsePositive": 87, - "precision": 0.9371, - "predictedPositive": 1383, - "recall": 0.6857, - "total": 2970, - "trueNegative": 993, - "truePositive": 1296, - "wilsonPrecisionLower95": 0.923 - }, - "scheduleNegotiation": { - "f1": 0.8941, - "falseNegative": 14, - "falsePositive": 4, - "precision": 0.95, - "predictedPositive": 80, - "recall": 0.8444, - "total": 2970, - "trueNegative": 2876, - "truePositive": 76, - "wilsonPrecisionLower95": 0.8784 - }, - "task": { - "f1": 0.5108, - "falseNegative": 461, - "falsePositive": 35, - "precision": 0.881, - "predictedPositive": 294, - "recall": 0.3597, - "total": 2970, - "trueNegative": 2215, - "truePositive": 259, - "wilsonPrecisionLower95": 0.8389 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9862, - "falseNegative": 1, - "falsePositive": 4, - "precision": 0.9781, - "predictedPositive": 183, - "recall": 0.9944, - "total": 1485, - "trueNegative": 1301, - "truePositive": 179, - "wilsonPrecisionLower95": 0.9452 - }, - "threshold": 0.74 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9222, - "falseNegative": 20, - "falsePositive": 7, - "precision": 0.9581, - "predictedPositive": 167, - "recall": 0.8889, - "total": 1485, - "trueNegative": 1298, - "truePositive": 160, - "wilsonPrecisionLower95": 0.916 - }, - "threshold": 0.62 - } - }, - "metrics": { - "f1": 0.9528, - "falseNegative": 17, - "falsePositive": 17, - "precision": 0.9528, - "predictedPositive": 360, - "recall": 0.9528, - "total": 2970, - "trueNegative": 2593, - "truePositive": 343, - "wilsonPrecisionLower95": 0.9257 - }, - "threshold": 0.5 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9492, - "falseNegative": 6, - "falsePositive": 3, - "precision": 0.9655, - "predictedPositive": 87, - "recall": 0.9333, - "total": 1485, - "trueNegative": 1392, - "truePositive": 84, - "wilsonPrecisionLower95": 0.9035 - }, - "threshold": 0.965 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7042, - "falseNegative": 40, - "falsePositive": 2, - "precision": 0.9615, - "predictedPositive": 52, - "recall": 0.5556, - "total": 1485, - "trueNegative": 1393, - "truePositive": 50, - "wilsonPrecisionLower95": 0.8702 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.76, - "falseNegative": 66, - "falsePositive": 6, - "precision": 0.95, - "predictedPositive": 120, - "recall": 0.6333, - "total": 2970, - "trueNegative": 2784, - "truePositive": 114, - "wilsonPrecisionLower95": 0.8952 - }, - "threshold": 0.945 - }, - "followUpReminder": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8243, - "falseNegative": 51, - "falsePositive": 4, - "precision": 0.9699, - "predictedPositive": 133, - "recall": 0.7167, - "total": 1485, - "trueNegative": 1301, - "truePositive": 129, - "wilsonPrecisionLower95": 0.9252 - }, - "threshold": 0.99 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.6316, - "falseNegative": 192, - "falsePositive": 4, - "precision": 0.9767, - "predictedPositive": 172, - "recall": 0.4667, - "total": 2970, - "trueNegative": 2606, - "truePositive": 168, - "wilsonPrecisionLower95": 0.9417 - }, - "threshold": 0.995 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.544, - "falseNegative": 56, - "falsePositive": 1, - "precision": 0.9714, - "predictedPositive": 35, - "recall": 0.3778, - "total": 1485, - "trueNegative": 1394, - "truePositive": 34, - "wilsonPrecisionLower95": 0.8547 - }, - "threshold": 0.82 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.6694, - "falseNegative": 153, - "falsePositive": 7, - "precision": 0.9586, - "predictedPositive": 169, - "recall": 0.5143, - "total": 1485, - "trueNegative": 1163, - "truePositive": 162, - "wilsonPrecisionLower95": 0.917 - }, - "threshold": 0.985 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6762, - "falseNegative": 150, - "falsePositive": 8, - "precision": 0.9538, - "predictedPositive": 173, - "recall": 0.5238, - "total": 1485, - "trueNegative": 1162, - "truePositive": 165, - "wilsonPrecisionLower95": 0.9114 - }, - "threshold": 0.64 - } - }, - "metrics": { - "f1": 0.5392, - "falseNegative": 393, - "falsePositive": 12, - "precision": 0.9518, - "predictedPositive": 249, - "recall": 0.3762, - "total": 2970, - "trueNegative": 2328, - "truePositive": 237, - "wilsonPrecisionLower95": 0.9177 - }, - "threshold": 0.965 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9407, - "falseNegative": 65, - "falsePositive": 46, - "precision": 0.9503, - "predictedPositive": 926, - "recall": 0.9312, - "total": 1485, - "trueNegative": 494, - "truePositive": 880, - "wilsonPrecisionLower95": 0.9344 - }, - "threshold": 0.745 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6118, - "falseNegative": 521, - "falsePositive": 17, - "precision": 0.9615, - "predictedPositive": 441, - "recall": 0.4487, - "total": 1485, - "trueNegative": 523, - "truePositive": 424, - "wilsonPrecisionLower95": 0.9391 - }, - "threshold": 0.965 - } - }, - "metrics": { - "f1": 0.8037, - "falseNegative": 576, - "falsePositive": 66, - "precision": 0.9522, - "predictedPositive": 1380, - "recall": 0.6952, - "total": 2970, - "trueNegative": 1014, - "truePositive": 1314, - "wilsonPrecisionLower95": 0.9396 - }, - "threshold": 0.95 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9286, - "falseNegative": 6, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 39, - "recall": 0.8667, - "total": 1485, - "trueNegative": 1440, - "truePositive": 39, - "wilsonPrecisionLower95": 0.9103 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9302, - "falseNegative": 5, - "falsePositive": 1, - "precision": 0.9756, - "predictedPositive": 41, - "recall": 0.8889, - "total": 1485, - "trueNegative": 1439, - "truePositive": 40, - "wilsonPrecisionLower95": 0.874 - }, - "threshold": 0.61 - } - }, - "metrics": { - "f1": 0.9294, - "falseNegative": 11, - "falsePositive": 1, - "precision": 0.9875, - "predictedPositive": 80, - "recall": 0.8778, - "total": 2970, - "trueNegative": 2879, - "truePositive": 79, - "wilsonPrecisionLower95": 0.9325 - }, - "threshold": 0.61 - }, - "task": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8403, - "falseNegative": 89, - "falsePositive": 14, - "precision": 0.9509, - "predictedPositive": 285, - "recall": 0.7528, - "total": 1485, - "trueNegative": 1111, - "truePositive": 271, - "wilsonPrecisionLower95": 0.9192 - }, - "threshold": 0.93 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.2384, - "falseNegative": 311, - "falsePositive": 2, - "precision": 0.9608, - "predictedPositive": 51, - "recall": 0.1361, - "total": 1485, - "trueNegative": 1123, - "truePositive": 49, - "wilsonPrecisionLower95": 0.8678 - }, - "threshold": 0.995 - } - }, - "metrics": { - "f1": 0.6415, - "falseNegative": 372, - "falsePositive": 17, - "precision": 0.9534, - "predictedPositive": 365, - "recall": 0.4833, - "total": 2970, - "trueNegative": 2233, - "truePositive": 348, - "wilsonPrecisionLower95": 0.9267 - }, - "threshold": 0.975 - } - }, - "trainingRecords": 17823, - "validation": { - "intentsMeeting95Precision": 8, - "macroF1": 0.6858, - "macroPrecision": 0.9638, - "macroRecall": 0.5714, - "minimumPrecision": 0.9211, - "perIntent": { - "blessing": { - "f1": 0.2857, - "falseNegative": 75, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 15, - "recall": 0.1667, - "total": 2970, - "trueNegative": 2880, - "truePositive": 15, - "wilsonPrecisionLower95": 0.7961 - }, - "complaint": { - "f1": 0.9549, - "falseNegative": 21, - "falsePositive": 11, - "precision": 0.9686, - "predictedPositive": 350, - "recall": 0.9417, - "total": 2970, - "trueNegative": 2599, - "truePositive": 339, - "wilsonPrecisionLower95": 0.9446 - }, - "confirmationDecision": { - "f1": 0.8401, - "falseNegative": 46, - "falsePositive": 5, - "precision": 0.964, - "predictedPositive": 139, - "recall": 0.7444, - "total": 2970, - "trueNegative": 2785, - "truePositive": 134, - "wilsonPrecisionLower95": 0.9186 - }, - "followUpReminder": { - "f1": 0.5352, - "falseNegative": 227, - "falsePositive": 4, - "precision": 0.9708, - "predictedPositive": 137, - "recall": 0.3694, - "total": 2970, - "trueNegative": 2606, - "truePositive": 133, - "wilsonPrecisionLower95": 0.9273 - }, - "invitation": { - "f1": 0.5469, - "falseNegative": 110, - "falsePositive": 6, - "precision": 0.9211, - "predictedPositive": 76, - "recall": 0.3889, - "total": 2970, - "trueNegative": 2784, - "truePositive": 70, - "wilsonPrecisionLower95": 0.8383 - }, - "question": { - "f1": 0.6728, - "falseNegative": 303, - "falsePositive": 15, - "precision": 0.9561, - "predictedPositive": 342, - "recall": 0.519, - "total": 2970, - "trueNegative": 2325, - "truePositive": 327, - "wilsonPrecisionLower95": 0.9289 - }, - "replyableMessage": { - "f1": 0.8007, - "falseNegative": 586, - "falsePositive": 63, - "precision": 0.9539, - "predictedPositive": 1367, - "recall": 0.6899, - "total": 2970, - "trueNegative": 1017, - "truePositive": 1304, - "wilsonPrecisionLower95": 0.9415 - }, - "scheduleNegotiation": { - "f1": 0.9294, - "falseNegative": 11, - "falsePositive": 1, - "precision": 0.9875, - "predictedPositive": 80, - "recall": 0.8778, - "total": 2970, - "trueNegative": 2879, - "truePositive": 79, - "wilsonPrecisionLower95": 0.9325 - }, - "task": { - "f1": 0.6061, - "falseNegative": 400, - "falsePositive": 16, - "precision": 0.9524, - "predictedPositive": 336, - "recall": 0.4444, - "total": 2970, - "trueNegative": 2234, - "truePositive": 320, - "wilsonPrecisionLower95": 0.9241 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 6e-06, - "augmentation": "punctuation", - "char_max": 6, - "char_min": 2, - "external_weight": 0.75, - "hard_example_weight": 1.8, - "l1_ratio": 0.2, - "max_char_features": 75000, - "max_word_features": 38000, - "min_df": 1, - "pseudo_weight": 0.3, - "round": 10, - "word_max": 3 - }, - "featureCount": 113000, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 0 - }, - "complaint": { - "negative": 250, - "positive": 0 - }, - "confirmationDecision": { - "negative": 250, - "positive": 11 - }, - "followUpReminder": { - "negative": 250, - "positive": 4 - }, - "invitation": { - "negative": 250, - "positive": 0 - }, - "question": { - "negative": 182, - "positive": 10 - }, - "replyableMessage": { - "negative": 11, - "positive": 250 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 0 - }, - "task": { - "negative": 250, - "positive": 1 - } - }, - "round": 10, - "selectionObjective": 0.7542, - "syntheticRegression": { - "intentsMeeting95Precision": 4, - "macroF1": 0.6448, - "macroPrecision": 0.9609, - "macroRecall": 0.5363, - "minimumPrecision": 0.8734, - "perIntent": { - "blessing": { - "f1": 0.125, - "falseNegative": 84, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 6, - "recall": 0.0667, - "total": 2970, - "trueNegative": 2880, - "truePositive": 6, - "wilsonPrecisionLower95": 0.6097 - }, - "complaint": { - "f1": 0.9366, - "falseNegative": 20, - "falsePositive": 26, - "precision": 0.929, - "predictedPositive": 366, - "recall": 0.9444, - "total": 2970, - "trueNegative": 2584, - "truePositive": 340, - "wilsonPrecisionLower95": 0.8979 - }, - "confirmationDecision": { - "f1": 0.8166, - "falseNegative": 42, - "falsePositive": 20, - "precision": 0.8734, - "predictedPositive": 158, - "recall": 0.7667, - "total": 2970, - "trueNegative": 2770, - "truePositive": 138, - "wilsonPrecisionLower95": 0.8126 - }, - "followUpReminder": { - "f1": 0.5397, - "falseNegative": 224, - "falsePositive": 8, - "precision": 0.9444, - "predictedPositive": 144, - "recall": 0.3778, - "total": 2970, - "trueNegative": 2602, - "truePositive": 136, - "wilsonPrecisionLower95": 0.8942 - }, - "invitation": { - "f1": 0.5124, - "falseNegative": 118, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 62, - "recall": 0.3444, - "total": 2970, - "trueNegative": 2790, - "truePositive": 62, - "wilsonPrecisionLower95": 0.9417 - }, - "question": { - "f1": 0.5488, - "falseNegative": 391, - "falsePositive": 2, - "precision": 0.9917, - "predictedPositive": 241, - "recall": 0.3794, - "total": 2970, - "trueNegative": 2338, - "truePositive": 239, - "wilsonPrecisionLower95": 0.9703 - }, - "replyableMessage": { - "f1": 0.7216, - "falseNegative": 808, - "falsePositive": 27, - "precision": 0.9757, - "predictedPositive": 1109, - "recall": 0.5725, - "total": 2970, - "trueNegative": 1053, - "truePositive": 1082, - "wilsonPrecisionLower95": 0.9648 - }, - "scheduleNegotiation": { - "f1": 0.882, - "falseNegative": 19, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 71, - "recall": 0.7889, - "total": 2970, - "trueNegative": 2880, - "truePositive": 71, - "wilsonPrecisionLower95": 0.9487 - }, - "task": { - "f1": 0.7201, - "falseNegative": 298, - "falsePositive": 30, - "precision": 0.9336, - "predictedPositive": 452, - "recall": 0.5861, - "total": 2970, - "trueNegative": 2220, - "truePositive": 422, - "wilsonPrecisionLower95": 0.9068 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9807, - "falseNegative": 2, - "falsePositive": 5, - "precision": 0.9727, - "predictedPositive": 183, - "recall": 0.9889, - "total": 1485, - "trueNegative": 1300, - "truePositive": 178, - "wilsonPrecisionLower95": 0.9376 - }, - "threshold": 0.765 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9288, - "falseNegative": 17, - "falsePositive": 8, - "precision": 0.9532, - "predictedPositive": 171, - "recall": 0.9056, - "total": 1485, - "trueNegative": 1297, - "truePositive": 163, - "wilsonPrecisionLower95": 0.9104 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.9455, - "falseNegative": 22, - "falsePositive": 17, - "precision": 0.9521, - "predictedPositive": 355, - "recall": 0.9389, - "total": 2970, - "trueNegative": 2593, - "truePositive": 338, - "wilsonPrecisionLower95": 0.9247 - }, - "threshold": 0.595 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.7568, - "falseNegative": 34, - "falsePositive": 2, - "precision": 0.9655, - "predictedPositive": 58, - "recall": 0.6222, - "total": 1485, - "trueNegative": 1393, - "truePositive": 56, - "wilsonPrecisionLower95": 0.8827 - }, - "threshold": 0.955 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7733, - "falseNegative": 32, - "falsePositive": 2, - "precision": 0.9667, - "predictedPositive": 60, - "recall": 0.6444, - "total": 1485, - "trueNegative": 1393, - "truePositive": 58, - "wilsonPrecisionLower95": 0.8864 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.7723, - "falseNegative": 63, - "falsePositive": 6, - "precision": 0.9512, - "predictedPositive": 123, - "recall": 0.65, - "total": 2970, - "trueNegative": 2784, - "truePositive": 117, - "wilsonPrecisionLower95": 0.8977 - }, - "threshold": 0.765 - }, - "followUpReminder": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8483, - "falseNegative": 43, - "falsePositive": 6, - "precision": 0.958, - "predictedPositive": 143, - "recall": 0.7611, - "total": 1485, - "trueNegative": 1299, - "truePositive": 137, - "wilsonPrecisionLower95": 0.9115 - }, - "threshold": 0.995 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.3529, - "falseNegative": 141, - "falsePositive": 2, - "precision": 0.9512, - "predictedPositive": 41, - "recall": 0.2167, - "total": 1485, - "trueNegative": 1303, - "truePositive": 39, - "wilsonPrecisionLower95": 0.8386 - }, - "threshold": 0.99 - } - }, - "metrics": { - "f1": 0.6292, - "falseNegative": 192, - "falsePositive": 6, - "precision": 0.9655, - "predictedPositive": 174, - "recall": 0.4667, - "total": 2970, - "trueNegative": 2604, - "truePositive": 168, - "wilsonPrecisionLower95": 0.9268 - }, - "threshold": 0.995 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.3894, - "falseNegative": 68, - "falsePositive": 1, - "precision": 0.9565, - "predictedPositive": 23, - "recall": 0.2444, - "total": 1485, - "trueNegative": 1394, - "truePositive": 22, - "wilsonPrecisionLower95": 0.7901 - }, - "threshold": 0.98 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8175, - "falseNegative": 91, - "falsePositive": 9, - "precision": 0.9614, - "predictedPositive": 233, - "recall": 0.7111, - "total": 1485, - "trueNegative": 1161, - "truePositive": 224, - "wilsonPrecisionLower95": 0.9282 - }, - "threshold": 0.6 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.7182, - "falseNegative": 401, - "falsePositive": 26, - "precision": 0.9544, - "predictedPositive": 570, - "recall": 0.5757, - "total": 1485, - "trueNegative": 514, - "truePositive": 544, - "wilsonPrecisionLower95": 0.934 - }, - "threshold": 0.985 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6667, - "falseNegative": 460, - "falsePositive": 25, - "precision": 0.951, - "predictedPositive": 510, - "recall": 0.5132, - "total": 1485, - "trueNegative": 515, - "truePositive": 485, - "wilsonPrecisionLower95": 0.9286 - }, - "threshold": 0.945 - } - }, - "metrics": { - "f1": 0.7151, - "falseNegative": 807, - "falsePositive": 56, - "precision": 0.9508, - "predictedPositive": 1139, - "recall": 0.573, - "total": 2970, - "trueNegative": 1024, - "truePositive": 1083, - "wilsonPrecisionLower95": 0.9367 - }, - "threshold": 0.96 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9412, - "falseNegative": 5, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 40, - "recall": 0.8889, - "total": 1485, - "trueNegative": 1440, - "truePositive": 40, - "wilsonPrecisionLower95": 0.9124 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8052, - "falseNegative": 14, - "falsePositive": 1, - "precision": 0.9688, - "predictedPositive": 32, - "recall": 0.6889, - "total": 1485, - "trueNegative": 1439, - "truePositive": 31, - "wilsonPrecisionLower95": 0.8426 - }, - "threshold": 0.98 - } - }, - "metrics": { - "f1": 0.8375, - "falseNegative": 23, - "falsePositive": 3, - "precision": 0.9571, - "predictedPositive": 70, - "recall": 0.7444, - "total": 2970, - "trueNegative": 2877, - "truePositive": 67, - "wilsonPrecisionLower95": 0.8814 - }, - "threshold": 0.92 - }, - "task": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.7783, - "falseNegative": 123, - "falsePositive": 12, - "precision": 0.9518, - "predictedPositive": 249, - "recall": 0.6583, - "total": 1485, - "trueNegative": 1113, - "truePositive": 237, - "wilsonPrecisionLower95": 0.9177 - }, - "threshold": 0.84 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7994, - "falseNegative": 113, - "falsePositive": 11, - "precision": 0.9574, - "predictedPositive": 258, - "recall": 0.6861, - "total": 1485, - "trueNegative": 1114, - "truePositive": 247, - "wilsonPrecisionLower95": 0.9253 - }, - "threshold": 0.755 - } - }, - "metrics": { - "f1": 0.7977, - "falseNegative": 225, - "falsePositive": 26, - "precision": 0.9501, - "predictedPositive": 521, - "recall": 0.6875, - "total": 2970, - "trueNegative": 2224, - "truePositive": 495, - "wilsonPrecisionLower95": 0.9279 - }, - "threshold": 0.77 - } - }, - "trainingRecords": 17763, - "validation": { - "intentsMeeting95Precision": 8, - "macroF1": 0.6563, - "macroPrecision": 0.9695, - "macroRecall": 0.5386, - "minimumPrecision": 0.9528, - "perIntent": { - "blessing": { - "f1": 0.1818, - "falseNegative": 81, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 9, - "recall": 0.1, - "total": 2970, - "trueNegative": 2880, - "truePositive": 9, - "wilsonPrecisionLower95": 0.7009 - }, - "complaint": { - "f1": 0.9552, - "falseNegative": 19, - "falsePositive": 13, - "precision": 0.9633, - "predictedPositive": 354, - "recall": 0.9472, - "total": 2970, - "trueNegative": 2597, - "truePositive": 341, - "wilsonPrecisionLower95": 0.9382 - }, - "confirmationDecision": { - "f1": 0.7651, - "falseNegative": 66, - "falsePositive": 4, - "precision": 0.9661, - "predictedPositive": 118, - "recall": 0.6333, - "total": 2970, - "trueNegative": 2786, - "truePositive": 114, - "wilsonPrecisionLower95": 0.9161 - }, - "followUpReminder": { - "f1": 0.6471, - "falseNegative": 184, - "falsePositive": 8, - "precision": 0.9565, - "predictedPositive": 184, - "recall": 0.4889, - "total": 2970, - "trueNegative": 2602, - "truePositive": 176, - "wilsonPrecisionLower95": 0.9166 - }, - "invitation": { - "f1": 0.479, - "falseNegative": 123, - "falsePositive": 1, - "precision": 0.9828, - "predictedPositive": 58, - "recall": 0.3167, - "total": 2970, - "trueNegative": 2789, - "truePositive": 57, - "wilsonPrecisionLower95": 0.9086 - }, - "question": { - "f1": 0.5191, - "falseNegative": 406, - "falsePositive": 9, - "precision": 0.9614, - "predictedPositive": 233, - "recall": 0.3556, - "total": 2970, - "trueNegative": 2331, - "truePositive": 224, - "wilsonPrecisionLower95": 0.9282 - }, - "replyableMessage": { - "f1": 0.6929, - "falseNegative": 861, - "falsePositive": 51, - "precision": 0.9528, - "predictedPositive": 1080, - "recall": 0.5444, - "total": 2970, - "trueNegative": 1029, - "truePositive": 1029, - "wilsonPrecisionLower95": 0.9384 - }, - "scheduleNegotiation": { - "f1": 0.8765, - "falseNegative": 19, - "falsePositive": 1, - "precision": 0.9861, - "predictedPositive": 72, - "recall": 0.7889, - "total": 2970, - "trueNegative": 2879, - "truePositive": 71, - "wilsonPrecisionLower95": 0.9254 - }, - "task": { - "f1": 0.7896, - "falseNegative": 236, - "falsePositive": 22, - "precision": 0.9565, - "predictedPositive": 506, - "recall": 0.6722, - "total": 2970, - "trueNegative": 2228, - "truePositive": 484, - "wilsonPrecisionLower95": 0.935 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 5e-06, - "augmentation": "prefix", - "char_max": 6, - "char_min": 2, - "external_weight": 0.77, - "hard_example_weight": 1.8, - "l1_ratio": 0.1, - "max_char_features": 70000, - "max_word_features": 30000, - "min_df": 2, - "pseudo_weight": 0.31499999999999995, - "round": 11, - "word_max": 2 - }, - "featureCount": 81451, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 0 - }, - "complaint": { - "negative": 250, - "positive": 0 - }, - "confirmationDecision": { - "negative": 250, - "positive": 14 - }, - "followUpReminder": { - "negative": 250, - "positive": 2 - }, - "invitation": { - "negative": 250, - "positive": 1 - }, - "question": { - "negative": 233, - "positive": 0 - }, - "replyableMessage": { - "negative": 8, - "positive": 250 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 0 - }, - "task": { - "negative": 250, - "positive": 0 - } - }, - "round": 11, - "selectionObjective": 0.8015, - "syntheticRegression": { - "intentsMeeting95Precision": 4, - "macroF1": 0.6807, - "macroPrecision": 0.963, - "macroRecall": 0.6053, - "minimumPrecision": 0.8782, - "perIntent": { - "blessing": { - "f1": 0.125, - "falseNegative": 84, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 6, - "recall": 0.0667, - "total": 2970, - "trueNegative": 2880, - "truePositive": 6, - "wilsonPrecisionLower95": 0.6097 - }, - "complaint": { - "f1": 0.9655, - "falseNegative": 24, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 336, - "recall": 0.9333, - "total": 2970, - "trueNegative": 2610, - "truePositive": 336, - "wilsonPrecisionLower95": 0.9887 - }, - "confirmationDecision": { - "f1": 0.8155, - "falseNegative": 43, - "falsePositive": 19, - "precision": 0.8782, - "predictedPositive": 156, - "recall": 0.7611, - "total": 2970, - "trueNegative": 2771, - "truePositive": 137, - "wilsonPrecisionLower95": 0.8176 - }, - "followUpReminder": { - "f1": 0.2439, - "falseNegative": 310, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 50, - "recall": 0.1389, - "total": 2970, - "trueNegative": 2610, - "truePositive": 50, - "wilsonPrecisionLower95": 0.9287 - }, - "invitation": { - "f1": 0.5425, - "falseNegative": 113, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 67, - "recall": 0.3722, - "total": 2970, - "trueNegative": 2790, - "truePositive": 67, - "wilsonPrecisionLower95": 0.9458 - }, - "question": { - "f1": 0.8494, - "falseNegative": 162, - "falsePositive": 4, - "precision": 0.9915, - "predictedPositive": 472, - "recall": 0.7429, - "total": 2970, - "trueNegative": 2336, - "truePositive": 468, - "wilsonPrecisionLower95": 0.9784 - }, - "replyableMessage": { - "f1": 0.8975, - "falseNegative": 253, - "falsePositive": 121, - "precision": 0.9312, - "predictedPositive": 1758, - "recall": 0.8661, - "total": 2970, - "trueNegative": 959, - "truePositive": 1637, - "wilsonPrecisionLower95": 0.9184 - }, - "scheduleNegotiation": { - "f1": 0.9444, - "falseNegative": 5, - "falsePositive": 5, - "precision": 0.9444, - "predictedPositive": 90, - "recall": 0.9444, - "total": 2970, - "trueNegative": 2875, - "truePositive": 85, - "wilsonPrecisionLower95": 0.8765 - }, - "task": { - "f1": 0.743, - "falseNegative": 272, - "falsePositive": 38, - "precision": 0.9218, - "predictedPositive": 486, - "recall": 0.6222, - "total": 2970, - "trueNegative": 2212, - "truePositive": 448, - "wilsonPrecisionLower95": 0.8945 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9917, - "falseNegative": 0, - "falsePositive": 3, - "precision": 0.9836, - "predictedPositive": 183, - "recall": 1.0, - "total": 1485, - "trueNegative": 1302, - "truePositive": 180, - "wilsonPrecisionLower95": 0.9529 - }, - "threshold": 0.99 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9318, - "falseNegative": 16, - "falsePositive": 8, - "precision": 0.9535, - "predictedPositive": 172, - "recall": 0.9111, - "total": 1485, - "trueNegative": 1297, - "truePositive": 164, - "wilsonPrecisionLower95": 0.9109 - }, - "threshold": 0.84 - } - }, - "metrics": { - "f1": 0.9529, - "falseNegative": 16, - "falsePositive": 18, - "precision": 0.9503, - "predictedPositive": 362, - "recall": 0.9556, - "total": 2970, - "trueNegative": 2592, - "truePositive": 344, - "wilsonPrecisionLower95": 0.9228 - }, - "threshold": 0.84 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.931, - "falseNegative": 9, - "falsePositive": 3, - "precision": 0.9643, - "predictedPositive": 84, - "recall": 0.9, - "total": 1485, - "trueNegative": 1392, - "truePositive": 81, - "wilsonPrecisionLower95": 0.9002 - }, - "threshold": 0.97 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7222, - "falseNegative": 38, - "falsePositive": 2, - "precision": 0.963, - "predictedPositive": 54, - "recall": 0.5778, - "total": 1485, - "trueNegative": 1393, - "truePositive": 52, - "wilsonPrecisionLower95": 0.8746 - }, - "threshold": 0.86 - } - }, - "metrics": { - "f1": 0.8077, - "falseNegative": 54, - "falsePositive": 6, - "precision": 0.9545, - "predictedPositive": 132, - "recall": 0.7, - "total": 2970, - "trueNegative": 2784, - "truePositive": 126, - "wilsonPrecisionLower95": 0.9044 - }, - "threshold": 0.95 - }, - "followUpReminder": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.4833, - "falseNegative": 61, - "falsePositive": 1, - "precision": 0.9667, - "predictedPositive": 30, - "recall": 0.3222, - "total": 1485, - "trueNegative": 1394, - "truePositive": 29, - "wilsonPrecisionLower95": 0.8333 - }, - "threshold": 0.965 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.7993, - "falseNegative": 98, - "falsePositive": 11, - "precision": 0.9518, - "predictedPositive": 228, - "recall": 0.6889, - "total": 1485, - "trueNegative": 1159, - "truePositive": 217, - "wilsonPrecisionLower95": 0.9157 - }, - "threshold": 0.92 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7572, - "falseNegative": 117, - "falsePositive": 10, - "precision": 0.9519, - "predictedPositive": 208, - "recall": 0.6286, - "total": 1485, - "trueNegative": 1160, - "truePositive": 198, - "wilsonPrecisionLower95": 0.9138 - }, - "threshold": 0.735 - } - }, - "metrics": { - "f1": 0.797, - "falseNegative": 198, - "falsePositive": 22, - "precision": 0.9515, - "predictedPositive": 454, - "recall": 0.6857, - "total": 2970, - "trueNegative": 2318, - "truePositive": 432, - "wilsonPrecisionLower95": 0.9277 - }, - "threshold": 0.8 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9373, - "falseNegative": 71, - "falsePositive": 46, - "precision": 0.95, - "predictedPositive": 920, - "recall": 0.9249, - "total": 1485, - "trueNegative": 494, - "truePositive": 874, - "wilsonPrecisionLower95": 0.934 - }, - "threshold": 0.79 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8754, - "falseNegative": 179, - "falsePositive": 39, - "precision": 0.9516, - "predictedPositive": 805, - "recall": 0.8106, - "total": 1485, - "trueNegative": 501, - "truePositive": 766, - "wilsonPrecisionLower95": 0.9345 - }, - "threshold": 0.91 - } - }, - "metrics": { - "f1": 0.9105, - "falseNegative": 242, - "falsePositive": 82, - "precision": 0.9526, - "predictedPositive": 1730, - "recall": 0.872, - "total": 2970, - "trueNegative": 998, - "truePositive": 1648, - "wilsonPrecisionLower95": 0.9415 - }, - "threshold": 0.895 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9773, - "falseNegative": 2, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 43, - "recall": 0.9556, - "total": 1485, - "trueNegative": 1440, - "truePositive": 43, - "wilsonPrecisionLower95": 0.918 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9318, - "falseNegative": 4, - "falsePositive": 2, - "precision": 0.9535, - "predictedPositive": 43, - "recall": 0.9111, - "total": 1485, - "trueNegative": 1438, - "truePositive": 41, - "wilsonPrecisionLower95": 0.8454 - }, - "threshold": 0.9 - } - }, - "metrics": { - "f1": 0.9551, - "falseNegative": 5, - "falsePositive": 3, - "precision": 0.9659, - "predictedPositive": 88, - "recall": 0.9444, - "total": 2970, - "trueNegative": 2877, - "truePositive": 85, - "wilsonPrecisionLower95": 0.9045 - }, - "threshold": 0.885 - }, - "task": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8315, - "falseNegative": 96, - "falsePositive": 11, - "precision": 0.96, - "predictedPositive": 275, - "recall": 0.7333, - "total": 1485, - "trueNegative": 1114, - "truePositive": 264, - "wilsonPrecisionLower95": 0.9298 - }, - "threshold": 0.985 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8349, - "falseNegative": 92, - "falsePositive": 14, - "precision": 0.9504, - "predictedPositive": 282, - "recall": 0.7444, - "total": 1485, - "trueNegative": 1111, - "truePositive": 268, - "wilsonPrecisionLower95": 0.9184 - }, - "threshold": 0.845 - } - }, - "metrics": { - "f1": 0.7688, - "falseNegative": 256, - "falsePositive": 23, - "precision": 0.9528, - "predictedPositive": 487, - "recall": 0.6444, - "total": 2970, - "trueNegative": 2227, - "truePositive": 464, - "wilsonPrecisionLower95": 0.9301 - }, - "threshold": 0.975 - } - }, - "trainingRecords": 15345, - "validation": { - "intentsMeeting95Precision": 9, - "macroF1": 0.7269, - "macroPrecision": 0.9659, - "macroRecall": 0.6366, - "minimumPrecision": 0.9507, - "perIntent": { - "blessing": { - "f1": 0.2692, - "falseNegative": 76, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 14, - "recall": 0.1556, - "total": 2970, - "trueNegative": 2880, - "truePositive": 14, - "wilsonPrecisionLower95": 0.7847 - }, - "complaint": { - "f1": 0.9622, - "falseNegative": 16, - "falsePositive": 11, - "precision": 0.969, - "predictedPositive": 355, - "recall": 0.9556, - "total": 2970, - "trueNegative": 2599, - "truePositive": 344, - "wilsonPrecisionLower95": 0.9454 - }, - "confirmationDecision": { - "f1": 0.8365, - "falseNegative": 47, - "falsePositive": 5, - "precision": 0.9638, - "predictedPositive": 138, - "recall": 0.7389, - "total": 2970, - "trueNegative": 2785, - "truePositive": 133, - "wilsonPrecisionLower95": 0.918 - }, - "followUpReminder": { - "f1": 0.4335, - "falseNegative": 259, - "falsePositive": 5, - "precision": 0.9528, - "predictedPositive": 106, - "recall": 0.2806, - "total": 2970, - "trueNegative": 2605, - "truePositive": 101, - "wilsonPrecisionLower95": 0.8943 - }, - "invitation": { - "f1": 0.5669, - "falseNegative": 108, - "falsePositive": 2, - "precision": 0.973, - "predictedPositive": 74, - "recall": 0.4, - "total": 2970, - "trueNegative": 2788, - "truePositive": 72, - "wilsonPrecisionLower95": 0.9067 - }, - "question": { - "f1": 0.7786, - "falseNegative": 215, - "falsePositive": 21, - "precision": 0.9518, - "predictedPositive": 436, - "recall": 0.6587, - "total": 2970, - "trueNegative": 2319, - "truePositive": 415, - "wilsonPrecisionLower95": 0.9275 - }, - "replyableMessage": { - "f1": 0.9073, - "falseNegative": 250, - "falsePositive": 85, - "precision": 0.9507, - "predictedPositive": 1725, - "recall": 0.8677, - "total": 2970, - "trueNegative": 995, - "truePositive": 1640, - "wilsonPrecisionLower95": 0.9395 - }, - "scheduleNegotiation": { - "f1": 0.9545, - "falseNegative": 6, - "falsePositive": 2, - "precision": 0.9767, - "predictedPositive": 86, - "recall": 0.9333, - "total": 2970, - "trueNegative": 2878, - "truePositive": 84, - "wilsonPrecisionLower95": 0.9191 - }, - "task": { - "f1": 0.8332, - "falseNegative": 188, - "falsePositive": 25, - "precision": 0.9551, - "predictedPositive": 557, - "recall": 0.7389, - "total": 2970, - "trueNegative": 2225, - "truePositive": 532, - "wilsonPrecisionLower95": 0.9346 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 4e-06, - "augmentation": "prefix", - "char_max": 6, - "char_min": 3, - "external_weight": 0.79, - "hard_example_weight": 2.0, - "l1_ratio": 0.15, - "max_char_features": 75000, - "max_word_features": 35000, - "min_df": 2, - "pseudo_weight": 0.32999999999999996, - "round": 12, - "word_max": 3 - }, - "featureCount": 94458, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 0 - }, - "complaint": { - "negative": 250, - "positive": 0 - }, - "confirmationDecision": { - "negative": 250, - "positive": 15 - }, - "followUpReminder": { - "negative": 250, - "positive": 0 - }, - "invitation": { - "negative": 250, - "positive": 0 - }, - "question": { - "negative": 232, - "positive": 0 - }, - "replyableMessage": { - "negative": 10, - "positive": 250 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 0 - }, - "task": { - "negative": 250, - "positive": 0 - } - }, - "round": 12, - "selectionObjective": 0.7563, - "syntheticRegression": { - "intentsMeeting95Precision": 5, - "macroF1": 0.6379, - "macroPrecision": 0.9476, - "macroRecall": 0.5322, - "minimumPrecision": 0.8452, - "perIntent": { - "blessing": { - "f1": 0.1053, - "falseNegative": 85, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 5, - "recall": 0.0556, - "total": 2970, - "trueNegative": 2880, - "truePositive": 5, - "wilsonPrecisionLower95": 0.5655 - }, - "complaint": { - "f1": 0.9625, - "falseNegative": 26, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 334, - "recall": 0.9278, - "total": 2970, - "trueNegative": 2610, - "truePositive": 334, - "wilsonPrecisionLower95": 0.9886 - }, - "confirmationDecision": { - "f1": 0.7821, - "falseNegative": 49, - "falsePositive": 24, - "precision": 0.8452, - "predictedPositive": 155, - "recall": 0.7278, - "total": 2970, - "trueNegative": 2766, - "truePositive": 131, - "wilsonPrecisionLower95": 0.7799 - }, - "followUpReminder": { - "f1": 0.6142, - "falseNegative": 200, - "falsePositive": 1, - "precision": 0.9938, - "predictedPositive": 161, - "recall": 0.4444, - "total": 2970, - "trueNegative": 2609, - "truePositive": 160, - "wilsonPrecisionLower95": 0.9657 - }, - "invitation": { - "f1": 0.5992, - "falseNegative": 103, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 77, - "recall": 0.4278, - "total": 2970, - "trueNegative": 2790, - "truePositive": 77, - "wilsonPrecisionLower95": 0.9525 - }, - "question": { - "f1": 0.4319, - "falseNegative": 454, - "falsePositive": 9, - "precision": 0.9514, - "predictedPositive": 185, - "recall": 0.2794, - "total": 2970, - "trueNegative": 2331, - "truePositive": 176, - "wilsonPrecisionLower95": 0.9101 - }, - "replyableMessage": { - "f1": 0.7888, - "falseNegative": 624, - "falsePositive": 54, - "precision": 0.9591, - "predictedPositive": 1320, - "recall": 0.6698, - "total": 2970, - "trueNegative": 1026, - "truePositive": 1266, - "wilsonPrecisionLower95": 0.947 - }, - "scheduleNegotiation": { - "f1": 0.8144, - "falseNegative": 22, - "falsePositive": 9, - "precision": 0.8831, - "predictedPositive": 77, - "recall": 0.7556, - "total": 2970, - "trueNegative": 2871, - "truePositive": 68, - "wilsonPrecisionLower95": 0.7925 - }, - "task": { - "f1": 0.6429, - "falseNegative": 359, - "falsePositive": 42, - "precision": 0.8958, - "predictedPositive": 403, - "recall": 0.5014, - "total": 2970, - "trueNegative": 2208, - "truePositive": 361, - "wilsonPrecisionLower95": 0.8621 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9917, - "falseNegative": 0, - "falsePositive": 3, - "precision": 0.9836, - "predictedPositive": 183, - "recall": 1.0, - "total": 1485, - "trueNegative": 1302, - "truePositive": 180, - "wilsonPrecisionLower95": 0.9529 - }, - "threshold": 0.885 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8974, - "falseNegative": 27, - "falsePositive": 8, - "precision": 0.9503, - "predictedPositive": 161, - "recall": 0.85, - "total": 1485, - "trueNegative": 1297, - "truePositive": 153, - "wilsonPrecisionLower95": 0.905 - }, - "threshold": 0.9 - } - }, - "metrics": { - "f1": 0.9515, - "falseNegative": 17, - "falsePositive": 18, - "precision": 0.9501, - "predictedPositive": 361, - "recall": 0.9528, - "total": 2970, - "trueNegative": 2592, - "truePositive": 343, - "wilsonPrecisionLower95": 0.9226 - }, - "threshold": 0.77 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9944, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 89, - "recall": 0.9889, - "total": 1485, - "trueNegative": 1395, - "truePositive": 89, - "wilsonPrecisionLower95": 0.9586 - }, - "threshold": 0.915 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.4959, - "falseNegative": 60, - "falsePositive": 1, - "precision": 0.9677, - "predictedPositive": 31, - "recall": 0.3333, - "total": 1485, - "trueNegative": 1394, - "truePositive": 30, - "wilsonPrecisionLower95": 0.8381 - }, - "threshold": 0.985 - } - }, - "metrics": { - "f1": 0.8217, - "falseNegative": 51, - "falsePositive": 5, - "precision": 0.9627, - "predictedPositive": 134, - "recall": 0.7167, - "total": 2970, - "trueNegative": 2785, - "truePositive": 129, - "wilsonPrecisionLower95": 0.9156 - }, - "threshold": 0.915 - }, - "followUpReminder": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.5647, - "falseNegative": 108, - "falsePositive": 3, - "precision": 0.96, - "predictedPositive": 75, - "recall": 0.4, - "total": 1485, - "trueNegative": 1302, - "truePositive": 72, - "wilsonPrecisionLower95": 0.8889 - }, - "threshold": 0.995 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6542, - "falseNegative": 158, - "falsePositive": 8, - "precision": 0.9515, - "predictedPositive": 165, - "recall": 0.4984, - "total": 1485, - "trueNegative": 1162, - "truePositive": 157, - "wilsonPrecisionLower95": 0.9073 - }, - "threshold": 0.805 - } - }, - "metrics": { - "f1": 0.5711, - "falseNegative": 373, - "falsePositive": 13, - "precision": 0.9519, - "predictedPositive": 270, - "recall": 0.4079, - "total": 2970, - "trueNegative": 2327, - "truePositive": 257, - "wilsonPrecisionLower95": 0.9194 - }, - "threshold": 0.985 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8956, - "falseNegative": 147, - "falsePositive": 39, - "precision": 0.9534, - "predictedPositive": 837, - "recall": 0.8444, - "total": 1485, - "trueNegative": 501, - "truePositive": 798, - "wilsonPrecisionLower95": 0.9369 - }, - "threshold": 0.975 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6735, - "falseNegative": 453, - "falsePositive": 24, - "precision": 0.9535, - "predictedPositive": 516, - "recall": 0.5206, - "total": 1485, - "trueNegative": 516, - "truePositive": 492, - "wilsonPrecisionLower95": 0.9317 - }, - "threshold": 0.985 - } - }, - "metrics": { - "f1": 0.7866, - "falseNegative": 626, - "falsePositive": 60, - "precision": 0.9547, - "predictedPositive": 1324, - "recall": 0.6688, - "total": 2970, - "trueNegative": 1020, - "truePositive": 1264, - "wilsonPrecisionLower95": 0.9421 - }, - "threshold": 0.985 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9888, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 44, - "recall": 0.9778, - "total": 1485, - "trueNegative": 1440, - "truePositive": 44, - "wilsonPrecisionLower95": 0.9197 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8052, - "falseNegative": 14, - "falsePositive": 1, - "precision": 0.9688, - "predictedPositive": 32, - "recall": 0.6889, - "total": 1485, - "trueNegative": 1439, - "truePositive": 31, - "wilsonPrecisionLower95": 0.8426 - }, - "threshold": 0.94 - } - }, - "metrics": { - "f1": 0.9364, - "falseNegative": 9, - "falsePositive": 2, - "precision": 0.9759, - "predictedPositive": 83, - "recall": 0.9, - "total": 2970, - "trueNegative": 2878, - "truePositive": 81, - "wilsonPrecisionLower95": 0.9163 - }, - "threshold": 0.715 - }, - "task": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8475, - "falseNegative": 85, - "falsePositive": 14, - "precision": 0.9516, - "predictedPositive": 289, - "recall": 0.7639, - "total": 1485, - "trueNegative": 1111, - "truePositive": 275, - "wilsonPrecisionLower95": 0.9203 - }, - "threshold": 0.985 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.501, - "falseNegative": 238, - "falsePositive": 5, - "precision": 0.9606, - "predictedPositive": 127, - "recall": 0.3389, - "total": 1485, - "trueNegative": 1120, - "truePositive": 122, - "wilsonPrecisionLower95": 0.9111 - }, - "threshold": 0.985 - } - }, - "metrics": { - "f1": 0.6989, - "falseNegative": 323, - "falsePositive": 19, - "precision": 0.9543, - "predictedPositive": 416, - "recall": 0.5514, - "total": 2970, - "trueNegative": 2231, - "truePositive": 397, - "wilsonPrecisionLower95": 0.9298 - }, - "threshold": 0.985 - } - }, - "trainingRecords": 15345, - "validation": { - "intentsMeeting95Precision": 8, - "macroF1": 0.6603, - "macroPrecision": 0.9686, - "macroRecall": 0.5413, - "minimumPrecision": 0.9298, - "perIntent": { - "blessing": { - "f1": 0.2857, - "falseNegative": 75, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 15, - "recall": 0.1667, - "total": 2970, - "trueNegative": 2880, - "truePositive": 15, - "wilsonPrecisionLower95": 0.7961 - }, - "complaint": { - "f1": 0.946, - "falseNegative": 27, - "falsePositive": 11, - "precision": 0.968, - "predictedPositive": 344, - "recall": 0.925, - "total": 2970, - "trueNegative": 2599, - "truePositive": 333, - "wilsonPrecisionLower95": 0.9437 - }, - "confirmationDecision": { - "f1": 0.7933, - "falseNegative": 61, - "falsePositive": 1, - "precision": 0.9917, - "predictedPositive": 120, - "recall": 0.6611, - "total": 2970, - "trueNegative": 2789, - "truePositive": 119, - "wilsonPrecisionLower95": 0.9543 - }, - "followUpReminder": { - "f1": 0.6494, - "falseNegative": 185, - "falsePositive": 4, - "precision": 0.9777, - "predictedPositive": 179, - "recall": 0.4861, - "total": 2970, - "trueNegative": 2606, - "truePositive": 175, - "wilsonPrecisionLower95": 0.944 - }, - "invitation": { - "f1": 0.4473, - "falseNegative": 127, - "falsePositive": 4, - "precision": 0.9298, - "predictedPositive": 57, - "recall": 0.2944, - "total": 2970, - "trueNegative": 2786, - "truePositive": 53, - "wilsonPrecisionLower95": 0.833 - }, - "question": { - "f1": 0.4227, - "falseNegative": 459, - "falsePositive": 8, - "precision": 0.9553, - "predictedPositive": 179, - "recall": 0.2714, - "total": 2970, - "trueNegative": 2332, - "truePositive": 171, - "wilsonPrecisionLower95": 0.9143 - }, - "replyableMessage": { - "f1": 0.7956, - "falseNegative": 600, - "falsePositive": 63, - "precision": 0.9534, - "predictedPositive": 1353, - "recall": 0.6825, - "total": 2970, - "trueNegative": 1017, - "truePositive": 1290, - "wilsonPrecisionLower95": 0.9409 - }, - "scheduleNegotiation": { - "f1": 0.9036, - "falseNegative": 15, - "falsePositive": 1, - "precision": 0.9868, - "predictedPositive": 76, - "recall": 0.8333, - "total": 2970, - "trueNegative": 2879, - "truePositive": 75, - "wilsonPrecisionLower95": 0.9292 - }, - "task": { - "f1": 0.6989, - "falseNegative": 323, - "falsePositive": 19, - "precision": 0.9543, - "predictedPositive": 416, - "recall": 0.5514, - "total": 2970, - "trueNegative": 2231, - "truePositive": 397, - "wilsonPrecisionLower95": 0.9298 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 3e-06, - "augmentation": "prefix", - "char_max": 7, - "char_min": 2, - "external_weight": 0.81, - "hard_example_weight": 2.0, - "l1_ratio": 0.15, - "max_char_features": 80000, - "max_word_features": 38000, - "min_df": 2, - "pseudo_weight": 0.345, - "round": 13, - "word_max": 3 - }, - "featureCount": 99458, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 0 - }, - "complaint": { - "negative": 250, - "positive": 1 - }, - "confirmationDecision": { - "negative": 250, - "positive": 18 - }, - "followUpReminder": { - "negative": 250, - "positive": 0 - }, - "invitation": { - "negative": 250, - "positive": 0 - }, - "question": { - "negative": 250, - "positive": 0 - }, - "replyableMessage": { - "negative": 12, - "positive": 250 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 0 - }, - "task": { - "negative": 250, - "positive": 0 - } - }, - "round": 13, - "selectionObjective": 0.768, - "syntheticRegression": { - "intentsMeeting95Precision": 4, - "macroF1": 0.6577, - "macroPrecision": 0.9568, - "macroRecall": 0.5722, - "minimumPrecision": 0.859, - "perIntent": { - "blessing": { - "f1": 0.1443, - "falseNegative": 83, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 7, - "recall": 0.0778, - "total": 2970, - "trueNegative": 2880, - "truePositive": 7, - "wilsonPrecisionLower95": 0.6457 - }, - "complaint": { - "f1": 0.9744, - "falseNegative": 17, - "falsePositive": 1, - "precision": 0.9971, - "predictedPositive": 344, - "recall": 0.9528, - "total": 2970, - "trueNegative": 2609, - "truePositive": 343, - "wilsonPrecisionLower95": 0.9837 - }, - "confirmationDecision": { - "f1": 0.7976, - "falseNegative": 46, - "falsePositive": 22, - "precision": 0.859, - "predictedPositive": 156, - "recall": 0.7444, - "total": 2970, - "trueNegative": 2768, - "truePositive": 134, - "wilsonPrecisionLower95": 0.7957 - }, - "followUpReminder": { - "f1": 0.2972, - "falseNegative": 297, - "falsePositive": 1, - "precision": 0.9844, - "predictedPositive": 64, - "recall": 0.175, - "total": 2970, - "trueNegative": 2609, - "truePositive": 63, - "wilsonPrecisionLower95": 0.9167 - }, - "invitation": { - "f1": 0.6077, - "falseNegative": 101, - "falsePositive": 1, - "precision": 0.9875, - "predictedPositive": 80, - "recall": 0.4389, - "total": 2970, - "trueNegative": 2789, - "truePositive": 79, - "wilsonPrecisionLower95": 0.9325 - }, - "question": { - "f1": 0.5229, - "falseNegative": 407, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 223, - "recall": 0.354, - "total": 2970, - "trueNegative": 2340, - "truePositive": 223, - "wilsonPrecisionLower95": 0.9831 - }, - "replyableMessage": { - "f1": 0.9179, - "falseNegative": 202, - "falsePositive": 100, - "precision": 0.9441, - "predictedPositive": 1788, - "recall": 0.8931, - "total": 2970, - "trueNegative": 980, - "truePositive": 1688, - "wilsonPrecisionLower95": 0.9324 - }, - "scheduleNegotiation": { - "f1": 0.8889, - "falseNegative": 14, - "falsePositive": 5, - "precision": 0.9383, - "predictedPositive": 81, - "recall": 0.8444, - "total": 2970, - "trueNegative": 2875, - "truePositive": 76, - "wilsonPrecisionLower95": 0.8635 - }, - "task": { - "f1": 0.7681, - "falseNegative": 238, - "falsePositive": 53, - "precision": 0.9009, - "predictedPositive": 535, - "recall": 0.6694, - "total": 2970, - "trueNegative": 2197, - "truePositive": 482, - "wilsonPrecisionLower95": 0.8727 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9727, - "falseNegative": 2, - "falsePositive": 8, - "precision": 0.957, - "predictedPositive": 186, - "recall": 0.9889, - "total": 1485, - "trueNegative": 1297, - "truePositive": 178, - "wilsonPrecisionLower95": 0.9174 - }, - "threshold": 0.98 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.95, - "falseNegative": 9, - "falsePositive": 9, - "precision": 0.95, - "predictedPositive": 180, - "recall": 0.95, - "total": 1485, - "trueNegative": 1296, - "truePositive": 171, - "wilsonPrecisionLower95": 0.9077 - }, - "threshold": 0.905 - } - }, - "metrics": { - "f1": 0.9529, - "falseNegative": 16, - "falsePositive": 18, - "precision": 0.9503, - "predictedPositive": 362, - "recall": 0.9556, - "total": 2970, - "trueNegative": 2592, - "truePositive": 344, - "wilsonPrecisionLower95": 0.9228 - }, - "threshold": 0.965 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.828, - "falseNegative": 25, - "falsePositive": 2, - "precision": 0.9701, - "predictedPositive": 67, - "recall": 0.7222, - "total": 1485, - "trueNegative": 1393, - "truePositive": 65, - "wilsonPrecisionLower95": 0.8975 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6763, - "falseNegative": 43, - "falsePositive": 2, - "precision": 0.9592, - "predictedPositive": 49, - "recall": 0.5222, - "total": 1485, - "trueNegative": 1393, - "truePositive": 47, - "wilsonPrecisionLower95": 0.8629 - }, - "threshold": 0.93 - } - }, - "metrics": { - "f1": 0.7855, - "falseNegative": 61, - "falsePositive": 4, - "precision": 0.9675, - "predictedPositive": 123, - "recall": 0.6611, - "total": 2970, - "trueNegative": 2786, - "truePositive": 119, - "wilsonPrecisionLower95": 0.9194 - }, - "threshold": 0.82 - }, - "followUpReminder": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7452, - "falseNegative": 122, - "falsePositive": 10, - "precision": 0.9507, - "predictedPositive": 203, - "recall": 0.6127, - "total": 1485, - "trueNegative": 1160, - "truePositive": 193, - "wilsonPrecisionLower95": 0.9117 - }, - "threshold": 0.915 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9441, - "falseNegative": 59, - "falsePositive": 46, - "precision": 0.9506, - "predictedPositive": 932, - "recall": 0.9376, - "total": 1485, - "trueNegative": 494, - "truePositive": 886, - "wilsonPrecisionLower95": 0.9348 - }, - "threshold": 0.935 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9067, - "falseNegative": 129, - "falsePositive": 39, - "precision": 0.9544, - "predictedPositive": 855, - "recall": 0.8635, - "total": 1485, - "trueNegative": 501, - "truePositive": 816, - "wilsonPrecisionLower95": 0.9383 - }, - "threshold": 0.905 - } - }, - "metrics": { - "f1": 0.9267, - "falseNegative": 182, - "falsePositive": 88, - "precision": 0.951, - "predictedPositive": 1796, - "recall": 0.9037, - "total": 2970, - "trueNegative": 992, - "truePositive": 1708, - "wilsonPrecisionLower95": 0.94 - }, - "threshold": 0.905 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9888, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 44, - "recall": 0.9778, - "total": 1485, - "trueNegative": 1440, - "truePositive": 44, - "wilsonPrecisionLower95": 0.9197 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9655, - "falseNegative": 3, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 42, - "recall": 0.9333, - "total": 1485, - "trueNegative": 1440, - "truePositive": 42, - "wilsonPrecisionLower95": 0.9162 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.9773, - "falseNegative": 4, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 86, - "recall": 0.9556, - "total": 2970, - "trueNegative": 2880, - "truePositive": 86, - "wilsonPrecisionLower95": 0.9572 - }, - "threshold": 0.5 - }, - "task": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.7876, - "falseNegative": 119, - "falsePositive": 11, - "precision": 0.9563, - "predictedPositive": 252, - "recall": 0.6694, - "total": 1485, - "trueNegative": 1114, - "truePositive": 241, - "wilsonPrecisionLower95": 0.9235 - }, - "threshold": 0.995 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8416, - "falseNegative": 89, - "falsePositive": 13, - "precision": 0.9542, - "predictedPositive": 284, - "recall": 0.7528, - "total": 1485, - "trueNegative": 1112, - "truePositive": 271, - "wilsonPrecisionLower95": 0.9233 - }, - "threshold": 0.895 - } - }, - "metrics": { - "f1": 0.6962, - "falseNegative": 327, - "falsePositive": 16, - "precision": 0.9609, - "predictedPositive": 409, - "recall": 0.5458, - "total": 2970, - "trueNegative": 2234, - "truePositive": 393, - "wilsonPrecisionLower95": 0.9374 - }, - "threshold": 0.995 - } - }, - "trainingRecords": 15346, - "validation": { - "intentsMeeting95Precision": 9, - "macroF1": 0.6724, - "macroPrecision": 0.9724, - "macroRecall": 0.5779, - "minimumPrecision": 0.9524, - "perIntent": { - "blessing": { - "f1": 0.2178, - "falseNegative": 79, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 11, - "recall": 0.1222, - "total": 2970, - "trueNegative": 2880, - "truePositive": 11, - "wilsonPrecisionLower95": 0.7412 - }, - "complaint": { - "f1": 0.9614, - "falseNegative": 11, - "falsePositive": 17, - "precision": 0.9536, - "predictedPositive": 366, - "recall": 0.9694, - "total": 2970, - "trueNegative": 2593, - "truePositive": 349, - "wilsonPrecisionLower95": 0.9269 - }, - "confirmationDecision": { - "f1": 0.7568, - "falseNegative": 68, - "falsePositive": 4, - "precision": 0.9655, - "predictedPositive": 116, - "recall": 0.6222, - "total": 2970, - "trueNegative": 2786, - "truePositive": 112, - "wilsonPrecisionLower95": 0.9147 - }, - "followUpReminder": { - "f1": 0.4895, - "falseNegative": 243, - "falsePositive": 1, - "precision": 0.9915, - "predictedPositive": 118, - "recall": 0.325, - "total": 2970, - "trueNegative": 2609, - "truePositive": 117, - "wilsonPrecisionLower95": 0.9536 - }, - "invitation": { - "f1": 0.4192, - "falseNegative": 132, - "falsePositive": 1, - "precision": 0.9796, - "predictedPositive": 49, - "recall": 0.2667, - "total": 2970, - "trueNegative": 2789, - "truePositive": 48, - "wilsonPrecisionLower95": 0.8931 - }, - "question": { - "f1": 0.4888, - "falseNegative": 423, - "falsePositive": 10, - "precision": 0.9539, - "predictedPositive": 217, - "recall": 0.3286, - "total": 2970, - "trueNegative": 2330, - "truePositive": 207, - "wilsonPrecisionLower95": 0.9173 - }, - "replyableMessage": { - "f1": 0.9258, - "falseNegative": 188, - "falsePositive": 85, - "precision": 0.9524, - "predictedPositive": 1787, - "recall": 0.9005, - "total": 2970, - "trueNegative": 995, - "truePositive": 1702, - "wilsonPrecisionLower95": 0.9416 - }, - "scheduleNegotiation": { - "f1": 0.9773, - "falseNegative": 4, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 86, - "recall": 0.9556, - "total": 2970, - "trueNegative": 2880, - "truePositive": 86, - "wilsonPrecisionLower95": 0.9572 - }, - "task": { - "f1": 0.8153, - "falseNegative": 208, - "falsePositive": 24, - "precision": 0.9552, - "predictedPositive": 536, - "recall": 0.7111, - "total": 2970, - "trueNegative": 2226, - "truePositive": 512, - "wilsonPrecisionLower95": 0.9342 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 2.5e-06, - "augmentation": "prefix", - "char_max": 7, - "char_min": 3, - "external_weight": 0.8300000000000001, - "hard_example_weight": 2.2, - "l1_ratio": 0.2, - "max_char_features": 80000, - "max_word_features": 40000, - "min_df": 1, - "pseudo_weight": 0.36, - "round": 14, - "word_max": 3 - }, - "featureCount": 120000, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 0 - }, - "complaint": { - "negative": 250, - "positive": 8 - }, - "confirmationDecision": { - "negative": 250, - "positive": 26 - }, - "followUpReminder": { - "negative": 250, - "positive": 0 - }, - "invitation": { - "negative": 250, - "positive": 0 - }, - "question": { - "negative": 250, - "positive": 0 - }, - "replyableMessage": { - "negative": 34, - "positive": 250 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 0 - }, - "task": { - "negative": 250, - "positive": 3 - } - }, - "round": 14, - "selectionObjective": 0.7391, - "syntheticRegression": { - "intentsMeeting95Precision": 5, - "macroF1": 0.6159, - "macroPrecision": 0.9488, - "macroRecall": 0.5171, - "minimumPrecision": 0.8315, - "perIntent": { - "blessing": { - "f1": 0.1443, - "falseNegative": 83, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 7, - "recall": 0.0778, - "total": 2970, - "trueNegative": 2880, - "truePositive": 7, - "wilsonPrecisionLower95": 0.6457 - }, - "complaint": { - "f1": 0.7987, - "falseNegative": 114, - "falsePositive": 10, - "precision": 0.9609, - "predictedPositive": 256, - "recall": 0.6833, - "total": 2970, - "trueNegative": 2600, - "truePositive": 246, - "wilsonPrecisionLower95": 0.9296 - }, - "confirmationDecision": { - "f1": 0.8407, - "falseNegative": 27, - "falsePositive": 31, - "precision": 0.8315, - "predictedPositive": 184, - "recall": 0.85, - "total": 2970, - "trueNegative": 2759, - "truePositive": 153, - "wilsonPrecisionLower95": 0.7708 - }, - "followUpReminder": { - "f1": 0.7361, - "falseNegative": 148, - "falsePositive": 4, - "precision": 0.9815, - "predictedPositive": 216, - "recall": 0.5889, - "total": 2970, - "trueNegative": 2606, - "truePositive": 212, - "wilsonPrecisionLower95": 0.9534 - }, - "invitation": { - "f1": 0.5591, - "falseNegative": 109, - "falsePositive": 3, - "precision": 0.9595, - "predictedPositive": 74, - "recall": 0.3944, - "total": 2970, - "trueNegative": 2787, - "truePositive": 71, - "wilsonPrecisionLower95": 0.8875 - }, - "question": { - "f1": 0.7917, - "falseNegative": 212, - "falsePositive": 8, - "precision": 0.9812, - "predictedPositive": 426, - "recall": 0.6635, - "total": 2970, - "trueNegative": 2332, - "truePositive": 418, - "wilsonPrecisionLower95": 0.9634 - }, - "replyableMessage": { - "f1": 0.6113, - "falseNegative": 1037, - "falsePositive": 48, - "precision": 0.9467, - "predictedPositive": 901, - "recall": 0.4513, - "total": 2970, - "trueNegative": 1032, - "truePositive": 853, - "wilsonPrecisionLower95": 0.9301 - }, - "scheduleNegotiation": { - "f1": 0.9176, - "falseNegative": 12, - "falsePositive": 2, - "precision": 0.975, - "predictedPositive": 80, - "recall": 0.8667, - "total": 2970, - "trueNegative": 2878, - "truePositive": 78, - "wilsonPrecisionLower95": 0.9134 - }, - "task": { - "f1": 0.1432, - "falseNegative": 664, - "falsePositive": 6, - "precision": 0.9032, - "predictedPositive": 62, - "recall": 0.0778, - "total": 2970, - "trueNegative": 2244, - "truePositive": 56, - "wilsonPrecisionLower95": 0.8045 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9809, - "falseNegative": 0, - "falsePositive": 7, - "precision": 0.9626, - "predictedPositive": 187, - "recall": 1.0, - "total": 1485, - "trueNegative": 1298, - "truePositive": 180, - "wilsonPrecisionLower95": 0.9248 - }, - "threshold": 0.99 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8625, - "falseNegative": 21, - "falsePositive": 1, - "precision": 0.9857, - "predictedPositive": 70, - "recall": 0.7667, - "total": 1485, - "trueNegative": 1394, - "truePositive": 69, - "wilsonPrecisionLower95": 0.9234 - }, - "threshold": 0.905 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.828, - "falseNegative": 25, - "falsePositive": 2, - "precision": 0.9701, - "predictedPositive": 67, - "recall": 0.7222, - "total": 1485, - "trueNegative": 1393, - "truePositive": 65, - "wilsonPrecisionLower95": 0.8975 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.8328, - "falseNegative": 48, - "falsePositive": 5, - "precision": 0.9635, - "predictedPositive": 137, - "recall": 0.7333, - "total": 2970, - "trueNegative": 2785, - "truePositive": 132, - "wilsonPrecisionLower95": 0.9174 - }, - "threshold": 0.685 - }, - "followUpReminder": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8788, - "falseNegative": 35, - "falsePositive": 5, - "precision": 0.9667, - "predictedPositive": 150, - "recall": 0.8056, - "total": 1485, - "trueNegative": 1300, - "truePositive": 145, - "wilsonPrecisionLower95": 0.9243 - }, - "threshold": 0.845 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6113, - "falseNegative": 99, - "falsePositive": 4, - "precision": 0.9529, - "predictedPositive": 85, - "recall": 0.45, - "total": 1485, - "trueNegative": 1301, - "truePositive": 81, - "wilsonPrecisionLower95": 0.8852 - }, - "threshold": 0.97 - } - }, - "metrics": { - "f1": 0.7682, - "falseNegative": 128, - "falsePositive": 12, - "precision": 0.9508, - "predictedPositive": 244, - "recall": 0.6444, - "total": 2970, - "trueNegative": 2598, - "truePositive": 232, - "wilsonPrecisionLower95": 0.916 - }, - "threshold": 0.91 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.847, - "falseNegative": 77, - "falsePositive": 9, - "precision": 0.9636, - "predictedPositive": 247, - "recall": 0.7556, - "total": 1485, - "trueNegative": 1161, - "truePositive": 238, - "wilsonPrecisionLower95": 0.9322 - }, - "threshold": 0.995 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7168, - "falseNegative": 134, - "falsePositive": 9, - "precision": 0.9526, - "predictedPositive": 190, - "recall": 0.5746, - "total": 1485, - "trueNegative": 1161, - "truePositive": 181, - "wilsonPrecisionLower95": 0.9124 - }, - "threshold": 0.865 - } - }, - "metrics": { - "f1": 0.7524, - "falseNegative": 238, - "falsePositive": 20, - "precision": 0.9515, - "predictedPositive": 412, - "recall": 0.6222, - "total": 2970, - "trueNegative": 2320, - "truePositive": 392, - "wilsonPrecisionLower95": 0.9262 - }, - "threshold": 0.975 - }, - "replyableMessage": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9349, - "falseNegative": 76, - "falsePositive": 45, - "precision": 0.9508, - "predictedPositive": 914, - "recall": 0.9196, - "total": 1485, - "trueNegative": 495, - "truePositive": 869, - "wilsonPrecisionLower95": 0.9348 - }, - "threshold": 0.955 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 45, - "recall": 1.0, - "total": 1485, - "trueNegative": 1440, - "truePositive": 45, - "wilsonPrecisionLower95": 0.9213 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.967, - "falseNegative": 1, - "falsePositive": 2, - "precision": 0.9565, - "predictedPositive": 46, - "recall": 0.9778, - "total": 1485, - "trueNegative": 1438, - "truePositive": 44, - "wilsonPrecisionLower95": 0.8547 - }, - "threshold": 0.995 - } - }, - "metrics": { - "f1": 0.9727, - "falseNegative": 1, - "falsePositive": 4, - "precision": 0.957, - "predictedPositive": 93, - "recall": 0.9889, - "total": 2970, - "trueNegative": 2876, - "truePositive": 89, - "wilsonPrecisionLower95": 0.8946 - }, - "threshold": 0.64 - }, - "task": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "trainingRecords": 15345, - "validation": { - "intentsMeeting95Precision": 8, - "macroF1": 0.6332, - "macroPrecision": 0.9668, - "macroRecall": 0.5246, - "minimumPrecision": 0.9, - "perIntent": { - "blessing": { - "f1": 0.3019, - "falseNegative": 74, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 16, - "recall": 0.1778, - "total": 2970, - "trueNegative": 2880, - "truePositive": 16, - "wilsonPrecisionLower95": 0.8064 - }, - "complaint": { - "f1": 0.8199, - "falseNegative": 105, - "falsePositive": 7, - "precision": 0.9733, - "predictedPositive": 262, - "recall": 0.7083, - "total": 2970, - "trueNegative": 2603, - "truePositive": 255, - "wilsonPrecisionLower95": 0.9459 - }, - "confirmationDecision": { - "f1": 0.8454, - "falseNegative": 46, - "falsePositive": 3, - "precision": 0.9781, - "predictedPositive": 137, - "recall": 0.7444, - "total": 2970, - "trueNegative": 2787, - "truePositive": 134, - "wilsonPrecisionLower95": 0.9376 - }, - "followUpReminder": { - "f1": 0.7597, - "falseNegative": 134, - "falsePositive": 9, - "precision": 0.9617, - "predictedPositive": 235, - "recall": 0.6278, - "total": 2970, - "trueNegative": 2601, - "truePositive": 226, - "wilsonPrecisionLower95": 0.9288 - }, - "invitation": { - "f1": 0.3273, - "falseNegative": 144, - "falsePositive": 4, - "precision": 0.9, - "predictedPositive": 40, - "recall": 0.2, - "total": 2970, - "trueNegative": 2786, - "truePositive": 36, - "wilsonPrecisionLower95": 0.7695 - }, - "question": { - "f1": 0.7854, - "falseNegative": 211, - "falsePositive": 18, - "precision": 0.9588, - "predictedPositive": 437, - "recall": 0.6651, - "total": 2970, - "trueNegative": 2322, - "truePositive": 419, - "wilsonPrecisionLower95": 0.9358 - }, - "replyableMessage": { - "f1": 0.6272, - "falseNegative": 1006, - "falsePositive": 45, - "precision": 0.9516, - "predictedPositive": 929, - "recall": 0.4677, - "total": 2970, - "trueNegative": 1035, - "truePositive": 884, - "wilsonPrecisionLower95": 0.9358 - }, - "scheduleNegotiation": { - "f1": 0.9834, - "falseNegative": 1, - "falsePositive": 2, - "precision": 0.978, - "predictedPositive": 91, - "recall": 0.9889, - "total": 2970, - "trueNegative": 2878, - "truePositive": 89, - "wilsonPrecisionLower95": 0.9234 - }, - "task": { - "f1": 0.2482, - "falseNegative": 618, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 102, - "recall": 0.1417, - "total": 2970, - "trueNegative": 2250, - "truePositive": 102, - "wilsonPrecisionLower95": 0.9637 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 3.5e-06, - "augmentation": "prefix", - "char_max": 6, - "char_min": 2, - "external_weight": 0.8500000000000001, - "hard_example_weight": 2.2, - "l1_ratio": 0.2, - "max_char_features": 85000, - "max_word_features": 42000, - "min_df": 1, - "pseudo_weight": 0.375, - "round": 15, - "word_max": 3 - }, - "featureCount": 127000, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 1 - }, - "complaint": { - "negative": 250, - "positive": 8 - }, - "confirmationDecision": { - "negative": 250, - "positive": 30 - }, - "followUpReminder": { - "negative": 250, - "positive": 0 - }, - "invitation": { - "negative": 250, - "positive": 0 - }, - "question": { - "negative": 250, - "positive": 0 - }, - "replyableMessage": { - "negative": 39, - "positive": 250 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 0 - }, - "task": { - "negative": 250, - "positive": 0 - } - }, - "round": 15, - "selectionObjective": 0.7896, - "syntheticRegression": { - "intentsMeeting95Precision": 7, - "macroF1": 0.7019, - "macroPrecision": 0.9601, - "macroRecall": 0.6059, - "minimumPrecision": 0.8634, - "perIntent": { - "blessing": { - "f1": 0.2, - "falseNegative": 80, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 10, - "recall": 0.1111, - "total": 2970, - "trueNegative": 2880, - "truePositive": 10, - "wilsonPrecisionLower95": 0.7225 - }, - "complaint": { - "f1": 0.9641, - "falseNegative": 24, - "falsePositive": 1, - "precision": 0.997, - "predictedPositive": 337, - "recall": 0.9333, - "total": 2970, - "trueNegative": 2609, - "truePositive": 336, - "wilsonPrecisionLower95": 0.9834 - }, - "confirmationDecision": { - "f1": 0.8152, - "falseNegative": 41, - "falsePositive": 22, - "precision": 0.8634, - "predictedPositive": 161, - "recall": 0.7722, - "total": 2970, - "trueNegative": 2768, - "truePositive": 139, - "wilsonPrecisionLower95": 0.8018 - }, - "followUpReminder": { - "f1": 0.6115, - "falseNegative": 201, - "falsePositive": 1, - "precision": 0.9938, - "predictedPositive": 160, - "recall": 0.4417, - "total": 2970, - "trueNegative": 2609, - "truePositive": 159, - "wilsonPrecisionLower95": 0.9655 - }, - "invitation": { - "f1": 0.6084, - "falseNegative": 100, - "falsePositive": 3, - "precision": 0.9639, - "predictedPositive": 83, - "recall": 0.4444, - "total": 2970, - "trueNegative": 2787, - "truePositive": 80, - "wilsonPrecisionLower95": 0.899 - }, - "question": { - "f1": 0.5421, - "falseNegative": 395, - "falsePositive": 2, - "precision": 0.9916, - "predictedPositive": 237, - "recall": 0.373, - "total": 2970, - "trueNegative": 2338, - "truePositive": 235, - "wilsonPrecisionLower95": 0.9698 - }, - "replyableMessage": { - "f1": 0.862, - "falseNegative": 410, - "falsePositive": 64, - "precision": 0.9585, - "predictedPositive": 1544, - "recall": 0.7831, - "total": 2970, - "trueNegative": 1016, - "truePositive": 1480, - "wilsonPrecisionLower95": 0.9474 - }, - "scheduleNegotiation": { - "f1": 0.9379, - "falseNegative": 7, - "falsePositive": 4, - "precision": 0.954, - "predictedPositive": 87, - "recall": 0.9222, - "total": 2970, - "trueNegative": 2876, - "truePositive": 83, - "wilsonPrecisionLower95": 0.8877 - }, - "task": { - "f1": 0.7763, - "falseNegative": 236, - "falsePositive": 43, - "precision": 0.9184, - "predictedPositive": 527, - "recall": 0.6722, - "total": 2970, - "trueNegative": 2207, - "truePositive": 484, - "wilsonPrecisionLower95": 0.8919 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9836, - "falseNegative": 0, - "falsePositive": 6, - "precision": 0.9677, - "predictedPositive": 186, - "recall": 1.0, - "total": 1485, - "trueNegative": 1299, - "truePositive": 180, - "wilsonPrecisionLower95": 0.9314 - }, - "threshold": 0.945 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.928, - "falseNegative": 19, - "falsePositive": 6, - "precision": 0.9641, - "predictedPositive": 167, - "recall": 0.8944, - "total": 1485, - "trueNegative": 1299, - "truePositive": 161, - "wilsonPrecisionLower95": 0.9238 - }, - "threshold": 0.77 - } - }, - "metrics": { - "f1": 0.9365, - "falseNegative": 28, - "falsePositive": 17, - "precision": 0.9513, - "predictedPositive": 349, - "recall": 0.9222, - "total": 2970, - "trueNegative": 2593, - "truePositive": 332, - "wilsonPrecisionLower95": 0.9234 - }, - "threshold": 0.88 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.954, - "falseNegative": 7, - "falsePositive": 1, - "precision": 0.9881, - "predictedPositive": 84, - "recall": 0.9222, - "total": 1485, - "trueNegative": 1394, - "truePositive": 83, - "wilsonPrecisionLower95": 0.9356 - }, - "threshold": 0.85 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6906, - "falseNegative": 42, - "falsePositive": 1, - "precision": 0.9796, - "predictedPositive": 49, - "recall": 0.5333, - "total": 1485, - "trueNegative": 1394, - "truePositive": 48, - "wilsonPrecisionLower95": 0.8931 - }, - "threshold": 0.985 - } - }, - "metrics": { - "f1": 0.8598, - "falseNegative": 39, - "falsePositive": 7, - "precision": 0.9527, - "predictedPositive": 148, - "recall": 0.7833, - "total": 2970, - "trueNegative": 2783, - "truePositive": 141, - "wilsonPrecisionLower95": 0.9056 - }, - "threshold": 0.935 - }, - "followUpReminder": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8328, - "falseNegative": 48, - "falsePositive": 5, - "precision": 0.9635, - "predictedPositive": 137, - "recall": 0.7333, - "total": 1485, - "trueNegative": 1300, - "truePositive": 132, - "wilsonPrecisionLower95": 0.9174 - }, - "threshold": 0.99 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9133, - "falseNegative": 11, - "falsePositive": 4, - "precision": 0.9518, - "predictedPositive": 83, - "recall": 0.8778, - "total": 1485, - "trueNegative": 1391, - "truePositive": 79, - "wilsonPrecisionLower95": 0.8825 - }, - "threshold": 0.97 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7548, - "falseNegative": 118, - "falsePositive": 10, - "precision": 0.9517, - "predictedPositive": 207, - "recall": 0.6254, - "total": 1485, - "trueNegative": 1160, - "truePositive": 197, - "wilsonPrecisionLower95": 0.9134 - }, - "threshold": 0.8 - } - }, - "metrics": { - "f1": 0.7678, - "falseNegative": 225, - "falsePositive": 20, - "precision": 0.9529, - "predictedPositive": 425, - "recall": 0.6429, - "total": 2970, - "trueNegative": 2320, - "truePositive": 405, - "wilsonPrecisionLower95": 0.9284 - }, - "threshold": 0.985 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.907, - "falseNegative": 126, - "falsePositive": 42, - "precision": 0.9512, - "predictedPositive": 861, - "recall": 0.8667, - "total": 1485, - "trueNegative": 498, - "truePositive": 819, - "wilsonPrecisionLower95": 0.9347 - }, - "threshold": 0.98 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8252, - "falseNegative": 258, - "falsePositive": 33, - "precision": 0.9542, - "predictedPositive": 720, - "recall": 0.727, - "total": 1485, - "trueNegative": 507, - "truePositive": 687, - "wilsonPrecisionLower95": 0.9363 - }, - "threshold": 0.96 - } - }, - "metrics": { - "f1": 0.8678, - "falseNegative": 384, - "falsePositive": 75, - "precision": 0.9526, - "predictedPositive": 1581, - "recall": 0.7968, - "total": 2970, - "trueNegative": 1005, - "truePositive": 1506, - "wilsonPrecisionLower95": 0.9409 - }, - "threshold": 0.965 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9157, - "falseNegative": 7, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 38, - "recall": 0.8444, - "total": 1485, - "trueNegative": 1440, - "truePositive": 38, - "wilsonPrecisionLower95": 0.9082 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9556, - "falseNegative": 2, - "falsePositive": 2, - "precision": 0.9556, - "predictedPositive": 45, - "recall": 0.9556, - "total": 1485, - "trueNegative": 1438, - "truePositive": 43, - "wilsonPrecisionLower95": 0.8517 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.9364, - "falseNegative": 9, - "falsePositive": 2, - "precision": 0.9759, - "predictedPositive": 83, - "recall": 0.9, - "total": 2970, - "trueNegative": 2878, - "truePositive": 81, - "wilsonPrecisionLower95": 0.9163 - }, - "threshold": 0.5 - }, - "task": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8593, - "falseNegative": 79, - "falsePositive": 13, - "precision": 0.9558, - "predictedPositive": 294, - "recall": 0.7806, - "total": 1485, - "trueNegative": 1112, - "truePositive": 281, - "wilsonPrecisionLower95": 0.9258 - }, - "threshold": 0.98 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8166, - "falseNegative": 104, - "falsePositive": 11, - "precision": 0.9588, - "predictedPositive": 267, - "recall": 0.7111, - "total": 1485, - "trueNegative": 1114, - "truePositive": 256, - "wilsonPrecisionLower95": 0.9277 - }, - "threshold": 0.96 - } - }, - "metrics": { - "f1": 0.8434, - "falseNegative": 176, - "falsePositive": 26, - "precision": 0.9544, - "predictedPositive": 570, - "recall": 0.7556, - "total": 2970, - "trueNegative": 2224, - "truePositive": 544, - "wilsonPrecisionLower95": 0.934 - }, - "threshold": 0.965 - } - }, - "trainingRecords": 15345, - "validation": { - "intentsMeeting95Precision": 8, - "macroF1": 0.711, - "macroPrecision": 0.9652, - "macroRecall": 0.6071, - "minimumPrecision": 0.9441, - "perIntent": { - "blessing": { - "f1": 0.3333, - "falseNegative": 72, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 18, - "recall": 0.2, - "total": 2970, - "trueNegative": 2880, - "truePositive": 18, - "wilsonPrecisionLower95": 0.8241 - }, - "complaint": { - "f1": 0.9565, - "falseNegative": 19, - "falsePositive": 12, - "precision": 0.966, - "predictedPositive": 353, - "recall": 0.9472, - "total": 2970, - "trueNegative": 2598, - "truePositive": 341, - "wilsonPrecisionLower95": 0.9415 - }, - "confirmationDecision": { - "f1": 0.8371, - "falseNegative": 49, - "falsePositive": 2, - "precision": 0.985, - "predictedPositive": 133, - "recall": 0.7278, - "total": 2970, - "trueNegative": 2788, - "truePositive": 131, - "wilsonPrecisionLower95": 0.9468 - }, - "followUpReminder": { - "f1": 0.5368, - "falseNegative": 225, - "falsePositive": 8, - "precision": 0.9441, - "predictedPositive": 143, - "recall": 0.375, - "total": 2970, - "trueNegative": 2602, - "truePositive": 135, - "wilsonPrecisionLower95": 0.8935 - }, - "invitation": { - "f1": 0.6061, - "falseNegative": 100, - "falsePositive": 4, - "precision": 0.9524, - "predictedPositive": 84, - "recall": 0.4444, - "total": 2970, - "trueNegative": 2786, - "truePositive": 80, - "wilsonPrecisionLower95": 0.8839 - }, - "question": { - "f1": 0.487, - "falseNegative": 424, - "falsePositive": 10, - "precision": 0.9537, - "predictedPositive": 216, - "recall": 0.327, - "total": 2970, - "trueNegative": 2330, - "truePositive": 206, - "wilsonPrecisionLower95": 0.9169 - }, - "replyableMessage": { - "f1": 0.8678, - "falseNegative": 384, - "falsePositive": 75, - "precision": 0.9526, - "predictedPositive": 1581, - "recall": 0.7968, - "total": 2970, - "trueNegative": 1005, - "truePositive": 1506, - "wilsonPrecisionLower95": 0.9409 - }, - "scheduleNegotiation": { - "f1": 0.9364, - "falseNegative": 9, - "falsePositive": 2, - "precision": 0.9759, - "predictedPositive": 83, - "recall": 0.9, - "total": 2970, - "trueNegative": 2878, - "truePositive": 81, - "wilsonPrecisionLower95": 0.9163 - }, - "task": { - "f1": 0.8384, - "falseNegative": 183, - "falsePositive": 24, - "precision": 0.9572, - "predictedPositive": 561, - "recall": 0.7458, - "total": 2970, - "trueNegative": 2226, - "truePositive": 537, - "wilsonPrecisionLower95": 0.9371 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 3e-06, - "augmentation": "numeric", - "char_max": 7, - "char_min": 2, - "external_weight": 0.8700000000000001, - "hard_example_weight": 2.3, - "l1_ratio": 0.15, - "max_char_features": 80000, - "max_word_features": 38000, - "min_df": 2, - "pseudo_weight": 0.39, - "round": 16, - "word_max": 3 - }, - "featureCount": 100252, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 3 - }, - "complaint": { - "negative": 250, - "positive": 11 - }, - "confirmationDecision": { - "negative": 250, - "positive": 25 - }, - "followUpReminder": { - "negative": 250, - "positive": 0 - }, - "invitation": { - "negative": 250, - "positive": 0 - }, - "question": { - "negative": 250, - "positive": 0 - }, - "replyableMessage": { - "negative": 76, - "positive": 250 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 0 - }, - "task": { - "negative": 250, - "positive": 0 - } - }, - "round": 16, - "selectionObjective": 0.8105, - "syntheticRegression": { - "intentsMeeting95Precision": 5, - "macroF1": 0.726, - "macroPrecision": 0.9485, - "macroRecall": 0.6388, - "minimumPrecision": 0.8195, - "perIntent": { - "blessing": { - "f1": 0.125, - "falseNegative": 84, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 6, - "recall": 0.0667, - "total": 2970, - "trueNegative": 2880, - "truePositive": 6, - "wilsonPrecisionLower95": 0.6097 - }, - "complaint": { - "f1": 0.9132, - "falseNegative": 55, - "falsePositive": 3, - "precision": 0.9903, - "predictedPositive": 308, - "recall": 0.8472, - "total": 2970, - "trueNegative": 2607, - "truePositive": 305, - "wilsonPrecisionLower95": 0.9718 - }, - "confirmationDecision": { - "f1": 0.6965, - "falseNegative": 71, - "falsePositive": 24, - "precision": 0.8195, - "predictedPositive": 133, - "recall": 0.6056, - "total": 2970, - "trueNegative": 2766, - "truePositive": 109, - "wilsonPrecisionLower95": 0.7455 - }, - "followUpReminder": { - "f1": 0.7409, - "falseNegative": 147, - "falsePositive": 2, - "precision": 0.9907, - "predictedPositive": 215, - "recall": 0.5917, - "total": 2970, - "trueNegative": 2608, - "truePositive": 213, - "wilsonPrecisionLower95": 0.9667 - }, - "invitation": { - "f1": 0.6264, - "falseNegative": 97, - "falsePositive": 2, - "precision": 0.9765, - "predictedPositive": 85, - "recall": 0.4611, - "total": 2970, - "trueNegative": 2788, - "truePositive": 83, - "wilsonPrecisionLower95": 0.9182 - }, - "question": { - "f1": 0.8457, - "falseNegative": 164, - "falsePositive": 6, - "precision": 0.9873, - "predictedPositive": 472, - "recall": 0.7397, - "total": 2970, - "trueNegative": 2334, - "truePositive": 466, - "wilsonPrecisionLower95": 0.9725 - }, - "replyableMessage": { - "f1": 0.8725, - "falseNegative": 371, - "falsePositive": 73, - "precision": 0.9541, - "predictedPositive": 1592, - "recall": 0.8037, - "total": 2970, - "trueNegative": 1007, - "truePositive": 1519, - "wilsonPrecisionLower95": 0.9427 - }, - "scheduleNegotiation": { - "f1": 0.9061, - "falseNegative": 8, - "falsePositive": 9, - "precision": 0.9011, - "predictedPositive": 91, - "recall": 0.9111, - "total": 2970, - "trueNegative": 2871, - "truePositive": 82, - "wilsonPrecisionLower95": 0.8226 - }, - "task": { - "f1": 0.8081, - "falseNegative": 200, - "falsePositive": 47, - "precision": 0.9171, - "predictedPositive": 567, - "recall": 0.7222, - "total": 2970, - "trueNegative": 2203, - "truePositive": 520, - "wilsonPrecisionLower95": 0.8915 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9371, - "falseNegative": 16, - "falsePositive": 6, - "precision": 0.9647, - "predictedPositive": 170, - "recall": 0.9111, - "total": 1485, - "trueNegative": 1299, - "truePositive": 164, - "wilsonPrecisionLower95": 0.9251 - }, - "threshold": 0.995 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8836, - "falseNegative": 32, - "falsePositive": 7, - "precision": 0.9548, - "predictedPositive": 155, - "recall": 0.8222, - "total": 1485, - "trueNegative": 1298, - "truePositive": 148, - "wilsonPrecisionLower95": 0.9097 - }, - "threshold": 0.99 - } - }, - "metrics": { - "f1": 0.8994, - "falseNegative": 56, - "falsePositive": 12, - "precision": 0.962, - "predictedPositive": 316, - "recall": 0.8444, - "total": 2970, - "trueNegative": 2598, - "truePositive": 304, - "wilsonPrecisionLower95": 0.9348 - }, - "threshold": 0.995 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8625, - "falseNegative": 21, - "falsePositive": 1, - "precision": 0.9857, - "predictedPositive": 70, - "recall": 0.7667, - "total": 1485, - "trueNegative": 1394, - "truePositive": 69, - "wilsonPrecisionLower95": 0.9234 - }, - "threshold": 0.915 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.7415, - "falseNegative": 71, - "falsePositive": 5, - "precision": 0.9561, - "predictedPositive": 114, - "recall": 0.6056, - "total": 2970, - "trueNegative": 2785, - "truePositive": 109, - "wilsonPrecisionLower95": 0.9014 - }, - "threshold": 0.96 - }, - "followUpReminder": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8667, - "falseNegative": 37, - "falsePositive": 7, - "precision": 0.9533, - "predictedPositive": 150, - "recall": 0.7944, - "total": 1485, - "trueNegative": 1298, - "truePositive": 143, - "wilsonPrecisionLower95": 0.9068 - }, - "threshold": 0.96 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6031, - "falseNegative": 101, - "falsePositive": 3, - "precision": 0.9634, - "predictedPositive": 82, - "recall": 0.4389, - "total": 1485, - "trueNegative": 1302, - "truePositive": 79, - "wilsonPrecisionLower95": 0.8979 - }, - "threshold": 0.97 - } - }, - "metrics": { - "f1": 0.7492, - "falseNegative": 139, - "falsePositive": 9, - "precision": 0.9609, - "predictedPositive": 230, - "recall": 0.6139, - "total": 2970, - "trueNegative": 2601, - "truePositive": 221, - "wilsonPrecisionLower95": 0.9273 - }, - "threshold": 0.97 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.3604, - "falseNegative": 70, - "falsePositive": 1, - "precision": 0.9524, - "predictedPositive": 21, - "recall": 0.2222, - "total": 1485, - "trueNegative": 1394, - "truePositive": 20, - "wilsonPrecisionLower95": 0.7733 - }, - "threshold": 0.965 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8273, - "falseNegative": 85, - "falsePositive": 11, - "precision": 0.9544, - "predictedPositive": 241, - "recall": 0.7302, - "total": 1485, - "trueNegative": 1159, - "truePositive": 230, - "wilsonPrecisionLower95": 0.9201 - }, - "threshold": 0.985 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.797, - "falseNegative": 99, - "falsePositive": 11, - "precision": 0.9515, - "predictedPositive": 227, - "recall": 0.6857, - "total": 1485, - "trueNegative": 1159, - "truePositive": 216, - "wilsonPrecisionLower95": 0.9153 - }, - "threshold": 0.835 - } - }, - "metrics": { - "f1": 0.8051, - "falseNegative": 190, - "falsePositive": 23, - "precision": 0.9503, - "predictedPositive": 463, - "recall": 0.6984, - "total": 2970, - "trueNegative": 2317, - "truePositive": 440, - "wilsonPrecisionLower95": 0.9266 - }, - "threshold": 0.94 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9343, - "falseNegative": 77, - "falsePositive": 45, - "precision": 0.9507, - "predictedPositive": 913, - "recall": 0.9185, - "total": 1485, - "trueNegative": 495, - "truePositive": 868, - "wilsonPrecisionLower95": 0.9347 - }, - "threshold": 0.91 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8207, - "falseNegative": 263, - "falsePositive": 35, - "precision": 0.9512, - "predictedPositive": 717, - "recall": 0.7217, - "total": 1485, - "trueNegative": 505, - "truePositive": 682, - "wilsonPrecisionLower95": 0.9329 - }, - "threshold": 0.98 - } - }, - "metrics": { - "f1": 0.8572, - "falseNegative": 419, - "falsePositive": 71, - "precision": 0.954, - "predictedPositive": 1542, - "recall": 0.7783, - "total": 2970, - "trueNegative": 1009, - "truePositive": 1471, - "wilsonPrecisionLower95": 0.9423 - }, - "threshold": 0.98 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9888, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 44, - "recall": 0.9778, - "total": 1485, - "trueNegative": 1440, - "truePositive": 44, - "wilsonPrecisionLower95": 0.9197 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.989, - "falseNegative": 0, - "falsePositive": 1, - "precision": 0.9783, - "predictedPositive": 46, - "recall": 1.0, - "total": 1485, - "trueNegative": 1439, - "truePositive": 45, - "wilsonPrecisionLower95": 0.8866 - }, - "threshold": 0.955 - } - }, - "metrics": { - "f1": 0.9778, - "falseNegative": 2, - "falsePositive": 2, - "precision": 0.9778, - "predictedPositive": 90, - "recall": 0.9778, - "total": 2970, - "trueNegative": 2878, - "truePositive": 88, - "wilsonPrecisionLower95": 0.9226 - }, - "threshold": 0.935 - }, - "task": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8836, - "falseNegative": 64, - "falsePositive": 14, - "precision": 0.9548, - "predictedPositive": 310, - "recall": 0.8222, - "total": 1485, - "trueNegative": 1111, - "truePositive": 296, - "wilsonPrecisionLower95": 0.9256 - }, - "threshold": 0.975 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7896, - "falseNegative": 118, - "falsePositive": 11, - "precision": 0.9565, - "predictedPositive": 253, - "recall": 0.6722, - "total": 1485, - "trueNegative": 1114, - "truePositive": 242, - "wilsonPrecisionLower95": 0.9238 - }, - "threshold": 0.96 - } - }, - "metrics": { - "f1": 0.8316, - "falseNegative": 189, - "falsePositive": 26, - "precision": 0.9533, - "predictedPositive": 557, - "recall": 0.7375, - "total": 2970, - "trueNegative": 2224, - "truePositive": 531, - "wilsonPrecisionLower95": 0.9325 - }, - "threshold": 0.97 - } - }, - "trainingRecords": 16726, - "validation": { - "intentsMeeting95Precision": 9, - "macroF1": 0.7397, - "macroPrecision": 0.9707, - "macroRecall": 0.6388, - "minimumPrecision": 0.9509, - "perIntent": { - "blessing": { - "f1": 0.3636, - "falseNegative": 70, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 20, - "recall": 0.2222, - "total": 2970, - "trueNegative": 2880, - "truePositive": 20, - "wilsonPrecisionLower95": 0.8389 - }, - "complaint": { - "f1": 0.9109, - "falseNegative": 48, - "falsePositive": 13, - "precision": 0.96, - "predictedPositive": 325, - "recall": 0.8667, - "total": 2970, - "trueNegative": 2597, - "truePositive": 312, - "wilsonPrecisionLower95": 0.9328 - }, - "confirmationDecision": { - "f1": 0.6236, - "falseNegative": 98, - "falsePositive": 1, - "precision": 0.988, - "predictedPositive": 83, - "recall": 0.4556, - "total": 2970, - "trueNegative": 2789, - "truePositive": 82, - "wilsonPrecisionLower95": 0.9349 - }, - "followUpReminder": { - "f1": 0.75, - "falseNegative": 138, - "falsePositive": 10, - "precision": 0.9569, - "predictedPositive": 232, - "recall": 0.6167, - "total": 2970, - "trueNegative": 2600, - "truePositive": 222, - "wilsonPrecisionLower95": 0.9225 - }, - "invitation": { - "f1": 0.4917, - "falseNegative": 121, - "falsePositive": 1, - "precision": 0.9833, - "predictedPositive": 60, - "recall": 0.3278, - "total": 2970, - "trueNegative": 2789, - "truePositive": 59, - "wilsonPrecisionLower95": 0.9114 - }, - "question": { - "f1": 0.8124, - "falseNegative": 184, - "falsePositive": 22, - "precision": 0.953, - "predictedPositive": 468, - "recall": 0.7079, - "total": 2970, - "trueNegative": 2318, - "truePositive": 446, - "wilsonPrecisionLower95": 0.9299 - }, - "replyableMessage": { - "f1": 0.8807, - "falseNegative": 340, - "falsePositive": 80, - "precision": 0.9509, - "predictedPositive": 1630, - "recall": 0.8201, - "total": 2970, - "trueNegative": 1000, - "truePositive": 1550, - "wilsonPrecisionLower95": 0.9393 - }, - "scheduleNegotiation": { - "f1": 0.9889, - "falseNegative": 1, - "falsePositive": 1, - "precision": 0.9889, - "predictedPositive": 90, - "recall": 0.9889, - "total": 2970, - "trueNegative": 2879, - "truePositive": 89, - "wilsonPrecisionLower95": 0.9397 - }, - "task": { - "f1": 0.8359, - "falseNegative": 185, - "falsePositive": 25, - "precision": 0.9554, - "predictedPositive": 560, - "recall": 0.7431, - "total": 2970, - "trueNegative": 2225, - "truePositive": 535, - "wilsonPrecisionLower95": 0.9349 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 2e-06, - "augmentation": "numeric", - "char_max": 7, - "char_min": 3, - "external_weight": 0.8900000000000001, - "hard_example_weight": 2.5, - "l1_ratio": 0.2, - "max_char_features": 85000, - "max_word_features": 42000, - "min_df": 1, - "pseudo_weight": 0.405, - "round": 17, - "word_max": 3 - }, - "featureCount": 127000, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 1 - }, - "complaint": { - "negative": 250, - "positive": 4 - }, - "confirmationDecision": { - "negative": 250, - "positive": 23 - }, - "followUpReminder": { - "negative": 250, - "positive": 3 - }, - "invitation": { - "negative": 250, - "positive": 0 - }, - "question": { - "negative": 250, - "positive": 0 - }, - "replyableMessage": { - "negative": 70, - "positive": 250 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 0 - }, - "task": { - "negative": 250, - "positive": 0 - } - }, - "round": 17, - "selectionObjective": 0.6932, - "syntheticRegression": { - "intentsMeeting95Precision": 4, - "macroF1": 0.5086, - "macroPrecision": 0.9202, - "macroRecall": 0.4091, - "minimumPrecision": 0.7329, - "perIntent": { - "blessing": { - "f1": 0.1633, - "falseNegative": 82, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 8, - "recall": 0.0889, - "total": 2970, - "trueNegative": 2880, - "truePositive": 8, - "wilsonPrecisionLower95": 0.6756 - }, - "complaint": { - "f1": 0.914, - "falseNegative": 36, - "falsePositive": 25, - "precision": 0.9284, - "predictedPositive": 349, - "recall": 0.9, - "total": 2970, - "trueNegative": 2585, - "truePositive": 324, - "wilsonPrecisionLower95": 0.8964 - }, - "confirmationDecision": { - "f1": 0.6564, - "falseNegative": 73, - "falsePositive": 39, - "precision": 0.7329, - "predictedPositive": 146, - "recall": 0.5944, - "total": 2970, - "trueNegative": 2751, - "truePositive": 107, - "wilsonPrecisionLower95": 0.6558 - }, - "followUpReminder": { - "f1": 0.4, - "falseNegative": 270, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 90, - "recall": 0.25, - "total": 2970, - "trueNegative": 2610, - "truePositive": 90, - "wilsonPrecisionLower95": 0.9591 - }, - "invitation": { - "f1": 0.5954, - "falseNegative": 102, - "falsePositive": 4, - "precision": 0.9512, - "predictedPositive": 82, - "recall": 0.4333, - "total": 2970, - "trueNegative": 2786, - "truePositive": 78, - "wilsonPrecisionLower95": 0.8812 - }, - "question": { - "f1": 0.4976, - "falseNegative": 419, - "falsePositive": 7, - "precision": 0.9679, - "predictedPositive": 218, - "recall": 0.3349, - "total": 2970, - "trueNegative": 2333, - "truePositive": 211, - "wilsonPrecisionLower95": 0.9352 - }, - "replyableMessage": { - "f1": 0.2549, - "falseNegative": 1612, - "falsePositive": 13, - "precision": 0.9553, - "predictedPositive": 291, - "recall": 0.1471, - "total": 2970, - "trueNegative": 1067, - "truePositive": 278, - "wilsonPrecisionLower95": 0.9251 - }, - "scheduleNegotiation": { - "f1": 0.8333, - "falseNegative": 20, - "falsePositive": 8, - "precision": 0.8974, - "predictedPositive": 78, - "recall": 0.7778, - "total": 2970, - "trueNegative": 2872, - "truePositive": 70, - "wilsonPrecisionLower95": 0.8105 - }, - "task": { - "f1": 0.2629, - "falseNegative": 608, - "falsePositive": 20, - "precision": 0.8485, - "predictedPositive": 132, - "recall": 0.1556, - "total": 2970, - "trueNegative": 2230, - "truePositive": 112, - "wilsonPrecisionLower95": 0.7775 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.989, - "falseNegative": 0, - "falsePositive": 4, - "precision": 0.9783, - "predictedPositive": 184, - "recall": 1.0, - "total": 1485, - "trueNegative": 1301, - "truePositive": 180, - "wilsonPrecisionLower95": 0.9454 - }, - "threshold": 0.995 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8869, - "falseNegative": 31, - "falsePositive": 7, - "precision": 0.9551, - "predictedPositive": 156, - "recall": 0.8278, - "total": 1485, - "trueNegative": 1298, - "truePositive": 149, - "wilsonPrecisionLower95": 0.9103 - }, - "threshold": 0.995 - } - }, - "metrics": { - "f1": 0.944, - "falseNegative": 23, - "falsePositive": 17, - "precision": 0.952, - "predictedPositive": 354, - "recall": 0.9361, - "total": 2970, - "trueNegative": 2593, - "truePositive": 337, - "wilsonPrecisionLower95": 0.9244 - }, - "threshold": 0.975 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8447, - "falseNegative": 22, - "falsePositive": 3, - "precision": 0.9577, - "predictedPositive": 71, - "recall": 0.7556, - "total": 1485, - "trueNegative": 1392, - "truePositive": 68, - "wilsonPrecisionLower95": 0.883 - }, - "threshold": 0.825 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.7723, - "falseNegative": 63, - "falsePositive": 6, - "precision": 0.9512, - "predictedPositive": 123, - "recall": 0.65, - "total": 2970, - "trueNegative": 2784, - "truePositive": 117, - "wilsonPrecisionLower95": 0.8977 - }, - "threshold": 0.965 - }, - "followUpReminder": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6625, - "falseNegative": 155, - "falsePositive": 8, - "precision": 0.9524, - "predictedPositive": 168, - "recall": 0.5079, - "total": 1485, - "trueNegative": 1162, - "truePositive": 160, - "wilsonPrecisionLower95": 0.9089 - }, - "threshold": 0.97 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "replyableMessage": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9655, - "falseNegative": 3, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 42, - "recall": 0.9333, - "total": 1485, - "trueNegative": 1440, - "truePositive": 42, - "wilsonPrecisionLower95": 0.9162 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9425, - "falseNegative": 4, - "falsePositive": 1, - "precision": 0.9762, - "predictedPositive": 42, - "recall": 0.9111, - "total": 1485, - "trueNegative": 1439, - "truePositive": 41, - "wilsonPrecisionLower95": 0.8768 - }, - "threshold": 0.73 - } - }, - "metrics": { - "f1": 0.9486, - "falseNegative": 7, - "falsePositive": 2, - "precision": 0.9765, - "predictedPositive": 85, - "recall": 0.9222, - "total": 2970, - "trueNegative": 2878, - "truePositive": 83, - "wilsonPrecisionLower95": 0.9182 - }, - "threshold": 0.5 - }, - "task": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "trainingRecords": 16820, - "validation": { - "intentsMeeting95Precision": 8, - "macroF1": 0.5631, - "macroPrecision": 0.9693, - "macroRecall": 0.4421, - "minimumPrecision": 0.9474, - "perIntent": { - "blessing": { - "f1": 0.3019, - "falseNegative": 74, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 16, - "recall": 0.1778, - "total": 2970, - "trueNegative": 2880, - "truePositive": 16, - "wilsonPrecisionLower95": 0.8064 - }, - "complaint": { - "f1": 0.94, - "falseNegative": 31, - "falsePositive": 11, - "precision": 0.9676, - "predictedPositive": 340, - "recall": 0.9139, - "total": 2970, - "trueNegative": 2599, - "truePositive": 329, - "wilsonPrecisionLower95": 0.943 - }, - "confirmationDecision": { - "f1": 0.6545, - "falseNegative": 90, - "falsePositive": 5, - "precision": 0.9474, - "predictedPositive": 95, - "recall": 0.5, - "total": 2970, - "trueNegative": 2785, - "truePositive": 90, - "wilsonPrecisionLower95": 0.8827 - }, - "followUpReminder": { - "f1": 0.5282, - "falseNegative": 229, - "falsePositive": 5, - "precision": 0.9632, - "predictedPositive": 136, - "recall": 0.3639, - "total": 2970, - "trueNegative": 2605, - "truePositive": 131, - "wilsonPrecisionLower95": 0.9168 - }, - "invitation": { - "f1": 0.4378, - "falseNegative": 129, - "falsePositive": 2, - "precision": 0.9623, - "predictedPositive": 53, - "recall": 0.2833, - "total": 2970, - "trueNegative": 2788, - "truePositive": 51, - "wilsonPrecisionLower95": 0.8725 - }, - "question": { - "f1": 0.5428, - "falseNegative": 392, - "falsePositive": 9, - "precision": 0.9636, - "predictedPositive": 247, - "recall": 0.3778, - "total": 2970, - "trueNegative": 2331, - "truePositive": 238, - "wilsonPrecisionLower95": 0.9322 - }, - "replyableMessage": { - "f1": 0.2757, - "falseNegative": 1586, - "falsePositive": 11, - "precision": 0.9651, - "predictedPositive": 315, - "recall": 0.1608, - "total": 2970, - "trueNegative": 1069, - "truePositive": 304, - "wilsonPrecisionLower95": 0.9386 - }, - "scheduleNegotiation": { - "f1": 0.954, - "falseNegative": 7, - "falsePositive": 1, - "precision": 0.9881, - "predictedPositive": 84, - "recall": 0.9222, - "total": 2970, - "trueNegative": 2879, - "truePositive": 83, - "wilsonPrecisionLower95": 0.9356 - }, - "task": { - "f1": 0.4332, - "falseNegative": 519, - "falsePositive": 7, - "precision": 0.9663, - "predictedPositive": 208, - "recall": 0.2792, - "total": 2970, - "trueNegative": 2243, - "truePositive": 201, - "wilsonPrecisionLower95": 0.9322 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 1.5e-06, - "augmentation": "numeric", - "char_max": 8, - "char_min": 2, - "external_weight": 0.91, - "hard_example_weight": 2.5, - "l1_ratio": 0.2, - "max_char_features": 90000, - "max_word_features": 45000, - "min_df": 1, - "pseudo_weight": 0.42000000000000004, - "round": 18, - "word_max": 3 - }, - "featureCount": 135000, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 1 - }, - "complaint": { - "negative": 250, - "positive": 8 - }, - "confirmationDecision": { - "negative": 250, - "positive": 27 - }, - "followUpReminder": { - "negative": 250, - "positive": 0 - }, - "invitation": { - "negative": 250, - "positive": 0 - }, - "question": { - "negative": 250, - "positive": 0 - }, - "replyableMessage": { - "negative": 90, - "positive": 32 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 0 - }, - "task": { - "negative": 250, - "positive": 0 - } - }, - "round": 18, - "selectionObjective": 0.7725, - "syntheticRegression": { - "intentsMeeting95Precision": 4, - "macroF1": 0.6489, - "macroPrecision": 0.9452, - "macroRecall": 0.5429, - "minimumPrecision": 0.8205, - "perIntent": { - "blessing": { - "f1": 0.2178, - "falseNegative": 79, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 11, - "recall": 0.1222, - "total": 2970, - "trueNegative": 2880, - "truePositive": 11, - "wilsonPrecisionLower95": 0.7412 - }, - "complaint": { - "f1": 0.8661, - "falseNegative": 85, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 275, - "recall": 0.7639, - "total": 2970, - "trueNegative": 2610, - "truePositive": 275, - "wilsonPrecisionLower95": 0.9862 - }, - "confirmationDecision": { - "f1": 0.6465, - "falseNegative": 84, - "falsePositive": 21, - "precision": 0.8205, - "predictedPositive": 117, - "recall": 0.5333, - "total": 2970, - "trueNegative": 2769, - "truePositive": 96, - "wilsonPrecisionLower95": 0.7411 - }, - "followUpReminder": { - "f1": 0.5851, - "falseNegative": 207, - "falsePositive": 10, - "precision": 0.9387, - "predictedPositive": 163, - "recall": 0.425, - "total": 2970, - "trueNegative": 2600, - "truePositive": 153, - "wilsonPrecisionLower95": 0.8908 - }, - "invitation": { - "f1": 0.626, - "falseNegative": 98, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 82, - "recall": 0.4556, - "total": 2970, - "trueNegative": 2790, - "truePositive": 82, - "wilsonPrecisionLower95": 0.9552 - }, - "question": { - "f1": 0.7311, - "falseNegative": 267, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 363, - "recall": 0.5762, - "total": 2970, - "trueNegative": 2340, - "truePositive": 363, - "wilsonPrecisionLower95": 0.9895 - }, - "replyableMessage": { - "f1": 0.8939, - "falseNegative": 273, - "falsePositive": 111, - "precision": 0.9358, - "predictedPositive": 1728, - "recall": 0.8556, - "total": 2970, - "trueNegative": 969, - "truePositive": 1617, - "wilsonPrecisionLower95": 0.9232 - }, - "scheduleNegotiation": { - "f1": 0.9341, - "falseNegative": 5, - "falsePositive": 7, - "precision": 0.9239, - "predictedPositive": 92, - "recall": 0.9444, - "total": 2970, - "trueNegative": 2873, - "truePositive": 85, - "wilsonPrecisionLower95": 0.8512 - }, - "task": { - "f1": 0.3393, - "falseNegative": 569, - "falsePositive": 19, - "precision": 0.8882, - "predictedPositive": 170, - "recall": 0.2097, - "total": 2970, - "trueNegative": 2231, - "truePositive": 151, - "wilsonPrecisionLower95": 0.832 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9486, - "falseNegative": 7, - "falsePositive": 2, - "precision": 0.9765, - "predictedPositive": 85, - "recall": 0.9222, - "total": 1485, - "trueNegative": 1393, - "truePositive": 83, - "wilsonPrecisionLower95": 0.9182 - }, - "threshold": 0.9 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.8447, - "falseNegative": 44, - "falsePositive": 6, - "precision": 0.9577, - "predictedPositive": 142, - "recall": 0.7556, - "total": 2970, - "trueNegative": 2784, - "truePositive": 136, - "wilsonPrecisionLower95": 0.9109 - }, - "threshold": 0.955 - }, - "followUpReminder": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8822, - "falseNegative": 34, - "falsePositive": 5, - "precision": 0.9669, - "predictedPositive": 151, - "recall": 0.8111, - "total": 1485, - "trueNegative": 1300, - "truePositive": 146, - "wilsonPrecisionLower95": 0.9248 - }, - "threshold": 0.98 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.7641, - "falseNegative": 130, - "falsePositive": 12, - "precision": 0.9504, - "predictedPositive": 242, - "recall": 0.6389, - "total": 2970, - "trueNegative": 2598, - "truePositive": 230, - "wilsonPrecisionLower95": 0.9153 - }, - "threshold": 0.99 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8103, - "falseNegative": 95, - "falsePositive": 8, - "precision": 0.9649, - "predictedPositive": 228, - "recall": 0.6984, - "total": 1485, - "trueNegative": 1162, - "truePositive": 220, - "wilsonPrecisionLower95": 0.9323 - }, - "threshold": 0.915 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9575, - "falseNegative": 33, - "falsePositive": 48, - "precision": 0.95, - "predictedPositive": 960, - "recall": 0.9651, - "total": 1485, - "trueNegative": 492, - "truePositive": 912, - "wilsonPrecisionLower95": 0.9343 - }, - "threshold": 0.845 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8361, - "falseNegative": 241, - "falsePositive": 35, - "precision": 0.9526, - "predictedPositive": 739, - "recall": 0.745, - "total": 1485, - "trueNegative": 505, - "truePositive": 704, - "wilsonPrecisionLower95": 0.9348 - }, - "threshold": 0.98 - } - }, - "metrics": { - "f1": 0.901, - "falseNegative": 274, - "falsePositive": 81, - "precision": 0.9523, - "predictedPositive": 1697, - "recall": 0.855, - "total": 2970, - "trueNegative": 999, - "truePositive": 1616, - "wilsonPrecisionLower95": 0.9411 - }, - "threshold": 0.975 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9655, - "falseNegative": 3, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 42, - "recall": 0.9333, - "total": 1485, - "trueNegative": 1440, - "truePositive": 42, - "wilsonPrecisionLower95": 0.9162 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 45, - "recall": 1.0, - "total": 1485, - "trueNegative": 1440, - "truePositive": 45, - "wilsonPrecisionLower95": 0.9213 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.9831, - "falseNegative": 3, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 87, - "recall": 0.9667, - "total": 2970, - "trueNegative": 2880, - "truePositive": 87, - "wilsonPrecisionLower95": 0.9577 - }, - "threshold": 0.5 - }, - "task": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "trainingRecords": 16841, - "validation": { - "intentsMeeting95Precision": 8, - "macroF1": 0.6817, - "macroPrecision": 0.9748, - "macroRecall": 0.5634, - "minimumPrecision": 0.931, - "perIntent": { - "blessing": { - "f1": 0.3486, - "falseNegative": 71, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 19, - "recall": 0.2111, - "total": 2970, - "trueNegative": 2880, - "truePositive": 19, - "wilsonPrecisionLower95": 0.8318 - }, - "complaint": { - "f1": 0.88, - "falseNegative": 74, - "falsePositive": 4, - "precision": 0.9862, - "predictedPositive": 290, - "recall": 0.7944, - "total": 2970, - "trueNegative": 2606, - "truePositive": 286, - "wilsonPrecisionLower95": 0.9651 - }, - "confirmationDecision": { - "f1": 0.7183, - "falseNegative": 78, - "falsePositive": 2, - "precision": 0.9808, - "predictedPositive": 104, - "recall": 0.5667, - "total": 2970, - "trueNegative": 2788, - "truePositive": 102, - "wilsonPrecisionLower95": 0.9326 - }, - "followUpReminder": { - "f1": 0.6345, - "falseNegative": 189, - "falsePositive": 8, - "precision": 0.9553, - "predictedPositive": 179, - "recall": 0.475, - "total": 2970, - "trueNegative": 2602, - "truePositive": 171, - "wilsonPrecisionLower95": 0.9143 - }, - "invitation": { - "f1": 0.4538, - "falseNegative": 126, - "falsePositive": 4, - "precision": 0.931, - "predictedPositive": 58, - "recall": 0.3, - "total": 2970, - "trueNegative": 2786, - "truePositive": 54, - "wilsonPrecisionLower95": 0.8357 - }, - "question": { - "f1": 0.7187, - "falseNegative": 271, - "falsePositive": 10, - "precision": 0.9729, - "predictedPositive": 369, - "recall": 0.5698, - "total": 2970, - "trueNegative": 2330, - "truePositive": 359, - "wilsonPrecisionLower95": 0.9508 - }, - "replyableMessage": { - "f1": 0.9005, - "falseNegative": 274, - "falsePositive": 83, - "precision": 0.9511, - "predictedPositive": 1699, - "recall": 0.855, - "total": 2970, - "trueNegative": 997, - "truePositive": 1616, - "wilsonPrecisionLower95": 0.9398 - }, - "scheduleNegotiation": { - "f1": 0.9831, - "falseNegative": 3, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 87, - "recall": 0.9667, - "total": 2970, - "trueNegative": 2880, - "truePositive": 87, - "wilsonPrecisionLower95": 0.9577 - }, - "task": { - "f1": 0.4979, - "falseNegative": 481, - "falsePositive": 1, - "precision": 0.9958, - "predictedPositive": 240, - "recall": 0.3319, - "total": 2970, - "trueNegative": 2249, - "truePositive": 239, - "wilsonPrecisionLower95": 0.9768 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 1.2e-06, - "augmentation": "numeric", - "char_max": 8, - "char_min": 3, - "external_weight": 0.93, - "hard_example_weight": 2.7, - "l1_ratio": 0.25, - "max_char_features": 95000, - "max_word_features": 45000, - "min_df": 1, - "pseudo_weight": 0.43499999999999994, - "round": 19, - "word_max": 3 - }, - "featureCount": 140000, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 1 - }, - "complaint": { - "negative": 250, - "positive": 2 - }, - "confirmationDecision": { - "negative": 250, - "positive": 36 - }, - "followUpReminder": { - "negative": 250, - "positive": 0 - }, - "invitation": { - "negative": 250, - "positive": 0 - }, - "question": { - "negative": 250, - "positive": 4 - }, - "replyableMessage": { - "negative": 99, - "positive": 32 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 0 - }, - "task": { - "negative": 250, - "positive": 0 - } - }, - "round": 19, - "selectionObjective": 0.7336, - "syntheticRegression": { - "intentsMeeting95Precision": 6, - "macroF1": 0.6005, - "macroPrecision": 0.9664, - "macroRecall": 0.4678, - "minimumPrecision": 0.887, - "perIntent": { - "blessing": { - "f1": 0.2, - "falseNegative": 80, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 10, - "recall": 0.1111, - "total": 2970, - "trueNegative": 2880, - "truePositive": 10, - "wilsonPrecisionLower95": 0.7225 - }, - "complaint": { - "f1": 0.9205, - "falseNegative": 53, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 307, - "recall": 0.8528, - "total": 2970, - "trueNegative": 2610, - "truePositive": 307, - "wilsonPrecisionLower95": 0.9876 - }, - "confirmationDecision": { - "f1": 0.6915, - "falseNegative": 78, - "falsePositive": 13, - "precision": 0.887, - "predictedPositive": 115, - "recall": 0.5667, - "total": 2970, - "trueNegative": 2777, - "truePositive": 102, - "wilsonPrecisionLower95": 0.8162 - }, - "followUpReminder": { - "f1": 0.4778, - "falseNegative": 247, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 113, - "recall": 0.3139, - "total": 2970, - "trueNegative": 2610, - "truePositive": 113, - "wilsonPrecisionLower95": 0.9671 - }, - "invitation": { - "f1": 0.7092, - "falseNegative": 80, - "falsePositive": 2, - "precision": 0.9804, - "predictedPositive": 102, - "recall": 0.5556, - "total": 2970, - "trueNegative": 2788, - "truePositive": 100, - "wilsonPrecisionLower95": 0.9313 - }, - "question": { - "f1": 0.4703, - "falseNegative": 436, - "falsePositive": 1, - "precision": 0.9949, - "predictedPositive": 195, - "recall": 0.3079, - "total": 2970, - "trueNegative": 2339, - "truePositive": 194, - "wilsonPrecisionLower95": 0.9715 - }, - "replyableMessage": { - "f1": 0.6399, - "falseNegative": 957, - "falsePositive": 93, - "precision": 0.9094, - "predictedPositive": 1026, - "recall": 0.4937, - "total": 2970, - "trueNegative": 987, - "truePositive": 933, - "wilsonPrecisionLower95": 0.8902 - }, - "scheduleNegotiation": { - "f1": 0.8079, - "falseNegative": 29, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 61, - "recall": 0.6778, - "total": 2970, - "trueNegative": 2880, - "truePositive": 61, - "wilsonPrecisionLower95": 0.9408 - }, - "task": { - "f1": 0.4872, - "falseNegative": 482, - "falsePositive": 19, - "precision": 0.9261, - "predictedPositive": 257, - "recall": 0.3306, - "total": 2970, - "trueNegative": 2231, - "truePositive": 238, - "wilsonPrecisionLower95": 0.8874 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "confirmationDecision": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "followUpReminder": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "replyableMessage": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9309, - "falseNegative": 83, - "falsePositive": 45, - "precision": 0.9504, - "predictedPositive": 907, - "recall": 0.9122, - "total": 1485, - "trueNegative": 495, - "truePositive": 862, - "wilsonPrecisionLower95": 0.9343 - }, - "threshold": 0.98 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 45, - "recall": 1.0, - "total": 1485, - "trueNegative": 1440, - "truePositive": 45, - "wilsonPrecisionLower95": 0.9213 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9195, - "falseNegative": 5, - "falsePositive": 2, - "precision": 0.9524, - "predictedPositive": 42, - "recall": 0.8889, - "total": 1485, - "trueNegative": 1438, - "truePositive": 40, - "wilsonPrecisionLower95": 0.8421 - }, - "threshold": 0.945 - } - }, - "metrics": { - "f1": 0.9667, - "falseNegative": 3, - "falsePositive": 3, - "precision": 0.9667, - "predictedPositive": 90, - "recall": 0.9667, - "total": 2970, - "trueNegative": 2877, - "truePositive": 87, - "wilsonPrecisionLower95": 0.9065 - }, - "threshold": 0.83 - }, - "task": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "trainingRecords": 16801, - "validation": { - "intentsMeeting95Precision": 6, - "macroF1": 0.6316, - "macroPrecision": 0.9616, - "macroRecall": 0.4969, - "minimumPrecision": 0.9242, - "perIntent": { - "blessing": { - "f1": 0.4483, - "falseNegative": 64, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 26, - "recall": 0.2889, - "total": 2970, - "trueNegative": 2880, - "truePositive": 26, - "wilsonPrecisionLower95": 0.8713 - }, - "complaint": { - "f1": 0.8953, - "falseNegative": 65, - "falsePositive": 4, - "precision": 0.9866, - "predictedPositive": 299, - "recall": 0.8194, - "total": 2970, - "trueNegative": 2606, - "truePositive": 295, - "wilsonPrecisionLower95": 0.9661 - }, - "confirmationDecision": { - "f1": 0.5891, - "falseNegative": 104, - "falsePositive": 2, - "precision": 0.9744, - "predictedPositive": 78, - "recall": 0.4222, - "total": 2970, - "trueNegative": 2788, - "truePositive": 76, - "wilsonPrecisionLower95": 0.9112 - }, - "followUpReminder": { - "f1": 0.528, - "falseNegative": 228, - "falsePositive": 8, - "precision": 0.9429, - "predictedPositive": 140, - "recall": 0.3667, - "total": 2970, - "trueNegative": 2602, - "truePositive": 132, - "wilsonPrecisionLower95": 0.8913 - }, - "invitation": { - "f1": 0.6015, - "falseNegative": 100, - "falsePositive": 6, - "precision": 0.9302, - "predictedPositive": 86, - "recall": 0.4444, - "total": 2970, - "trueNegative": 2784, - "truePositive": 80, - "wilsonPrecisionLower95": 0.856 - }, - "question": { - "f1": 0.442, - "falseNegative": 447, - "falsePositive": 15, - "precision": 0.9242, - "predictedPositive": 198, - "recall": 0.2905, - "total": 2970, - "trueNegative": 2325, - "truePositive": 183, - "wilsonPrecisionLower95": 0.8788 - }, - "replyableMessage": { - "f1": 0.6347, - "falseNegative": 990, - "falsePositive": 46, - "precision": 0.9514, - "predictedPositive": 946, - "recall": 0.4762, - "total": 2970, - "trueNegative": 1034, - "truePositive": 900, - "wilsonPrecisionLower95": 0.9357 - }, - "scheduleNegotiation": { - "f1": 0.9605, - "falseNegative": 5, - "falsePositive": 2, - "precision": 0.977, - "predictedPositive": 87, - "recall": 0.9444, - "total": 2970, - "trueNegative": 2878, - "truePositive": 85, - "wilsonPrecisionLower95": 0.92 - }, - "task": { - "f1": 0.5853, - "falseNegative": 418, - "falsePositive": 10, - "precision": 0.9679, - "predictedPositive": 312, - "recall": 0.4194, - "total": 2970, - "trueNegative": 2240, - "truePositive": 302, - "wilsonPrecisionLower95": 0.942 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 1e-06, - "augmentation": "numeric", - "char_max": 8, - "char_min": 2, - "external_weight": 0.9500000000000001, - "hard_example_weight": 3.0, - "l1_ratio": 0.25, - "max_char_features": 100000, - "max_word_features": 50000, - "min_df": 1, - "pseudo_weight": 0.44999999999999996, - "round": 20, - "word_max": 3 - }, - "featureCount": 150000, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 1 - }, - "complaint": { - "negative": 250, - "positive": 2 - }, - "confirmationDecision": { - "negative": 250, - "positive": 25 - }, - "followUpReminder": { - "negative": 250, - "positive": 0 - }, - "invitation": { - "negative": 250, - "positive": 1 - }, - "question": { - "negative": 250, - "positive": 2 - }, - "replyableMessage": { - "negative": 104, - "positive": 32 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 1 - }, - "task": { - "negative": 250, - "positive": 0 - } - }, - "round": 20, - "selectionObjective": 0.7724, - "syntheticRegression": { - "intentsMeeting95Precision": 7, - "macroF1": 0.6637, - "macroPrecision": 0.9685, - "macroRecall": 0.5421, - "minimumPrecision": 0.9094, - "perIntent": { - "blessing": { - "f1": 0.2353, - "falseNegative": 78, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 12, - "recall": 0.1333, - "total": 2970, - "trueNegative": 2880, - "truePositive": 12, - "wilsonPrecisionLower95": 0.7575 - }, - "complaint": { - "f1": 0.8274, - "falseNegative": 106, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 254, - "recall": 0.7056, - "total": 2970, - "trueNegative": 2610, - "truePositive": 254, - "wilsonPrecisionLower95": 0.9851 - }, - "confirmationDecision": { - "f1": 0.8261, - "falseNegative": 47, - "falsePositive": 9, - "precision": 0.9366, - "predictedPositive": 142, - "recall": 0.7389, - "total": 2970, - "trueNegative": 2781, - "truePositive": 133, - "wilsonPrecisionLower95": 0.8839 - }, - "followUpReminder": { - "f1": 0.5155, - "falseNegative": 235, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 125, - "recall": 0.3472, - "total": 2970, - "trueNegative": 2610, - "truePositive": 125, - "wilsonPrecisionLower95": 0.9702 - }, - "invitation": { - "f1": 0.7241, - "falseNegative": 75, - "falsePositive": 5, - "precision": 0.9545, - "predictedPositive": 110, - "recall": 0.5833, - "total": 2970, - "trueNegative": 2785, - "truePositive": 105, - "wilsonPrecisionLower95": 0.898 - }, - "question": { - "f1": 0.5534, - "falseNegative": 389, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 241, - "recall": 0.3825, - "total": 2970, - "trueNegative": 2340, - "truePositive": 241, - "wilsonPrecisionLower95": 0.9843 - }, - "replyableMessage": { - "f1": 0.7688, - "falseNegative": 671, - "falsePositive": 62, - "precision": 0.9516, - "predictedPositive": 1281, - "recall": 0.645, - "total": 2970, - "trueNegative": 1018, - "truePositive": 1219, - "wilsonPrecisionLower95": 0.9384 - }, - "scheduleNegotiation": { - "f1": 0.9371, - "falseNegative": 8, - "falsePositive": 3, - "precision": 0.9647, - "predictedPositive": 85, - "recall": 0.9111, - "total": 2970, - "trueNegative": 2877, - "truePositive": 82, - "wilsonPrecisionLower95": 0.9013 - }, - "task": { - "f1": 0.5857, - "falseNegative": 409, - "falsePositive": 31, - "precision": 0.9094, - "predictedPositive": 342, - "recall": 0.4319, - "total": 2970, - "trueNegative": 2219, - "truePositive": 311, - "wilsonPrecisionLower95": 0.8742 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "confirmationDecision": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.5954, - "falseNegative": 51, - "falsePositive": 2, - "precision": 0.9512, - "predictedPositive": 41, - "recall": 0.4333, - "total": 1485, - "trueNegative": 1393, - "truePositive": 39, - "wilsonPrecisionLower95": 0.8386 - }, - "threshold": 0.995 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "followUpReminder": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8117, - "falseNegative": 279, - "falsePositive": 30, - "precision": 0.9569, - "predictedPositive": 696, - "recall": 0.7048, - "total": 1485, - "trueNegative": 510, - "truePositive": 666, - "wilsonPrecisionLower95": 0.9391 - }, - "threshold": 0.995 - } - }, - "metrics": { - "f1": 0.8736, - "falseNegative": 363, - "falsePositive": 79, - "precision": 0.9508, - "predictedPositive": 1606, - "recall": 0.8079, - "total": 2970, - "trueNegative": 1001, - "truePositive": 1527, - "wilsonPrecisionLower95": 0.9391 - }, - "threshold": 0.995 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 45, - "recall": 1.0, - "total": 1485, - "trueNegative": 1440, - "truePositive": 45, - "wilsonPrecisionLower95": 0.9213 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.967, - "falseNegative": 1, - "falsePositive": 2, - "precision": 0.9565, - "predictedPositive": 46, - "recall": 0.9778, - "total": 1485, - "trueNegative": 1438, - "truePositive": 44, - "wilsonPrecisionLower95": 0.8547 - }, - "threshold": 0.83 - } - }, - "metrics": { - "f1": 0.9834, - "falseNegative": 1, - "falsePositive": 2, - "precision": 0.978, - "predictedPositive": 91, - "recall": 0.9889, - "total": 2970, - "trueNegative": 2878, - "truePositive": 89, - "wilsonPrecisionLower95": 0.9234 - }, - "threshold": 0.83 - }, - "task": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "trainingRecords": 16755, - "validation": { - "intentsMeeting95Precision": 8, - "macroF1": 0.6867, - "macroPrecision": 0.9687, - "macroRecall": 0.5569, - "minimumPrecision": 0.9038, - "perIntent": { - "blessing": { - "f1": 0.3929, - "falseNegative": 68, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 22, - "recall": 0.2444, - "total": 2970, - "trueNegative": 2880, - "truePositive": 22, - "wilsonPrecisionLower95": 0.8513 - }, - "complaint": { - "f1": 0.8066, - "falseNegative": 114, - "falsePositive": 4, - "precision": 0.984, - "predictedPositive": 250, - "recall": 0.6833, - "total": 2970, - "trueNegative": 2606, - "truePositive": 246, - "wilsonPrecisionLower95": 0.9596 - }, - "confirmationDecision": { - "f1": 0.6618, - "falseNegative": 90, - "falsePositive": 2, - "precision": 0.9783, - "predictedPositive": 92, - "recall": 0.5, - "total": 2970, - "trueNegative": 2788, - "truePositive": 90, - "wilsonPrecisionLower95": 0.9242 - }, - "followUpReminder": { - "f1": 0.5977, - "falseNegative": 204, - "falsePositive": 6, - "precision": 0.963, - "predictedPositive": 162, - "recall": 0.4333, - "total": 2970, - "trueNegative": 2604, - "truePositive": 156, - "wilsonPrecisionLower95": 0.9216 - }, - "invitation": { - "f1": 0.662, - "falseNegative": 86, - "falsePositive": 10, - "precision": 0.9038, - "predictedPositive": 104, - "recall": 0.5222, - "total": 2970, - "trueNegative": 2780, - "truePositive": 94, - "wilsonPrecisionLower95": 0.832 - }, - "question": { - "f1": 0.5643, - "falseNegative": 380, - "falsePositive": 6, - "precision": 0.9766, - "predictedPositive": 256, - "recall": 0.3968, - "total": 2970, - "trueNegative": 2334, - "truePositive": 250, - "wilsonPrecisionLower95": 0.9498 - }, - "replyableMessage": { - "f1": 0.7934, - "falseNegative": 621, - "falsePositive": 40, - "precision": 0.9694, - "predictedPositive": 1309, - "recall": 0.6714, - "total": 2970, - "trueNegative": 1040, - "truePositive": 1269, - "wilsonPrecisionLower95": 0.9587 - }, - "scheduleNegotiation": { - "f1": 0.9834, - "falseNegative": 1, - "falsePositive": 2, - "precision": 0.978, - "predictedPositive": 91, - "recall": 0.9889, - "total": 2970, - "trueNegative": 2878, - "truePositive": 89, - "wilsonPrecisionLower95": 0.9234 - }, - "task": { - "f1": 0.7184, - "falseNegative": 308, - "falsePositive": 15, - "precision": 0.9649, - "predictedPositive": 427, - "recall": 0.5722, - "total": 2970, - "trueNegative": 2235, - "truePositive": 412, - "wilsonPrecisionLower95": 0.9429 - } - }, - "recordCount": 2970 - } - } -] diff --git a/ModelTraining/ClipboardSemantics/IterativeResearch/final-report.json b/ModelTraining/ClipboardSemantics/IterativeResearch/final-report.json deleted file mode 100644 index 96e85d0..0000000 --- a/ModelTraining/ClipboardSemantics/IterativeResearch/final-report.json +++ /dev/null @@ -1,8411 +0,0 @@ -{ - "baseTrainingRecords": 11500, - "deploymentDecision": "retain-current-deployed-models", - "elapsedSeconds": 170.1895, - "finalEvaluations": { - "golden": { - "byLanguage": { - "en": { - "intentsMeeting95Precision": 1, - "macroF1": 0.8445, - "macroPrecision": 0.9286, - "macroRecall": 0.8066, - "minimumPrecision": 0.7692, - "perIntent": { - "blessing": { - "f1": 0.8235, - "falseNegative": 3, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 7, - "recall": 0.7, - "total": 126, - "trueNegative": 116, - "truePositive": 7, - "wilsonPrecisionLower95": 0.6457 - }, - "complaint": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 6, - "recall": 1.0, - "total": 126, - "trueNegative": 120, - "truePositive": 6, - "wilsonPrecisionLower95": 0.6097 - }, - "confirmationDecision": { - "f1": 0.8696, - "falseNegative": 0, - "falsePositive": 3, - "precision": 0.7692, - "predictedPositive": 13, - "recall": 1.0, - "total": 126, - "trueNegative": 113, - "truePositive": 10, - "wilsonPrecisionLower95": 0.4974 - }, - "followUpReminder": { - "f1": 0.8696, - "falseNegative": 3, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 10, - "recall": 0.7692, - "total": 126, - "trueNegative": 113, - "truePositive": 10, - "wilsonPrecisionLower95": 0.7225 - }, - "invitation": { - "f1": 0.8571, - "falseNegative": 2, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 6, - "recall": 0.75, - "total": 126, - "trueNegative": 118, - "truePositive": 6, - "wilsonPrecisionLower95": 0.6097 - }, - "question": { - "f1": 0.8636, - "falseNegative": 4, - "falsePositive": 2, - "precision": 0.9048, - "predictedPositive": 21, - "recall": 0.8261, - "total": 126, - "trueNegative": 101, - "truePositive": 19, - "wilsonPrecisionLower95": 0.7109 - }, - "replyableMessage": { - "f1": 0.8889, - "falseNegative": 6, - "falsePositive": 10, - "precision": 0.8649, - "predictedPositive": 74, - "recall": 0.9143, - "total": 126, - "trueNegative": 46, - "truePositive": 64, - "wilsonPrecisionLower95": 0.7688 - }, - "scheduleNegotiation": { - "f1": 0.8571, - "falseNegative": 1, - "falsePositive": 2, - "precision": 0.8182, - "predictedPositive": 11, - "recall": 0.9, - "total": 126, - "trueNegative": 114, - "truePositive": 9, - "wilsonPrecisionLower95": 0.523 - }, - "task": { - "f1": 0.5714, - "falseNegative": 9, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 6, - "recall": 0.4, - "total": 126, - "trueNegative": 111, - "truePositive": 6, - "wilsonPrecisionLower95": 0.6097 - } - }, - "recordCount": 126 - }, - "zh-Hans": { - "intentsMeeting95Precision": 1, - "macroF1": 0.6673, - "macroPrecision": 0.9541, - "macroRecall": 0.5535, - "minimumPrecision": 0.7143, - "perIntent": { - "blessing": { - "f1": 0.4615, - "falseNegative": 7, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 3, - "recall": 0.3, - "total": 126, - "trueNegative": 116, - "truePositive": 3, - "wilsonPrecisionLower95": 0.4385 - }, - "complaint": { - "f1": 0.6667, - "falseNegative": 3, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 3, - "recall": 0.5, - "total": 126, - "trueNegative": 120, - "truePositive": 3, - "wilsonPrecisionLower95": 0.4385 - }, - "confirmationDecision": { - "f1": 0.5882, - "falseNegative": 5, - "falsePositive": 2, - "precision": 0.7143, - "predictedPositive": 7, - "recall": 0.5, - "total": 126, - "trueNegative": 114, - "truePositive": 5, - "wilsonPrecisionLower95": 0.3589 - }, - "followUpReminder": { - "f1": 0.8182, - "falseNegative": 4, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 9, - "recall": 0.6923, - "total": 126, - "trueNegative": 113, - "truePositive": 9, - "wilsonPrecisionLower95": 0.7009 - }, - "invitation": { - "f1": 0.5455, - "falseNegative": 5, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 3, - "recall": 0.375, - "total": 126, - "trueNegative": 118, - "truePositive": 3, - "wilsonPrecisionLower95": 0.4385 - }, - "question": { - "f1": 0.9787, - "falseNegative": 0, - "falsePositive": 1, - "precision": 0.9583, - "predictedPositive": 24, - "recall": 1.0, - "total": 126, - "trueNegative": 102, - "truePositive": 23, - "wilsonPrecisionLower95": 0.7976 - }, - "replyableMessage": { - "f1": 0.9143, - "falseNegative": 6, - "falsePositive": 6, - "precision": 0.9143, - "predictedPositive": 70, - "recall": 0.9143, - "total": 126, - "trueNegative": 50, - "truePositive": 64, - "wilsonPrecisionLower95": 0.8253 - }, - "scheduleNegotiation": { - "f1": 0.4615, - "falseNegative": 7, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 3, - "recall": 0.3, - "total": 126, - "trueNegative": 116, - "truePositive": 3, - "wilsonPrecisionLower95": 0.4385 - }, - "task": { - "f1": 0.5714, - "falseNegative": 9, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 6, - "recall": 0.4, - "total": 126, - "trueNegative": 111, - "truePositive": 6, - "wilsonPrecisionLower95": 0.6097 - } - }, - "recordCount": 126 - } - }, - "bySource": { - "acknowledgment": { - "intentsMeeting95Precision": 0, - "macroF1": 0.0, - "macroPrecision": 0.0, - "macroRecall": 0.0, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 1, - "precision": 0.0, - "predictedPositive": 1, - "recall": 0.0, - "total": 8, - "trueNegative": 7, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 8 - }, - "blessing_boundary": { - "intentsMeeting95Precision": 0, - "macroF1": 0.1482, - "macroPrecision": 0.1667, - "macroRecall": 0.1667, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.6667, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 1, - "recall": 0.5, - "total": 20, - "trueNegative": 18, - "truePositive": 1, - "wilsonPrecisionLower95": 0.2065 - }, - "replyableMessage": { - "f1": 0.6667, - "falseNegative": 0, - "falsePositive": 4, - "precision": 0.5, - "predictedPositive": 8, - "recall": 1.0, - "total": 20, - "trueNegative": 12, - "truePositive": 4, - "wilsonPrecisionLower95": 0.2152 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 2, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 18, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 20 - }, - "blessing_golden": { - "intentsMeeting95Precision": 2, - "macroF1": 0.1823, - "macroPrecision": 0.2222, - "macroRecall": 0.1611, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.6667, - "falseNegative": 10, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 10, - "recall": 0.5, - "total": 20, - "trueNegative": 0, - "truePositive": 10, - "wilsonPrecisionLower95": 0.7225 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.9744, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 19, - "recall": 0.95, - "total": 20, - "trueNegative": 0, - "truePositive": 19, - "wilsonPrecisionLower95": 0.8318 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 20 - }, - "complaint_request": { - "intentsMeeting95Precision": 0, - "macroF1": 0.2703, - "macroPrecision": 0.3333, - "macroRecall": 0.2315, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.6667, - "falseNegative": 3, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 3, - "recall": 0.5, - "total": 6, - "trueNegative": 0, - "truePositive": 3, - "wilsonPrecisionLower95": 0.4385 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.8571, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 3, - "recall": 0.75, - "total": 6, - "trueNegative": 2, - "truePositive": 3, - "wilsonPrecisionLower95": 0.4385 - }, - "replyableMessage": { - "f1": 0.9091, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 5, - "recall": 0.8333, - "total": 6, - "trueNegative": 0, - "truePositive": 5, - "wilsonPrecisionLower95": 0.5655 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 6 - }, - "complaint_statement": { - "intentsMeeting95Precision": 0, - "macroF1": 0.2222, - "macroPrecision": 0.2222, - "macroRecall": 0.2222, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 6, - "recall": 1.0, - "total": 6, - "trueNegative": 0, - "truePositive": 6, - "wilsonPrecisionLower95": 0.6097 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 6, - "recall": 1.0, - "total": 6, - "trueNegative": 0, - "truePositive": 6, - "wilsonPrecisionLower95": 0.6097 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 6 - }, - "confirmation_decision_boundary": { - "intentsMeeting95Precision": 0, - "macroF1": 0.0, - "macroPrecision": 0.0, - "macroRecall": 0.0, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 3, - "precision": 0.0, - "predictedPositive": 3, - "recall": 0.0, - "total": 20, - "trueNegative": 17, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 3, - "precision": 0.0, - "predictedPositive": 3, - "recall": 0.0, - "total": 20, - "trueNegative": 17, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 20 - }, - "confirmation_decision_golden": { - "intentsMeeting95Precision": 2, - "macroF1": 0.2063, - "macroPrecision": 0.2222, - "macroRecall": 0.1944, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.8571, - "falseNegative": 5, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 15, - "recall": 0.75, - "total": 20, - "trueNegative": 0, - "truePositive": 15, - "wilsonPrecisionLower95": 0.7961 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 20, - "recall": 1.0, - "total": 20, - "trueNegative": 0, - "truePositive": 20, - "wilsonPrecisionLower95": 0.8389 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 20 - }, - "conversational_message": { - "intentsMeeting95Precision": 1, - "macroF1": 0.1111, - "macroPrecision": 0.1111, - "macroRecall": 0.1111, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 10, - "trueNegative": 10, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 10, - "trueNegative": 10, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 10, - "trueNegative": 10, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 10, - "trueNegative": 10, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 10, - "trueNegative": 10, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 10, - "trueNegative": 10, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 10, - "recall": 1.0, - "total": 10, - "trueNegative": 0, - "truePositive": 10, - "wilsonPrecisionLower95": 0.7225 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 10, - "trueNegative": 10, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 10, - "trueNegative": 10, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 10 - }, - "event_statement": { - "intentsMeeting95Precision": 0, - "macroF1": 0.0, - "macroPrecision": 0.0, - "macroRecall": 0.0, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 4 - }, - "follow_up_reminder_boundary": { - "intentsMeeting95Precision": 0, - "macroF1": 0.0, - "macroPrecision": 0.0, - "macroRecall": 0.0, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 3, - "precision": 0.0, - "predictedPositive": 3, - "recall": 0.0, - "total": 20, - "trueNegative": 17, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 20 - }, - "follow_up_reminder_golden": { - "intentsMeeting95Precision": 1, - "macroF1": 0.2187, - "macroPrecision": 0.3333, - "macroRecall": 0.1706, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.8571, - "falseNegative": 5, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 15, - "recall": 0.75, - "total": 20, - "trueNegative": 0, - "truePositive": 15, - "wilsonPrecisionLower95": 0.7961 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.6667, - "falseNegative": 7, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 7, - "recall": 0.5, - "total": 20, - "trueNegative": 6, - "truePositive": 7, - "wilsonPrecisionLower95": 0.6457 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.4444, - "falseNegative": 10, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 4, - "recall": 0.2857, - "total": 20, - "trueNegative": 6, - "truePositive": 4, - "wilsonPrecisionLower95": 0.5101 - } - }, - "recordCount": 20 - }, - "information_question": { - "intentsMeeting95Precision": 0, - "macroF1": 0.1693, - "macroPrecision": 0.2222, - "macroRecall": 0.1389, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.6667, - "falseNegative": 2, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 2, - "recall": 0.5, - "total": 4, - "trueNegative": 0, - "truePositive": 2, - "wilsonPrecisionLower95": 0.3424 - }, - "replyableMessage": { - "f1": 0.8571, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 3, - "recall": 0.75, - "total": 4, - "trueNegative": 0, - "truePositive": 3, - "wilsonPrecisionLower95": 0.4385 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 4 - }, - "invitation_question": { - "intentsMeeting95Precision": 0, - "macroF1": 0.3175, - "macroPrecision": 0.3333, - "macroRecall": 0.3056, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.8571, - "falseNegative": 2, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 6, - "recall": 0.75, - "total": 8, - "trueNegative": 0, - "truePositive": 6, - "wilsonPrecisionLower95": 0.6097 - }, - "question": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 8, - "recall": 1.0, - "total": 8, - "trueNegative": 0, - "truePositive": 8, - "wilsonPrecisionLower95": 0.6756 - }, - "replyableMessage": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 8, - "recall": 1.0, - "total": 8, - "trueNegative": 0, - "truePositive": 8, - "wilsonPrecisionLower95": 0.6756 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 8 - }, - "negative_news": { - "intentsMeeting95Precision": 0, - "macroF1": 0.0, - "macroPrecision": 0.0, - "macroRecall": 0.0, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 1, - "precision": 0.0, - "predictedPositive": 1, - "recall": 0.0, - "total": 6, - "trueNegative": 5, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 6 - }, - "neutral_fact": { - "intentsMeeting95Precision": 0, - "macroF1": 0.0, - "macroPrecision": 0.0, - "macroRecall": 0.0, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 10, - "trueNegative": 10, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 10, - "trueNegative": 10, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 10, - "trueNegative": 10, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 10, - "trueNegative": 10, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 10, - "trueNegative": 10, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 10, - "trueNegative": 10, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 2, - "precision": 0.0, - "predictedPositive": 2, - "recall": 0.0, - "total": 10, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 1, - "precision": 0.0, - "predictedPositive": 1, - "recall": 0.0, - "total": 10, - "trueNegative": 9, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 10, - "trueNegative": 10, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 10 - }, - "positive_feedback": { - "intentsMeeting95Precision": 0, - "macroF1": 0.1111, - "macroPrecision": 0.1111, - "macroRecall": 0.1111, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 6, - "recall": 1.0, - "total": 6, - "trueNegative": 0, - "truePositive": 6, - "wilsonPrecisionLower95": 0.6097 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 6 - }, - "quoted_question": { - "intentsMeeting95Precision": 0, - "macroF1": 0.0, - "macroPrecision": 0.0, - "macroRecall": 0.0, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 4, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 4 - }, - "schedule_negotiation_boundary": { - "intentsMeeting95Precision": 0, - "macroF1": 0.2541, - "macroPrecision": 0.284, - "macroRecall": 0.2639, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 1, - "precision": 0.0, - "predictedPositive": 1, - "recall": 0.0, - "total": 20, - "trueNegative": 19, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.5455, - "falseNegative": 5, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 3, - "recall": 0.375, - "total": 20, - "trueNegative": 12, - "truePositive": 3, - "wilsonPrecisionLower95": 0.4385 - }, - "question": { - "f1": 0.8, - "falseNegative": 0, - "falsePositive": 2, - "precision": 0.6667, - "predictedPositive": 6, - "recall": 1.0, - "total": 20, - "trueNegative": 14, - "truePositive": 4, - "wilsonPrecisionLower95": 0.3 - }, - "replyableMessage": { - "f1": 0.9412, - "falseNegative": 0, - "falsePositive": 1, - "precision": 0.8889, - "predictedPositive": 9, - "recall": 1.0, - "total": 20, - "trueNegative": 11, - "truePositive": 8, - "wilsonPrecisionLower95": 0.565 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 1, - "precision": 0.0, - "predictedPositive": 1, - "recall": 0.0, - "total": 20, - "trueNegative": 19, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 20 - }, - "schedule_negotiation_golden": { - "intentsMeeting95Precision": 2, - "macroF1": 0.2997, - "macroPrecision": 0.3275, - "macroRecall": 0.2833, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.973, - "falseNegative": 0, - "falsePositive": 1, - "precision": 0.9474, - "predictedPositive": 19, - "recall": 1.0, - "total": 20, - "trueNegative": 1, - "truePositive": 18, - "wilsonPrecisionLower95": 0.7536 - }, - "replyableMessage": { - "f1": 0.9744, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 19, - "recall": 0.95, - "total": 20, - "trueNegative": 0, - "truePositive": 19, - "wilsonPrecisionLower95": 0.8318 - }, - "scheduleNegotiation": { - "f1": 0.75, - "falseNegative": 8, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 12, - "recall": 0.6, - "total": 20, - "trueNegative": 0, - "truePositive": 12, - "wilsonPrecisionLower95": 0.7575 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 20, - "trueNegative": 20, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 20 - }, - "self_plan": { - "intentsMeeting95Precision": 0, - "macroF1": 0.1111, - "macroPrecision": 0.1111, - "macroRecall": 0.1111, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 2, - "recall": 1.0, - "total": 6, - "trueNegative": 4, - "truePositive": 2, - "wilsonPrecisionLower95": 0.3424 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 2, - "precision": 0.0, - "predictedPositive": 2, - "recall": 0.0, - "total": 6, - "trueNegative": 4, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 6 - }, - "task_question": { - "intentsMeeting95Precision": 0, - "macroF1": 0.2963, - "macroPrecision": 0.3333, - "macroRecall": 0.2778, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 6, - "recall": 1.0, - "total": 6, - "trueNegative": 0, - "truePositive": 6, - "wilsonPrecisionLower95": 0.6097 - }, - "replyableMessage": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 6, - "recall": 1.0, - "total": 6, - "trueNegative": 0, - "truePositive": 6, - "wilsonPrecisionLower95": 0.6097 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 6, - "trueNegative": 6, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.6667, - "falseNegative": 3, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 3, - "recall": 0.5, - "total": 6, - "trueNegative": 0, - "truePositive": 3, - "wilsonPrecisionLower95": 0.4385 - } - }, - "recordCount": 6 - }, - "task_statement": { - "intentsMeeting95Precision": 0, - "macroF1": 0.2632, - "macroPrecision": 0.3333, - "macroRecall": 0.2222, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.6667, - "falseNegative": 2, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 2, - "recall": 0.5, - "total": 8, - "trueNegative": 4, - "truePositive": 2, - "wilsonPrecisionLower95": 0.3424 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.9333, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 7, - "recall": 0.875, - "total": 8, - "trueNegative": 0, - "truePositive": 7, - "wilsonPrecisionLower95": 0.6457 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 8, - "trueNegative": 8, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.7692, - "falseNegative": 3, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 5, - "recall": 0.625, - "total": 8, - "trueNegative": 0, - "truePositive": 5, - "wilsonPrecisionLower95": 0.5655 - } - }, - "recordCount": 8 - } - }, - "intentsMeeting95Precision": 3, - "macroF1": 0.7711, - "macroPrecision": 0.9366, - "macroRecall": 0.6801, - "minimumPrecision": 0.75, - "perIntent": { - "blessing": { - "f1": 0.6667, - "falseNegative": 10, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 10, - "recall": 0.5, - "total": 252, - "trueNegative": 232, - "truePositive": 10, - "wilsonPrecisionLower95": 0.7225 - }, - "complaint": { - "f1": 0.8571, - "falseNegative": 3, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 9, - "recall": 0.75, - "total": 252, - "trueNegative": 240, - "truePositive": 9, - "wilsonPrecisionLower95": 0.7009 - }, - "confirmationDecision": { - "f1": 0.75, - "falseNegative": 5, - "falsePositive": 5, - "precision": 0.75, - "predictedPositive": 20, - "recall": 0.75, - "total": 252, - "trueNegative": 227, - "truePositive": 15, - "wilsonPrecisionLower95": 0.5313 - }, - "followUpReminder": { - "f1": 0.8444, - "falseNegative": 7, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 19, - "recall": 0.7308, - "total": 252, - "trueNegative": 226, - "truePositive": 19, - "wilsonPrecisionLower95": 0.8318 - }, - "invitation": { - "f1": 0.72, - "falseNegative": 7, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 9, - "recall": 0.5625, - "total": 252, - "trueNegative": 236, - "truePositive": 9, - "wilsonPrecisionLower95": 0.7009 - }, - "question": { - "f1": 0.9231, - "falseNegative": 4, - "falsePositive": 3, - "precision": 0.9333, - "predictedPositive": 45, - "recall": 0.913, - "total": 252, - "trueNegative": 203, - "truePositive": 42, - "wilsonPrecisionLower95": 0.8214 - }, - "replyableMessage": { - "f1": 0.9014, - "falseNegative": 12, - "falsePositive": 16, - "precision": 0.8889, - "predictedPositive": 144, - "recall": 0.9143, - "total": 252, - "trueNegative": 96, - "truePositive": 128, - "wilsonPrecisionLower95": 0.8271 - }, - "scheduleNegotiation": { - "f1": 0.7059, - "falseNegative": 8, - "falsePositive": 2, - "precision": 0.8571, - "predictedPositive": 14, - "recall": 0.6, - "total": 252, - "trueNegative": 230, - "truePositive": 12, - "wilsonPrecisionLower95": 0.6006 - }, - "task": { - "f1": 0.5714, - "falseNegative": 18, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 12, - "recall": 0.4, - "total": 252, - "trueNegative": 222, - "truePositive": 12, - "wilsonPrecisionLower95": 0.7575 - } - }, - "recordCount": 252 - }, - "randomHoldout": { - "byLanguage": { - "en": { - "intentsMeeting95Precision": 4, - "macroF1": 0.6163, - "macroPrecision": 0.8521, - "macroRecall": 0.5281, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 340, - "trueNegative": 340, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.3333, - "falseNegative": 16, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 4, - "recall": 0.2, - "total": 340, - "trueNegative": 320, - "truePositive": 4, - "wilsonPrecisionLower95": 0.5101 - }, - "confirmationDecision": { - "f1": 0.9302, - "falseNegative": 0, - "falsePositive": 3, - "precision": 0.8696, - "predictedPositive": 23, - "recall": 1.0, - "total": 340, - "trueNegative": 317, - "truePositive": 20, - "wilsonPrecisionLower95": 0.6787 - }, - "followUpReminder": { - "f1": 0.9027, - "falseNegative": 9, - "falsePositive": 2, - "precision": 0.9623, - "predictedPositive": 53, - "recall": 0.85, - "total": 340, - "trueNegative": 278, - "truePositive": 51, - "wilsonPrecisionLower95": 0.8725 - }, - "invitation": { - "f1": 0.5714, - "falseNegative": 12, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 8, - "recall": 0.4, - "total": 340, - "trueNegative": 320, - "truePositive": 8, - "wilsonPrecisionLower95": 0.6756 - }, - "question": { - "f1": 0.6087, - "falseNegative": 45, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 35, - "recall": 0.4375, - "total": 340, - "trueNegative": 260, - "truePositive": 35, - "wilsonPrecisionLower95": 0.9011 - }, - "replyableMessage": { - "f1": 0.8859, - "falseNegative": 37, - "falsePositive": 5, - "precision": 0.9702, - "predictedPositive": 168, - "recall": 0.815, - "total": 340, - "trueNegative": 135, - "truePositive": 163, - "wilsonPrecisionLower95": 0.9322 - }, - "scheduleNegotiation": { - "f1": 0.7429, - "falseNegative": 7, - "falsePositive": 2, - "precision": 0.8667, - "predictedPositive": 15, - "recall": 0.65, - "total": 340, - "trueNegative": 318, - "truePositive": 13, - "wilsonPrecisionLower95": 0.6212 - }, - "task": { - "f1": 0.5714, - "falseNegative": 36, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 24, - "recall": 0.4, - "total": 340, - "trueNegative": 280, - "truePositive": 24, - "wilsonPrecisionLower95": 0.862 - } - }, - "recordCount": 340 - }, - "zh-Hans": { - "intentsMeeting95Precision": 4, - "macroF1": 0.4934, - "macroPrecision": 0.7719, - "macroRecall": 0.3925, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 340, - "trueNegative": 340, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.3333, - "falseNegative": 16, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 4, - "recall": 0.2, - "total": 340, - "trueNegative": 320, - "truePositive": 4, - "wilsonPrecisionLower95": 0.5101 - }, - "confirmationDecision": { - "f1": 0.6667, - "falseNegative": 10, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 10, - "recall": 0.5, - "total": 340, - "trueNegative": 320, - "truePositive": 10, - "wilsonPrecisionLower95": 0.7225 - }, - "followUpReminder": { - "f1": 0.7879, - "falseNegative": 21, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 39, - "recall": 0.65, - "total": 340, - "trueNegative": 280, - "truePositive": 39, - "wilsonPrecisionLower95": 0.9103 - }, - "invitation": { - "f1": 0.6207, - "falseNegative": 11, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 9, - "recall": 0.45, - "total": 340, - "trueNegative": 320, - "truePositive": 9, - "wilsonPrecisionLower95": 0.7009 - }, - "question": { - "f1": 0.7786, - "falseNegative": 29, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 51, - "recall": 0.6375, - "total": 340, - "trueNegative": 260, - "truePositive": 51, - "wilsonPrecisionLower95": 0.93 - }, - "replyableMessage": { - "f1": 0.9203, - "falseNegative": 21, - "falsePositive": 10, - "precision": 0.9471, - "predictedPositive": 189, - "recall": 0.895, - "total": 340, - "trueNegative": 130, - "truePositive": 179, - "wilsonPrecisionLower95": 0.9054 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 20, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 340, - "trueNegative": 320, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.3333, - "falseNegative": 48, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 12, - "recall": 0.2, - "total": 340, - "trueNegative": 280, - "truePositive": 12, - "wilsonPrecisionLower95": 0.7575 - } - }, - "recordCount": 340 - } - }, - "bySource": { - "acknowledgment_boundary": { - "intentsMeeting95Precision": 0, - "macroF1": 0.0, - "macroPrecision": 0.0, - "macroRecall": 0.0, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 40 - }, - "complaint_support": { - "intentsMeeting95Precision": 1, - "macroF1": 0.1783, - "macroPrecision": 0.3333, - "macroRecall": 0.1389, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.3333, - "falseNegative": 32, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 8, - "recall": 0.2, - "total": 40, - "trueNegative": 0, - "truePositive": 8, - "wilsonPrecisionLower95": 0.6756 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.3673, - "falseNegative": 31, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 9, - "recall": 0.225, - "total": 40, - "trueNegative": 0, - "truePositive": 9, - "wilsonPrecisionLower95": 0.7009 - }, - "replyableMessage": { - "f1": 0.9041, - "falseNegative": 7, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 33, - "recall": 0.825, - "total": 40, - "trueNegative": 0, - "truePositive": 33, - "wilsonPrecisionLower95": 0.8957 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 40 - }, - "confirmation_decision": { - "intentsMeeting95Precision": 2, - "macroF1": 0.1957, - "macroPrecision": 0.2222, - "macroRecall": 0.175, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.8571, - "falseNegative": 10, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 30, - "recall": 0.75, - "total": 40, - "trueNegative": 0, - "truePositive": 30, - "wilsonPrecisionLower95": 0.8865 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.9041, - "falseNegative": 7, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 33, - "recall": 0.825, - "total": 40, - "trueNegative": 0, - "truePositive": 33, - "wilsonPrecisionLower95": 0.8957 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 40 - }, - "conversation": { - "intentsMeeting95Precision": 1, - "macroF1": 0.1053, - "macroPrecision": 0.1111, - "macroRecall": 0.1, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.9474, - "falseNegative": 4, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 36, - "recall": 0.9, - "total": 40, - "trueNegative": 0, - "truePositive": 36, - "wilsonPrecisionLower95": 0.9036 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 40 - }, - "event_boundary": { - "intentsMeeting95Precision": 0, - "macroF1": 0.0, - "macroPrecision": 0.0, - "macroRecall": 0.0, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 2, - "precision": 0.0, - "predictedPositive": 2, - "recall": 0.0, - "total": 40, - "trueNegative": 38, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 40 - }, - "fixed_invitation": { - "intentsMeeting95Precision": 3, - "macroF1": 0.2797, - "macroPrecision": 0.3333, - "macroRecall": 0.2528, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.5965, - "falseNegative": 23, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 17, - "recall": 0.425, - "total": 40, - "trueNegative": 0, - "truePositive": 17, - "wilsonPrecisionLower95": 0.8157 - }, - "question": { - "f1": 0.9333, - "falseNegative": 5, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 35, - "recall": 0.875, - "total": 40, - "trueNegative": 0, - "truePositive": 35, - "wilsonPrecisionLower95": 0.9011 - }, - "replyableMessage": { - "f1": 0.9873, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 39, - "recall": 0.975, - "total": 40, - "trueNegative": 0, - "truePositive": 39, - "wilsonPrecisionLower95": 0.9103 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 40 - }, - "follow_up_instruction": { - "intentsMeeting95Precision": 2, - "macroF1": 0.2153, - "macroPrecision": 0.3333, - "macroRecall": 0.1694, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.7302, - "falseNegative": 17, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 23, - "recall": 0.575, - "total": 40, - "trueNegative": 0, - "truePositive": 23, - "wilsonPrecisionLower95": 0.8569 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.8406, - "falseNegative": 11, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 29, - "recall": 0.725, - "total": 40, - "trueNegative": 0, - "truePositive": 29, - "wilsonPrecisionLower95": 0.883 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.3673, - "falseNegative": 31, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 9, - "recall": 0.225, - "total": 40, - "trueNegative": 0, - "truePositive": 9, - "wilsonPrecisionLower95": 0.7009 - } - }, - "recordCount": 40 - }, - "follow_up_personal_reminder": { - "intentsMeeting95Precision": 1, - "macroF1": 0.097, - "macroPrecision": 0.1111, - "macroRecall": 0.0861, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.8732, - "falseNegative": 9, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 31, - "recall": 0.775, - "total": 40, - "trueNegative": 0, - "truePositive": 31, - "wilsonPrecisionLower95": 0.8897 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 40 - }, - "information_question": { - "intentsMeeting95Precision": 2, - "macroF1": 0.1421, - "macroPrecision": 0.2222, - "macroRecall": 0.1083, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 3, - "precision": 0.0, - "predictedPositive": 3, - "recall": 0.0, - "total": 40, - "trueNegative": 37, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 2, - "precision": 0.0, - "predictedPositive": 2, - "recall": 0.0, - "total": 40, - "trueNegative": 38, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.4906, - "falseNegative": 27, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 13, - "recall": 0.325, - "total": 40, - "trueNegative": 0, - "truePositive": 13, - "wilsonPrecisionLower95": 0.7719 - }, - "replyableMessage": { - "f1": 0.7879, - "falseNegative": 14, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 26, - "recall": 0.65, - "total": 40, - "trueNegative": 0, - "truePositive": 26, - "wilsonPrecisionLower95": 0.8713 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 40 - }, - "negative_news": { - "intentsMeeting95Precision": 0, - "macroF1": 0.0, - "macroPrecision": 0.0, - "macroRecall": 0.0, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 3, - "precision": 0.0, - "predictedPositive": 3, - "recall": 0.0, - "total": 40, - "trueNegative": 37, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 2, - "precision": 0.0, - "predictedPositive": 2, - "recall": 0.0, - "total": 40, - "trueNegative": 38, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 40 - }, - "neutral_fact": { - "intentsMeeting95Precision": 0, - "macroF1": 0.0, - "macroPrecision": 0.0, - "macroRecall": 0.0, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 5, - "precision": 0.0, - "predictedPositive": 5, - "recall": 0.0, - "total": 40, - "trueNegative": 35, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 40 - }, - "positive_feedback": { - "intentsMeeting95Precision": 1, - "macroF1": 0.1068, - "macroPrecision": 0.1111, - "macroRecall": 0.1028, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.961, - "falseNegative": 3, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 37, - "recall": 0.925, - "total": 40, - "trueNegative": 0, - "truePositive": 37, - "wilsonPrecisionLower95": 0.9059 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 40 - }, - "quoted_question_boundary": { - "intentsMeeting95Precision": 0, - "macroF1": 0.0, - "macroPrecision": 0.0, - "macroRecall": 0.0, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 1, - "precision": 0.0, - "predictedPositive": 1, - "recall": 0.0, - "total": 40, - "trueNegative": 39, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 40 - }, - "schedule_negotiation": { - "intentsMeeting95Precision": 3, - "macroF1": 0.259, - "macroPrecision": 0.3333, - "macroRecall": 0.2278, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.8406, - "falseNegative": 11, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 29, - "recall": 0.725, - "total": 40, - "trueNegative": 0, - "truePositive": 29, - "wilsonPrecisionLower95": 0.883 - }, - "replyableMessage": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 40, - "recall": 1.0, - "total": 40, - "trueNegative": 0, - "truePositive": 40, - "wilsonPrecisionLower95": 0.9124 - }, - "scheduleNegotiation": { - "f1": 0.4906, - "falseNegative": 27, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 13, - "recall": 0.325, - "total": 40, - "trueNegative": 0, - "truePositive": 13, - "wilsonPrecisionLower95": 0.7719 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 40 - }, - "task_next_action": { - "intentsMeeting95Precision": 3, - "macroF1": 0.257, - "macroPrecision": 0.3333, - "macroRecall": 0.225, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.9474, - "falseNegative": 4, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 36, - "recall": 0.9, - "total": 40, - "trueNegative": 0, - "truePositive": 36, - "wilsonPrecisionLower95": 0.9036 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.9041, - "falseNegative": 7, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 33, - "recall": 0.825, - "total": 40, - "trueNegative": 0, - "truePositive": 33, - "wilsonPrecisionLower95": 0.8957 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.4615, - "falseNegative": 28, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 12, - "recall": 0.3, - "total": 40, - "trueNegative": 0, - "truePositive": 12, - "wilsonPrecisionLower95": 0.7575 - } - }, - "recordCount": 40 - }, - "task_request": { - "intentsMeeting95Precision": 2, - "macroF1": 0.1659, - "macroPrecision": 0.2222, - "macroRecall": 0.1417, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.9474, - "falseNegative": 4, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 36, - "recall": 0.9, - "total": 40, - "trueNegative": 0, - "truePositive": 36, - "wilsonPrecisionLower95": 0.9036 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.5455, - "falseNegative": 25, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 15, - "recall": 0.375, - "total": 40, - "trueNegative": 0, - "truePositive": 15, - "wilsonPrecisionLower95": 0.7961 - } - }, - "recordCount": 40 - }, - "vague_future_boundary": { - "intentsMeeting95Precision": 0, - "macroF1": 0.0, - "macroPrecision": 0.0, - "macroRecall": 0.0, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 4, - "precision": 0.0, - "predictedPositive": 4, - "recall": 0.0, - "total": 40, - "trueNegative": 36, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 40, - "trueNegative": 40, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 40 - } - }, - "intentsMeeting95Precision": 5, - "macroF1": 0.5709, - "macroPrecision": 0.8569, - "macroRecall": 0.4603, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 680, - "trueNegative": 680, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.3333, - "falseNegative": 32, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 8, - "recall": 0.2, - "total": 680, - "trueNegative": 640, - "truePositive": 8, - "wilsonPrecisionLower95": 0.6756 - }, - "confirmationDecision": { - "f1": 0.8219, - "falseNegative": 10, - "falsePositive": 3, - "precision": 0.9091, - "predictedPositive": 33, - "recall": 0.75, - "total": 680, - "trueNegative": 637, - "truePositive": 30, - "wilsonPrecisionLower95": 0.7643 - }, - "followUpReminder": { - "f1": 0.8491, - "falseNegative": 30, - "falsePositive": 2, - "precision": 0.9783, - "predictedPositive": 92, - "recall": 0.75, - "total": 680, - "trueNegative": 558, - "truePositive": 90, - "wilsonPrecisionLower95": 0.9242 - }, - "invitation": { - "f1": 0.5965, - "falseNegative": 23, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 17, - "recall": 0.425, - "total": 680, - "trueNegative": 640, - "truePositive": 17, - "wilsonPrecisionLower95": 0.8157 - }, - "question": { - "f1": 0.6992, - "falseNegative": 74, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 86, - "recall": 0.5375, - "total": 680, - "trueNegative": 520, - "truePositive": 86, - "wilsonPrecisionLower95": 0.9572 - }, - "replyableMessage": { - "f1": 0.9036, - "falseNegative": 58, - "falsePositive": 15, - "precision": 0.958, - "predictedPositive": 357, - "recall": 0.855, - "total": 680, - "trueNegative": 265, - "truePositive": 342, - "wilsonPrecisionLower95": 0.9318 - }, - "scheduleNegotiation": { - "f1": 0.4727, - "falseNegative": 27, - "falsePositive": 2, - "precision": 0.8667, - "predictedPositive": 15, - "recall": 0.325, - "total": 680, - "trueNegative": 638, - "truePositive": 13, - "wilsonPrecisionLower95": 0.6212 - }, - "task": { - "f1": 0.4615, - "falseNegative": 84, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 36, - "recall": 0.3, - "total": 680, - "trueNegative": 560, - "truePositive": 36, - "wilsonPrecisionLower95": 0.9036 - } - }, - "recordCount": 680 - }, - "researchOnlyComprehensive": { - "byLanguage": { - "en": { - "intentsMeeting95Precision": 0, - "macroF1": 0.2616, - "macroPrecision": 0.3966, - "macroRecall": 0.2659, - "minimumPrecision": 0.0012, - "perIntent": { - "blessing": { - "f1": 0.2308, - "falseNegative": 60, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 9, - "recall": 0.1304, - "total": 13176, - "trueNegative": 13107, - "truePositive": 9, - "wilsonPrecisionLower95": 0.7009 - }, - "complaint": { - "f1": 0.048, - "falseNegative": 421, - "falsePositive": 95, - "precision": 0.1204, - "predictedPositive": 108, - "recall": 0.03, - "total": 13176, - "trueNegative": 12647, - "truePositive": 13, - "wilsonPrecisionLower95": 0.0717 - }, - "confirmationDecision": { - "f1": 0.0024, - "falseNegative": 10, - "falsePositive": 1668, - "precision": 0.0012, - "predictedPositive": 1670, - "recall": 0.1667, - "total": 13176, - "trueNegative": 11496, - "truePositive": 2, - "wilsonPrecisionLower95": 0.0003 - }, - "followUpReminder": { - "f1": 0.1134, - "falseNegative": 114, - "falsePositive": 58, - "precision": 0.1594, - "predictedPositive": 69, - "recall": 0.088, - "total": 13176, - "trueNegative": 12993, - "truePositive": 11, - "wilsonPrecisionLower95": 0.0914 - }, - "invitation": { - "f1": 0.336, - "falseNegative": 40, - "falsePositive": 43, - "precision": 0.3281, - "predictedPositive": 64, - "recall": 0.3443, - "total": 13176, - "trueNegative": 13072, - "truePositive": 21, - "wilsonPrecisionLower95": 0.2257 - }, - "question": { - "f1": 0.4288, - "falseNegative": 1828, - "falsePositive": 378, - "precision": 0.6866, - "predictedPositive": 1206, - "recall": 0.3117, - "total": 13176, - "trueNegative": 10142, - "truePositive": 828, - "wilsonPrecisionLower95": 0.6598 - }, - "replyableMessage": { - "f1": 0.7642, - "falseNegative": 2967, - "falsePositive": 1917, - "precision": 0.805, - "predictedPositive": 9831, - "recall": 0.7273, - "total": 13176, - "trueNegative": 378, - "truePositive": 7914, - "wilsonPrecisionLower95": 0.7971 - }, - "scheduleNegotiation": { - "f1": 0.1683, - "falseNegative": 41, - "falsePositive": 216, - "precision": 0.1074, - "predictedPositive": 242, - "recall": 0.3881, - "total": 13176, - "trueNegative": 12893, - "truePositive": 26, - "wilsonPrecisionLower95": 0.0744 - }, - "task": { - "f1": 0.2628, - "falseNegative": 888, - "falsePositive": 408, - "precision": 0.3615, - "predictedPositive": 639, - "recall": 0.2064, - "total": 13176, - "trueNegative": 11649, - "truePositive": 231, - "wilsonPrecisionLower95": 0.3252 - } - }, - "recordCount": 13176 - }, - "zh-Hans": { - "intentsMeeting95Precision": 0, - "macroF1": 0.1774, - "macroPrecision": 0.3011, - "macroRecall": 0.149, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 17, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 28019, - "trueNegative": 28002, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.4077, - "falseNegative": 154, - "falsePositive": 183, - "precision": 0.388, - "predictedPositive": 299, - "recall": 0.4296, - "total": 28019, - "trueNegative": 27566, - "truePositive": 116, - "wilsonPrecisionLower95": 0.3345 - }, - "confirmationDecision": { - "f1": 0.0909, - "falseNegative": 53, - "falsePositive": 27, - "precision": 0.129, - "predictedPositive": 31, - "recall": 0.0702, - "total": 28019, - "trueNegative": 27935, - "truePositive": 4, - "wilsonPrecisionLower95": 0.0513 - }, - "followUpReminder": { - "f1": 0.1856, - "falseNegative": 234, - "falsePositive": 38, - "precision": 0.4493, - "predictedPositive": 69, - "recall": 0.117, - "total": 28019, - "trueNegative": 27716, - "truePositive": 31, - "wilsonPrecisionLower95": 0.3377 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 73, - "falsePositive": 9, - "precision": 0.0, - "predictedPositive": 9, - "recall": 0.0, - "total": 28019, - "trueNegative": 27937, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "question": { - "f1": 0.1419, - "falseNegative": 6619, - "falsePositive": 216, - "precision": 0.7234, - "predictedPositive": 781, - "recall": 0.0786, - "total": 28019, - "trueNegative": 20619, - "truePositive": 565, - "wilsonPrecisionLower95": 0.691 - }, - "replyableMessage": { - "f1": 0.7622, - "falseNegative": 9480, - "falsePositive": 1096, - "precision": 0.9393, - "predictedPositive": 18042, - "recall": 0.6413, - "total": 28019, - "trueNegative": 497, - "truePositive": 16946, - "wilsonPrecisionLower95": 0.9357 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 4, - "falsePositive": 2, - "precision": 0.0, - "predictedPositive": 2, - "recall": 0.0, - "total": 28019, - "trueNegative": 28013, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "task": { - "f1": 0.0086, - "falseNegative": 656, - "falsePositive": 34, - "precision": 0.0811, - "predictedPositive": 37, - "recall": 0.0046, - "total": 28019, - "trueNegative": 27326, - "truePositive": 3, - "wilsonPrecisionLower95": 0.028 - } - }, - "recordCount": 28019 - } - }, - "bySource": { - "AmazonScience/MASSIVE en-US test": { - "intentsMeeting95Precision": 2, - "macroF1": 0.1614, - "macroPrecision": 0.2661, - "macroRecall": 0.1233, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 2967, - "trueNegative": 2967, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 17, - "precision": 0.0, - "predictedPositive": 17, - "recall": 0.0, - "total": 2967, - "trueNegative": 2950, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 347, - "precision": 0.0, - "predictedPositive": 347, - "recall": 0.0, - "total": 2967, - "trueNegative": 2620, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "followUpReminder": { - "f1": 0.1447, - "falseNegative": 86, - "falsePositive": 44, - "precision": 0.2, - "predictedPositive": 55, - "recall": 0.1134, - "total": 2967, - "trueNegative": 2826, - "truePositive": 11, - "wilsonPrecisionLower95": 0.1155 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 7, - "precision": 0.0, - "predictedPositive": 7, - "recall": 0.0, - "total": 2967, - "trueNegative": 2960, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "question": { - "f1": 0.3532, - "falseNegative": 1266, - "falsePositive": 16, - "precision": 0.9563, - "predictedPositive": 366, - "recall": 0.2166, - "total": 2967, - "trueNegative": 1335, - "truePositive": 350, - "wilsonPrecisionLower95": 0.9302 - }, - "replyableMessage": { - "f1": 0.7951, - "falseNegative": 1009, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 1958, - "recall": 0.6599, - "total": 2967, - "trueNegative": 0, - "truePositive": 1958, - "wilsonPrecisionLower95": 0.998 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 101, - "precision": 0.0, - "predictedPositive": 101, - "recall": 0.0, - "total": 2967, - "trueNegative": 2866, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "task": { - "f1": 0.1599, - "falseNegative": 476, - "falsePositive": 207, - "precision": 0.239, - "predictedPositive": 272, - "recall": 0.1201, - "total": 2967, - "trueNegative": 2219, - "truePositive": 65, - "wilsonPrecisionLower95": 0.1921 - } - }, - "recordCount": 2967 - }, - "AmazonScience/MASSIVE zh-CN test": { - "intentsMeeting95Precision": 1, - "macroF1": 0.1368, - "macroPrecision": 0.2943, - "macroRecall": 0.0998, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 2850, - "trueNegative": 2850, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 2, - "precision": 0.0, - "predictedPositive": 2, - "recall": 0.0, - "total": 2850, - "trueNegative": 2848, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 4, - "precision": 0.0, - "predictedPositive": 4, - "recall": 0.0, - "total": 2850, - "trueNegative": 2846, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "followUpReminder": { - "f1": 0.2537, - "falseNegative": 140, - "falsePositive": 13, - "precision": 0.6667, - "predictedPositive": 39, - "recall": 0.1566, - "total": 2850, - "trueNegative": 2671, - "truePositive": 26, - "wilsonPrecisionLower95": 0.5098 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 7, - "precision": 0.0, - "predictedPositive": 7, - "recall": 0.0, - "total": 2850, - "trueNegative": 2843, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "question": { - "f1": 0.2054, - "falseNegative": 1381, - "falsePositive": 19, - "precision": 0.905, - "predictedPositive": 200, - "recall": 0.1159, - "total": 2850, - "trueNegative": 1269, - "truePositive": 181, - "wilsonPrecisionLower95": 0.8564 - }, - "replyableMessage": { - "f1": 0.7684, - "falseNegative": 1072, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 1778, - "recall": 0.6239, - "total": 2850, - "trueNegative": 0, - "truePositive": 1778, - "wilsonPrecisionLower95": 0.9978 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 2, - "precision": 0.0, - "predictedPositive": 2, - "recall": 0.0, - "total": 2850, - "trueNegative": 2848, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "task": { - "f1": 0.0041, - "falseNegative": 477, - "falsePositive": 12, - "precision": 0.0769, - "predictedPositive": 13, - "recall": 0.0021, - "total": 2850, - "trueNegative": 2360, - "truePositive": 1, - "wilsonPrecisionLower95": 0.0137 - } - }, - "recordCount": 2850 - }, - "CFPB via hpe-ai/customer-complaints": { - "intentsMeeting95Precision": 1, - "macroF1": 0.0837, - "macroPrecision": 0.2593, - "macroRecall": 0.063, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 250, - "trueNegative": 250, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0159, - "falseNegative": 248, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 2, - "recall": 0.008, - "total": 250, - "trueNegative": 0, - "truePositive": 2, - "wilsonPrecisionLower95": 0.3424 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 7, - "precision": 0.0, - "predictedPositive": 7, - "recall": 0.0, - "total": 250, - "trueNegative": 243, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 250, - "trueNegative": 250, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 250, - "trueNegative": 250, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 17, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 250, - "trueNegative": 233, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.7013, - "falseNegative": 115, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 135, - "recall": 0.54, - "total": 250, - "trueNegative": 0, - "truePositive": 135, - "wilsonPrecisionLower95": 0.9723 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 250, - "trueNegative": 250, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0357, - "falseNegative": 52, - "falsePositive": 2, - "precision": 0.3333, - "predictedPositive": 3, - "recall": 0.0189, - "total": 250, - "trueNegative": 195, - "truePositive": 1, - "wilsonPrecisionLower95": 0.0615 - } - }, - "recordCount": 250 - }, - "CPED test split": { - "intentsMeeting95Precision": 0, - "macroF1": 0.1219, - "macroPrecision": 0.2231, - "macroRecall": 0.096, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 17, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 25049, - "trueNegative": 25032, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.018, - "falseNegative": 147, - "falsePositive": 181, - "precision": 0.0163, - "predictedPositive": 184, - "recall": 0.02, - "total": 25049, - "trueNegative": 24718, - "truePositive": 3, - "wilsonPrecisionLower95": 0.0056 - }, - "confirmationDecision": { - "f1": 0.0952, - "falseNegative": 53, - "falsePositive": 23, - "precision": 0.1481, - "predictedPositive": 27, - "recall": 0.0702, - "total": 25049, - "trueNegative": 24969, - "truePositive": 4, - "wilsonPrecisionLower95": 0.0592 - }, - "followUpReminder": { - "f1": 0.0775, - "falseNegative": 94, - "falsePositive": 25, - "precision": 0.1667, - "predictedPositive": 30, - "recall": 0.0505, - "total": 25049, - "trueNegative": 24925, - "truePositive": 5, - "wilsonPrecisionLower95": 0.0734 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 73, - "falsePositive": 2, - "precision": 0.0, - "predictedPositive": 2, - "recall": 0.0, - "total": 25049, - "trueNegative": 24974, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "question": { - "f1": 0.1245, - "falseNegative": 5202, - "falsePositive": 197, - "precision": 0.6609, - "predictedPositive": 581, - "recall": 0.0687, - "total": 25049, - "trueNegative": 19266, - "truePositive": 384, - "wilsonPrecisionLower95": 0.6215 - }, - "replyableMessage": { - "f1": 0.7609, - "falseNegative": 8378, - "falsePositive": 1096, - "precision": 0.9322, - "predictedPositive": 16174, - "recall": 0.6428, - "total": 25049, - "trueNegative": 497, - "truePositive": 15078, - "wilsonPrecisionLower95": 0.9283 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 4, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 25049, - "trueNegative": 25045, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0208, - "falseNegative": 166, - "falsePositive": 22, - "precision": 0.0833, - "predictedPositive": 24, - "recall": 0.0119, - "total": 25049, - "trueNegative": 24859, - "truePositive": 2, - "wilsonPrecisionLower95": 0.0232 - } - }, - "recordCount": 25049 - }, - "Google Research GoEmotions test": { - "intentsMeeting95Precision": 1, - "macroF1": 0.1942, - "macroPrecision": 0.34, - "macroRecall": 0.1531, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.1918, - "falseNegative": 59, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 7, - "recall": 0.1061, - "total": 5418, - "trueNegative": 5352, - "truePositive": 7, - "wilsonPrecisionLower95": 0.6457 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 20, - "falsePositive": 27, - "precision": 0.0, - "predictedPositive": 27, - "recall": 0.0, - "total": 5418, - "trueNegative": 5371, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 881, - "precision": 0.0, - "predictedPositive": 881, - "recall": 0.0, - "total": 5418, - "trueNegative": 4537, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 6, - "precision": 0.0, - "predictedPositive": 6, - "recall": 0.0, - "total": 5418, - "trueNegative": 5412, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 8, - "precision": 0.0, - "predictedPositive": 8, - "recall": 0.0, - "total": 5418, - "trueNegative": 5410, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "question": { - "f1": 0.3991, - "falseNegative": 363, - "falsePositive": 146, - "precision": 0.5365, - "predictedPositive": 315, - "recall": 0.3177, - "total": 5418, - "trueNegative": 4740, - "truePositive": 169, - "wilsonPrecisionLower95": 0.4813 - }, - "replyableMessage": { - "f1": 0.8467, - "falseNegative": 1440, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 3978, - "recall": 0.7342, - "total": 5418, - "trueNegative": 0, - "truePositive": 3978, - "wilsonPrecisionLower95": 0.999 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 33, - "precision": 0.0, - "predictedPositive": 33, - "recall": 0.0, - "total": 5418, - "trueNegative": 5385, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "task": { - "f1": 0.3099, - "falseNegative": 39, - "falsePositive": 10, - "precision": 0.5238, - "predictedPositive": 21, - "recall": 0.22, - "total": 5418, - "trueNegative": 5358, - "truePositive": 11, - "wilsonPrecisionLower95": 0.3237 - } - }, - "recordCount": 5418 - }, - "Meituan-Dianping/ASAP test": { - "intentsMeeting95Precision": 2, - "macroF1": 0.203, - "macroPrecision": 0.2222, - "macroRecall": 0.188, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 120, - "trueNegative": 120, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.97, - "falseNegative": 7, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 113, - "recall": 0.9417, - "total": 120, - "trueNegative": 0, - "truePositive": 113, - "wilsonPrecisionLower95": 0.9671 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 120, - "trueNegative": 120, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 120, - "trueNegative": 120, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 120, - "trueNegative": 120, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 36, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 120, - "trueNegative": 84, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.8571, - "falseNegative": 30, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 90, - "recall": 0.75, - "total": 120, - "trueNegative": 0, - "truePositive": 90, - "wilsonPrecisionLower95": 0.9591 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 120, - "trueNegative": 120, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 13, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 120, - "trueNegative": 107, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 120 - }, - "Sulak2020 GitHub issues multi-repo dataset": { - "intentsMeeting95Precision": 1, - "macroF1": 0.28, - "macroPrecision": 0.415, - "macroRecall": 0.216, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.6667, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 1, - "recall": 0.5, - "total": 1004, - "trueNegative": 1002, - "truePositive": 1, - "wilsonPrecisionLower95": 0.2065 - }, - "complaint": { - "f1": 0.0995, - "falseNegative": 153, - "falsePositive": 46, - "precision": 0.193, - "predictedPositive": 57, - "recall": 0.0671, - "total": 1004, - "trueNegative": 794, - "truePositive": 11, - "wilsonPrecisionLower95": 0.1113 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 2, - "falsePositive": 61, - "precision": 0.0, - "predictedPositive": 61, - "recall": 0.0, - "total": 1004, - "trueNegative": 941, - "truePositive": 0, - "wilsonPrecisionLower95": -0.0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 2, - "falsePositive": 1, - "precision": 0.0, - "predictedPositive": 1, - "recall": 0.0, - "total": 1004, - "trueNegative": 1001, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 2, - "precision": 0.0, - "predictedPositive": 2, - "recall": 0.0, - "total": 1004, - "trueNegative": 1002, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "question": { - "f1": 0.4474, - "falseNegative": 39, - "falsePositive": 3, - "precision": 0.85, - "predictedPositive": 20, - "recall": 0.3036, - "total": 1004, - "trueNegative": 945, - "truePositive": 17, - "wilsonPrecisionLower95": 0.6396 - }, - "replyableMessage": { - "f1": 0.8326, - "falseNegative": 288, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 716, - "recall": 0.7131, - "total": 1004, - "trueNegative": 0, - "truePositive": 716, - "wilsonPrecisionLower95": 0.9947 - }, - "scheduleNegotiation": { - "f1": 0.4737, - "falseNegative": 16, - "falsePositive": 4, - "precision": 0.6923, - "predictedPositive": 13, - "recall": 0.36, - "total": 1004, - "trueNegative": 975, - "truePositive": 9, - "wilsonPrecisionLower95": 0.4237 - }, - "task": { - "f1": 0.0, - "falseNegative": 90, - "falsePositive": 3, - "precision": 0.0, - "predictedPositive": 3, - "recall": 0.0, - "total": 1004, - "trueNegative": 911, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - } - }, - "recordCount": 1004 - }, - "Verified Enron Intent Dataset": { - "intentsMeeting95Precision": 0, - "macroF1": 0.3577, - "macroPrecision": 0.3369, - "macroRecall": 0.4364, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 1, - "recall": 1.0, - "total": 3537, - "trueNegative": 3536, - "truePositive": 1, - "wilsonPrecisionLower95": 0.2065 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 5, - "precision": 0.0, - "predictedPositive": 5, - "recall": 0.0, - "total": 3537, - "trueNegative": 3532, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "confirmationDecision": { - "f1": 0.0104, - "falseNegative": 8, - "falsePositive": 372, - "precision": 0.0053, - "predictedPositive": 374, - "recall": 0.2, - "total": 3537, - "trueNegative": 3155, - "truePositive": 2, - "wilsonPrecisionLower95": 0.0015 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 26, - "falsePositive": 7, - "precision": 0.0, - "predictedPositive": 7, - "recall": 0.0, - "total": 3537, - "trueNegative": 3504, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "invitation": { - "f1": 0.3889, - "falseNegative": 40, - "falsePositive": 26, - "precision": 0.4468, - "predictedPositive": 47, - "recall": 0.3443, - "total": 3537, - "trueNegative": 3450, - "truePositive": 21, - "wilsonPrecisionLower95": 0.3141 - }, - "question": { - "f1": 0.6213, - "falseNegative": 143, - "falsePositive": 213, - "precision": 0.5782, - "predictedPositive": 505, - "recall": 0.6713, - "total": 3537, - "trueNegative": 2889, - "truePositive": 292, - "wilsonPrecisionLower95": 0.5347 - }, - "replyableMessage": { - "f1": 0.5259, - "falseNegative": 115, - "falsePositive": 1917, - "precision": 0.3702, - "predictedPositive": 3044, - "recall": 0.9074, - "total": 3537, - "trueNegative": 378, - "truePositive": 1127, - "wilsonPrecisionLower95": 0.3533 - }, - "scheduleNegotiation": { - "f1": 0.2482, - "falseNegative": 25, - "falsePositive": 78, - "precision": 0.1789, - "predictedPositive": 95, - "recall": 0.4048, - "total": 3537, - "trueNegative": 3417, - "truePositive": 17, - "wilsonPrecisionLower95": 0.1148 - }, - "task": { - "f1": 0.4248, - "falseNegative": 231, - "falsePositive": 186, - "precision": 0.4529, - "predictedPositive": 340, - "recall": 0.4, - "total": 3537, - "trueNegative": 2966, - "truePositive": 154, - "wilsonPrecisionLower95": 0.4008 - } - }, - "recordCount": 3537 - } - }, - "intentsMeeting95Precision": 0, - "macroF1": 0.2383, - "macroPrecision": 0.4398, - "macroRecall": 0.2161, - "minimumPrecision": 0.0035, - "perIntent": { - "blessing": { - "f1": 0.1895, - "falseNegative": 77, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 9, - "recall": 0.1047, - "total": 41195, - "trueNegative": 41109, - "truePositive": 9, - "wilsonPrecisionLower95": 0.7009 - }, - "complaint": { - "f1": 0.2322, - "falseNegative": 575, - "falsePositive": 278, - "precision": 0.317, - "predictedPositive": 407, - "recall": 0.1832, - "total": 41195, - "trueNegative": 40213, - "truePositive": 129, - "wilsonPrecisionLower95": 0.2736 - }, - "confirmationDecision": { - "f1": 0.0068, - "falseNegative": 63, - "falsePositive": 1695, - "precision": 0.0035, - "predictedPositive": 1701, - "recall": 0.087, - "total": 41195, - "trueNegative": 39431, - "truePositive": 6, - "wilsonPrecisionLower95": 0.0016 - }, - "followUpReminder": { - "f1": 0.1591, - "falseNegative": 348, - "falsePositive": 96, - "precision": 0.3043, - "predictedPositive": 138, - "recall": 0.1077, - "total": 41195, - "trueNegative": 40709, - "truePositive": 42, - "wilsonPrecisionLower95": 0.2337 - }, - "invitation": { - "f1": 0.2029, - "falseNegative": 113, - "falsePositive": 52, - "precision": 0.2877, - "predictedPositive": 73, - "recall": 0.1567, - "total": 41195, - "trueNegative": 41009, - "truePositive": 21, - "wilsonPrecisionLower95": 0.1965 - }, - "question": { - "f1": 0.2356, - "falseNegative": 8447, - "falsePositive": 594, - "precision": 0.7011, - "predictedPositive": 1987, - "recall": 0.1416, - "total": 41195, - "trueNegative": 30761, - "truePositive": 1393, - "wilsonPrecisionLower95": 0.6806 - }, - "replyableMessage": { - "f1": 0.7628, - "falseNegative": 12447, - "falsePositive": 3013, - "precision": 0.8919, - "predictedPositive": 27873, - "recall": 0.6664, - "total": 41195, - "trueNegative": 875, - "truePositive": 24860, - "wilsonPrecisionLower95": 0.8882 - }, - "scheduleNegotiation": { - "f1": 0.1651, - "falseNegative": 45, - "falsePositive": 218, - "precision": 0.1066, - "predictedPositive": 244, - "recall": 0.3662, - "total": 41195, - "trueNegative": 40906, - "truePositive": 26, - "wilsonPrecisionLower95": 0.0738 - }, - "task": { - "f1": 0.1907, - "falseNegative": 1544, - "falsePositive": 442, - "precision": 0.3462, - "predictedPositive": 676, - "recall": 0.1316, - "total": 41195, - "trueNegative": 38975, - "truePositive": 234, - "wilsonPrecisionLower95": 0.3113 - } - }, - "recordCount": 41195 - }, - "targetedRelease": { - "byLanguage": { - "en": { - "intentsMeeting95Precision": 4, - "macroF1": 0.4138, - "macroPrecision": 0.5476, - "macroRecall": 0.3512, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 320, - "trueNegative": 320, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.6957, - "falseNegative": 56, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 64, - "recall": 0.5333, - "total": 320, - "trueNegative": 200, - "truePositive": 64, - "wilsonPrecisionLower95": 0.9434 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 4, - "precision": 0.0, - "predictedPositive": 4, - "recall": 0.0, - "total": 320, - "trueNegative": 316, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "followUpReminder": { - "f1": 0.961, - "falseNegative": 3, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 37, - "recall": 0.925, - "total": 320, - "trueNegative": 280, - "truePositive": 37, - "wilsonPrecisionLower95": 0.9059 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 320, - "trueNegative": 320, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.4762, - "falseNegative": 55, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 25, - "recall": 0.3125, - "total": 320, - "trueNegative": 240, - "truePositive": 25, - "wilsonPrecisionLower95": 0.8668 - }, - "replyableMessage": { - "f1": 0.8819, - "falseNegative": 32, - "falsePositive": 13, - "precision": 0.9282, - "predictedPositive": 181, - "recall": 0.84, - "total": 320, - "trueNegative": 107, - "truePositive": 168, - "wilsonPrecisionLower95": 0.881 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 320, - "trueNegative": 320, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.7097, - "falseNegative": 36, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 44, - "recall": 0.55, - "total": 320, - "trueNegative": 240, - "truePositive": 44, - "wilsonPrecisionLower95": 0.9197 - } - }, - "recordCount": 320 - }, - "zh-Hans": { - "intentsMeeting95Precision": 4, - "macroF1": 0.3345, - "macroPrecision": 0.5501, - "macroRecall": 0.2766, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 320, - "trueNegative": 320, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.4516, - "falseNegative": 85, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 35, - "recall": 0.2917, - "total": 320, - "trueNegative": 200, - "truePositive": 35, - "wilsonPrecisionLower95": 0.9011 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 320, - "trueNegative": 320, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.2609, - "falseNegative": 34, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 6, - "recall": 0.15, - "total": 320, - "trueNegative": 280, - "truePositive": 6, - "wilsonPrecisionLower95": 0.6097 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 320, - "trueNegative": 320, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.8811, - "falseNegative": 17, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 63, - "recall": 0.7875, - "total": 320, - "trueNegative": 240, - "truePositive": 63, - "wilsonPrecisionLower95": 0.9425 - }, - "replyableMessage": { - "f1": 0.9552, - "falseNegative": 8, - "falsePositive": 10, - "precision": 0.9505, - "predictedPositive": 202, - "recall": 0.96, - "total": 320, - "trueNegative": 110, - "truePositive": 192, - "wilsonPrecisionLower95": 0.9113 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 320, - "trueNegative": 320, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.4615, - "falseNegative": 56, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 24, - "recall": 0.3, - "total": 320, - "trueNegative": 240, - "truePositive": 24, - "wilsonPrecisionLower95": 0.862 - } - }, - "recordCount": 320 - } - }, - "bySource": { - "release_complaint_brief": { - "intentsMeeting95Precision": 2, - "macroF1": 0.1893, - "macroPrecision": 0.2222, - "macroRecall": 0.1708, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.7097, - "falseNegative": 36, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 44, - "recall": 0.55, - "total": 80, - "trueNegative": 0, - "truePositive": 44, - "wilsonPrecisionLower95": 0.9197 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.9937, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 79, - "recall": 0.9875, - "total": 80, - "trueNegative": 0, - "truePositive": 79, - "wilsonPrecisionLower95": 0.9536 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 80 - }, - "release_complaint_question": { - "intentsMeeting95Precision": 3, - "macroF1": 0.2152, - "macroPrecision": 0.3333, - "macroRecall": 0.1667, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.5586, - "falseNegative": 49, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 31, - "recall": 0.3875, - "total": 80, - "trueNegative": 0, - "truePositive": 31, - "wilsonPrecisionLower95": 0.8897 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.5047, - "falseNegative": 53, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 27, - "recall": 0.3375, - "total": 80, - "trueNegative": 0, - "truePositive": 27, - "wilsonPrecisionLower95": 0.8754 - }, - "replyableMessage": { - "f1": 0.8732, - "falseNegative": 18, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 62, - "recall": 0.775, - "total": 80, - "trueNegative": 0, - "truePositive": 62, - "wilsonPrecisionLower95": 0.9417 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 80 - }, - "release_complaint_unspoken": { - "intentsMeeting95Precision": 2, - "macroF1": 0.1565, - "macroPrecision": 0.2222, - "macroRecall": 0.1333, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.4615, - "falseNegative": 56, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 24, - "recall": 0.3, - "total": 80, - "trueNegative": 0, - "truePositive": 24, - "wilsonPrecisionLower95": 0.862 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.9474, - "falseNegative": 8, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 72, - "recall": 0.9, - "total": 80, - "trueNegative": 0, - "truePositive": 72, - "wilsonPrecisionLower95": 0.9493 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 80 - }, - "release_negative_boundary": { - "intentsMeeting95Precision": 0, - "macroF1": 0.0, - "macroPrecision": 0.0, - "macroRecall": 0.0, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 80 - }, - "release_personal_boundary": { - "intentsMeeting95Precision": 1, - "macroF1": 0.0777, - "macroPrecision": 0.1111, - "macroRecall": 0.0597, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.6992, - "falseNegative": 37, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 43, - "recall": 0.5375, - "total": 80, - "trueNegative": 0, - "truePositive": 43, - "wilsonPrecisionLower95": 0.918 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 4, - "precision": 0.0, - "predictedPositive": 4, - "recall": 0.0, - "total": 80, - "trueNegative": 76, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 80 - }, - "release_resolved_boundary": { - "intentsMeeting95Precision": 0, - "macroF1": 0.0, - "macroPrecision": 0.0, - "macroRecall": 0.0, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 19, - "precision": 0.0, - "predictedPositive": 19, - "recall": 0.0, - "total": 80, - "trueNegative": 61, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - } - }, - "recordCount": 80 - }, - "release_task_owner": { - "intentsMeeting95Precision": 2, - "macroF1": 0.171, - "macroPrecision": 0.2222, - "macroRecall": 0.1514, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 4, - "precision": 0.0, - "predictedPositive": 4, - "recall": 0.0, - "total": 80, - "trueNegative": 76, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "replyableMessage": { - "f1": 0.9937, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 79, - "recall": 0.9875, - "total": 80, - "trueNegative": 0, - "truePositive": 79, - "wilsonPrecisionLower95": 0.9536 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.5455, - "falseNegative": 50, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 30, - "recall": 0.375, - "total": 80, - "trueNegative": 0, - "truePositive": 30, - "wilsonPrecisionLower95": 0.8865 - } - }, - "recordCount": 80 - }, - "release_task_polite_request": { - "intentsMeeting95Precision": 3, - "macroF1": 0.2698, - "macroPrecision": 0.3333, - "macroRecall": 0.2319, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "followUpReminder": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.8652, - "falseNegative": 19, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 61, - "recall": 0.7625, - "total": 80, - "trueNegative": 0, - "truePositive": 61, - "wilsonPrecisionLower95": 0.9408 - }, - "replyableMessage": { - "f1": 0.9189, - "falseNegative": 12, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 68, - "recall": 0.85, - "total": 80, - "trueNegative": 0, - "truePositive": 68, - "wilsonPrecisionLower95": 0.9465 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 80, - "trueNegative": 80, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.6441, - "falseNegative": 42, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 38, - "recall": 0.475, - "total": 80, - "trueNegative": 0, - "truePositive": 38, - "wilsonPrecisionLower95": 0.9082 - } - }, - "recordCount": 80 - } - }, - "intentsMeeting95Precision": 4, - "macroF1": 0.3899, - "macroPrecision": 0.5489, - "macroRecall": 0.3139, - "minimumPrecision": 0.0, - "perIntent": { - "blessing": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 640, - "trueNegative": 640, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "complaint": { - "f1": 0.5841, - "falseNegative": 141, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 99, - "recall": 0.4125, - "total": 640, - "trueNegative": 400, - "truePositive": 99, - "wilsonPrecisionLower95": 0.9626 - }, - "confirmationDecision": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 4, - "precision": 0.0, - "predictedPositive": 4, - "recall": 0.0, - "total": 640, - "trueNegative": 636, - "truePositive": 0, - "wilsonPrecisionLower95": 0.0 - }, - "followUpReminder": { - "f1": 0.6992, - "falseNegative": 37, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 43, - "recall": 0.5375, - "total": 640, - "trueNegative": 560, - "truePositive": 43, - "wilsonPrecisionLower95": 0.918 - }, - "invitation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 640, - "trueNegative": 640, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "question": { - "f1": 0.7097, - "falseNegative": 72, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 88, - "recall": 0.55, - "total": 640, - "trueNegative": 480, - "truePositive": 88, - "wilsonPrecisionLower95": 0.9582 - }, - "replyableMessage": { - "f1": 0.9195, - "falseNegative": 40, - "falsePositive": 23, - "precision": 0.9399, - "predictedPositive": 383, - "recall": 0.9, - "total": 640, - "trueNegative": 217, - "truePositive": 360, - "wilsonPrecisionLower95": 0.9115 - }, - "scheduleNegotiation": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 640, - "trueNegative": 640, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "task": { - "f1": 0.5965, - "falseNegative": 92, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 68, - "recall": 0.425, - "total": 640, - "trueNegative": 480, - "truePositive": 68, - "wilsonPrecisionLower95": 0.9465 - } - }, - "recordCount": 640 - } - }, - "generatedAtUnix": 1787825530, - "openTrainingRecords": 3845, - "overlapChecks": { - "ModelTraining/ClipboardSemantics/comprehensive-online-holdout-corpus.jsonl": 0, - "ModelTraining/ClipboardSemantics/random-holdout-corpus.jsonl": 0, - "ModelTraining/ClipboardSemantics/targeted-release-holdout-corpus.jsonl": 0 - }, - "platformLimitation": "Linux surrogate research only. Apple Create ML/Core ML artifacts must be retrained and tested on macOS before deployment.", - "releaseGatePassed": false, - "roundCount": 20, - "schemaVersion": 1, - "seed": 20260827, - "selectedConfiguration": { - "alpha": 2e-05, - "augmentation": "none", - "char_max": 5, - "char_min": 2, - "external_weight": 0.5700000000000001, - "hard_example_weight": 1.0, - "l1_ratio": 0.0, - "max_char_features": 45000, - "max_word_features": 22000, - "min_df": 2, - "pseudo_weight": 0.16499999999999998, - "round": 1, - "word_max": 2 - }, - "selectedRound": 1, - "selectedThresholds": { - "blessing": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9318, - "falseNegative": 4, - "falsePositive": 2, - "precision": 0.9535, - "predictedPositive": 43, - "recall": 0.9111, - "total": 1485, - "trueNegative": 1438, - "truePositive": 41, - "wilsonPrecisionLower95": 0.8454 - }, - "threshold": 0.915 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7568, - "falseNegative": 17, - "falsePositive": 1, - "precision": 0.9655, - "predictedPositive": 29, - "recall": 0.6222, - "total": 1485, - "trueNegative": 1439, - "truePositive": 28, - "wilsonPrecisionLower95": 0.8282 - }, - "threshold": 0.9 - } - }, - "metrics": { - "f1": 0.8375, - "falseNegative": 23, - "falsePositive": 3, - "precision": 0.9571, - "predictedPositive": 70, - "recall": 0.7444, - "total": 2970, - "trueNegative": 2877, - "truePositive": 67, - "wilsonPrecisionLower95": 0.8814 - }, - "threshold": 0.915 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 180, - "recall": 1.0, - "total": 1485, - "trueNegative": 1305, - "truePositive": 180, - "wilsonPrecisionLower95": 0.9791 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9526, - "falseNegative": 9, - "falsePositive": 8, - "precision": 0.9553, - "predictedPositive": 179, - "recall": 0.95, - "total": 1485, - "trueNegative": 1297, - "truePositive": 171, - "wilsonPrecisionLower95": 0.9143 - }, - "threshold": 0.54 - } - }, - "metrics": { - "f1": 0.9764, - "falseNegative": 9, - "falsePositive": 8, - "precision": 0.9777, - "predictedPositive": 359, - "recall": 0.975, - "total": 2970, - "trueNegative": 2602, - "truePositive": 351, - "wilsonPrecisionLower95": 0.9567 - }, - "threshold": 0.54 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8848, - "falseNegative": 17, - "falsePositive": 2, - "precision": 0.9733, - "predictedPositive": 75, - "recall": 0.8111, - "total": 1485, - "trueNegative": 1393, - "truePositive": 73, - "wilsonPrecisionLower95": 0.9079 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7947, - "falseNegative": 30, - "falsePositive": 1, - "precision": 0.9836, - "predictedPositive": 61, - "recall": 0.6667, - "total": 1485, - "trueNegative": 1394, - "truePositive": 60, - "wilsonPrecisionLower95": 0.9128 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.8418, - "falseNegative": 47, - "falsePositive": 3, - "precision": 0.9779, - "predictedPositive": 136, - "recall": 0.7389, - "total": 2970, - "trueNegative": 2787, - "truePositive": 133, - "wilsonPrecisionLower95": 0.9372 - }, - "threshold": 0.5 - }, - "followUpReminder": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8554, - "falseNegative": 41, - "falsePositive": 6, - "precision": 0.9586, - "predictedPositive": 145, - "recall": 0.7722, - "total": 1485, - "trueNegative": 1299, - "truePositive": 139, - "wilsonPrecisionLower95": 0.9127 - }, - "threshold": 0.83 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8354, - "falseNegative": 48, - "falsePositive": 4, - "precision": 0.9706, - "predictedPositive": 136, - "recall": 0.7333, - "total": 1485, - "trueNegative": 1301, - "truePositive": 132, - "wilsonPrecisionLower95": 0.9268 - }, - "threshold": 0.505 - } - }, - "metrics": { - "f1": 0.7994, - "falseNegative": 113, - "falsePositive": 11, - "precision": 0.9574, - "predictedPositive": 258, - "recall": 0.6861, - "total": 2970, - "trueNegative": 2599, - "truePositive": 247, - "wilsonPrecisionLower95": 0.9253 - }, - "threshold": 0.815 - }, - "invitation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8302, - "falseNegative": 24, - "falsePositive": 3, - "precision": 0.9565, - "predictedPositive": 69, - "recall": 0.7333, - "total": 1485, - "trueNegative": 1392, - "truePositive": 66, - "wilsonPrecisionLower95": 0.8798 - }, - "threshold": 0.935 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6765, - "falseNegative": 44, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 46, - "recall": 0.5111, - "total": 1485, - "trueNegative": 1395, - "truePositive": 46, - "wilsonPrecisionLower95": 0.9229 - }, - "threshold": 0.505 - } - }, - "metrics": { - "f1": 0.5476, - "falseNegative": 111, - "falsePositive": 3, - "precision": 0.9583, - "predictedPositive": 72, - "recall": 0.3833, - "total": 2970, - "trueNegative": 2787, - "truePositive": 69, - "wilsonPrecisionLower95": 0.8845 - }, - "threshold": 0.935 - }, - "question": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8445, - "falseNegative": 76, - "falsePositive": 12, - "precision": 0.9522, - "predictedPositive": 251, - "recall": 0.7587, - "total": 1485, - "trueNegative": 1158, - "truePositive": 239, - "wilsonPrecisionLower95": 0.9183 - }, - "threshold": 0.725 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8536, - "falseNegative": 76, - "falsePositive": 6, - "precision": 0.9755, - "predictedPositive": 245, - "recall": 0.7587, - "total": 1485, - "trueNegative": 1164, - "truePositive": 239, - "wilsonPrecisionLower95": 0.9476 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.8729, - "falseNegative": 122, - "falsePositive": 26, - "precision": 0.9513, - "predictedPositive": 534, - "recall": 0.8063, - "total": 2970, - "trueNegative": 2314, - "truePositive": 508, - "wilsonPrecisionLower95": 0.9296 - }, - "threshold": 0.515 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.948, - "falseNegative": 52, - "falsePositive": 46, - "precision": 0.951, - "predictedPositive": 939, - "recall": 0.945, - "total": 1485, - "trueNegative": 494, - "truePositive": 893, - "wilsonPrecisionLower95": 0.9353 - }, - "threshold": 0.7 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9424, - "falseNegative": 62, - "falsePositive": 46, - "precision": 0.9505, - "predictedPositive": 929, - "recall": 0.9344, - "total": 1485, - "trueNegative": 494, - "truePositive": 883, - "wilsonPrecisionLower95": 0.9346 - }, - "threshold": 0.695 - } - }, - "metrics": { - "f1": 0.9454, - "falseNegative": 115, - "falsePositive": 90, - "precision": 0.9517, - "predictedPositive": 1865, - "recall": 0.9392, - "total": 2970, - "trueNegative": 990, - "truePositive": 1775, - "wilsonPrecisionLower95": 0.9411 - }, - "threshold": 0.7 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 45, - "recall": 1.0, - "total": 1485, - "trueNegative": 1440, - "truePositive": 45, - "wilsonPrecisionLower95": 0.9213 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.967, - "falseNegative": 1, - "falsePositive": 2, - "precision": 0.9565, - "predictedPositive": 46, - "recall": 0.9778, - "total": 1485, - "trueNegative": 1438, - "truePositive": 44, - "wilsonPrecisionLower95": 0.8547 - }, - "threshold": 0.93 - } - }, - "metrics": { - "f1": 0.978, - "falseNegative": 1, - "falsePositive": 3, - "precision": 0.9674, - "predictedPositive": 92, - "recall": 0.9889, - "total": 2970, - "trueNegative": 2877, - "truePositive": 89, - "wilsonPrecisionLower95": 0.9085 - }, - "threshold": 0.91 - }, - "task": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8403, - "falseNegative": 89, - "falsePositive": 14, - "precision": 0.9509, - "predictedPositive": 285, - "recall": 0.7528, - "total": 1485, - "trueNegative": 1111, - "truePositive": 271, - "wilsonPrecisionLower95": 0.9192 - }, - "threshold": 0.835 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8278, - "falseNegative": 98, - "falsePositive": 11, - "precision": 0.9597, - "predictedPositive": 273, - "recall": 0.7278, - "total": 1485, - "trueNegative": 1114, - "truePositive": 262, - "wilsonPrecisionLower95": 0.9293 - }, - "threshold": 0.745 - } - }, - "metrics": { - "f1": 0.8637, - "falseNegative": 153, - "falsePositive": 26, - "precision": 0.9562, - "predictedPositive": 593, - "recall": 0.7875, - "total": 2970, - "trueNegative": 2224, - "truePositive": 567, - "wilsonPrecisionLower95": 0.9365 - }, - "threshold": 0.755 - } - }, - "selectionObjective": 0.8669, - "trainingPolicy": "No user clipboard data. Official partial labels plus deterministic label-preserving augmentation and three-round high-confidence self-training consensus. Frozen holdouts never enter training or threshold calibration." -} diff --git a/ModelTraining/ClipboardSemantics/IterativeResearch/rounds.json b/ModelTraining/ClipboardSemantics/IterativeResearch/rounds.json deleted file mode 100644 index f64dde3..0000000 --- a/ModelTraining/ClipboardSemantics/IterativeResearch/rounds.json +++ /dev/null @@ -1,14831 +0,0 @@ -[ - { - "configuration": { - "alpha": 2e-05, - "augmentation": "none", - "char_max": 5, - "char_min": 2, - "external_weight": 0.5700000000000001, - "hard_example_weight": 1.0, - "l1_ratio": 0.0, - "max_char_features": 45000, - "max_word_features": 22000, - "min_df": 2, - "pseudo_weight": 0.16499999999999998, - "round": 1, - "word_max": 2 - }, - "featureCount": 56451, - "pseudoLabelCounts": {}, - "round": 1, - "selectionObjective": 0.8669, - "syntheticRegression": { - "intentsMeeting95Precision": 6, - "macroF1": 0.8088, - "macroPrecision": 0.9661, - "macroRecall": 0.7541, - "minimumPrecision": 0.8589, - "perIntent": { - "blessing": { - "f1": 0.125, - "falseNegative": 84, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 6, - "recall": 0.0667, - "total": 2970, - "trueNegative": 2880, - "truePositive": 6, - "wilsonPrecisionLower95": 0.6097 - }, - "complaint": { - "f1": 0.9773, - "falseNegative": 16, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 344, - "recall": 0.9556, - "total": 2970, - "trueNegative": 2610, - "truePositive": 344, - "wilsonPrecisionLower95": 0.989 - }, - "confirmationDecision": { - "f1": 0.8163, - "falseNegative": 40, - "falsePositive": 23, - "precision": 0.8589, - "predictedPositive": 163, - "recall": 0.7778, - "total": 2970, - "trueNegative": 2767, - "truePositive": 140, - "wilsonPrecisionLower95": 0.7972 - }, - "followUpReminder": { - "f1": 0.8705, - "falseNegative": 81, - "falsePositive": 2, - "precision": 0.9929, - "predictedPositive": 281, - "recall": 0.775, - "total": 2970, - "trueNegative": 2608, - "truePositive": 279, - "wilsonPrecisionLower95": 0.9744 - }, - "invitation": { - "f1": 0.8545, - "falseNegative": 42, - "falsePositive": 5, - "precision": 0.965, - "predictedPositive": 143, - "recall": 0.7667, - "total": 2970, - "trueNegative": 2785, - "truePositive": 138, - "wilsonPrecisionLower95": 0.9208 - }, - "question": { - "f1": 0.9092, - "falseNegative": 104, - "falsePositive": 1, - "precision": 0.9981, - "predictedPositive": 527, - "recall": 0.8349, - "total": 2970, - "trueNegative": 2339, - "truePositive": 526, - "wilsonPrecisionLower95": 0.9893 - }, - "replyableMessage": { - "f1": 0.9465, - "falseNegative": 156, - "falsePositive": 40, - "precision": 0.9775, - "predictedPositive": 1774, - "recall": 0.9175, - "total": 2970, - "trueNegative": 1040, - "truePositive": 1734, - "wilsonPrecisionLower95": 0.9694 - }, - "scheduleNegotiation": { - "f1": 0.9727, - "falseNegative": 1, - "falsePositive": 4, - "precision": 0.957, - "predictedPositive": 93, - "recall": 0.9889, - "total": 2970, - "trueNegative": 2876, - "truePositive": 89, - "wilsonPrecisionLower95": 0.8946 - }, - "task": { - "f1": 0.8073, - "falseNegative": 213, - "falsePositive": 29, - "precision": 0.9459, - "predictedPositive": 536, - "recall": 0.7042, - "total": 2970, - "trueNegative": 2221, - "truePositive": 507, - "wilsonPrecisionLower95": 0.9234 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9318, - "falseNegative": 4, - "falsePositive": 2, - "precision": 0.9535, - "predictedPositive": 43, - "recall": 0.9111, - "total": 1485, - "trueNegative": 1438, - "truePositive": 41, - "wilsonPrecisionLower95": 0.8454 - }, - "threshold": 0.915 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7568, - "falseNegative": 17, - "falsePositive": 1, - "precision": 0.9655, - "predictedPositive": 29, - "recall": 0.6222, - "total": 1485, - "trueNegative": 1439, - "truePositive": 28, - "wilsonPrecisionLower95": 0.8282 - }, - "threshold": 0.9 - } - }, - "metrics": { - "f1": 0.8375, - "falseNegative": 23, - "falsePositive": 3, - "precision": 0.9571, - "predictedPositive": 70, - "recall": 0.7444, - "total": 2970, - "trueNegative": 2877, - "truePositive": 67, - "wilsonPrecisionLower95": 0.8814 - }, - "threshold": 0.915 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 180, - "recall": 1.0, - "total": 1485, - "trueNegative": 1305, - "truePositive": 180, - "wilsonPrecisionLower95": 0.9791 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9526, - "falseNegative": 9, - "falsePositive": 8, - "precision": 0.9553, - "predictedPositive": 179, - "recall": 0.95, - "total": 1485, - "trueNegative": 1297, - "truePositive": 171, - "wilsonPrecisionLower95": 0.9143 - }, - "threshold": 0.54 - } - }, - "metrics": { - "f1": 0.9764, - "falseNegative": 9, - "falsePositive": 8, - "precision": 0.9777, - "predictedPositive": 359, - "recall": 0.975, - "total": 2970, - "trueNegative": 2602, - "truePositive": 351, - "wilsonPrecisionLower95": 0.9567 - }, - "threshold": 0.54 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8848, - "falseNegative": 17, - "falsePositive": 2, - "precision": 0.9733, - "predictedPositive": 75, - "recall": 0.8111, - "total": 1485, - "trueNegative": 1393, - "truePositive": 73, - "wilsonPrecisionLower95": 0.9079 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7947, - "falseNegative": 30, - "falsePositive": 1, - "precision": 0.9836, - "predictedPositive": 61, - "recall": 0.6667, - "total": 1485, - "trueNegative": 1394, - "truePositive": 60, - "wilsonPrecisionLower95": 0.9128 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.8418, - "falseNegative": 47, - "falsePositive": 3, - "precision": 0.9779, - "predictedPositive": 136, - "recall": 0.7389, - "total": 2970, - "trueNegative": 2787, - "truePositive": 133, - "wilsonPrecisionLower95": 0.9372 - }, - "threshold": 0.5 - }, - "followUpReminder": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8554, - "falseNegative": 41, - "falsePositive": 6, - "precision": 0.9586, - "predictedPositive": 145, - "recall": 0.7722, - "total": 1485, - "trueNegative": 1299, - "truePositive": 139, - "wilsonPrecisionLower95": 0.9127 - }, - "threshold": 0.83 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8354, - "falseNegative": 48, - "falsePositive": 4, - "precision": 0.9706, - "predictedPositive": 136, - "recall": 0.7333, - "total": 1485, - "trueNegative": 1301, - "truePositive": 132, - "wilsonPrecisionLower95": 0.9268 - }, - "threshold": 0.505 - } - }, - "metrics": { - "f1": 0.7994, - "falseNegative": 113, - "falsePositive": 11, - "precision": 0.9574, - "predictedPositive": 258, - "recall": 0.6861, - "total": 2970, - "trueNegative": 2599, - "truePositive": 247, - "wilsonPrecisionLower95": 0.9253 - }, - "threshold": 0.815 - }, - "invitation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8302, - "falseNegative": 24, - "falsePositive": 3, - "precision": 0.9565, - "predictedPositive": 69, - "recall": 0.7333, - "total": 1485, - "trueNegative": 1392, - "truePositive": 66, - "wilsonPrecisionLower95": 0.8798 - }, - "threshold": 0.935 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6765, - "falseNegative": 44, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 46, - "recall": 0.5111, - "total": 1485, - "trueNegative": 1395, - "truePositive": 46, - "wilsonPrecisionLower95": 0.9229 - }, - "threshold": 0.505 - } - }, - "metrics": { - "f1": 0.5476, - "falseNegative": 111, - "falsePositive": 3, - "precision": 0.9583, - "predictedPositive": 72, - "recall": 0.3833, - "total": 2970, - "trueNegative": 2787, - "truePositive": 69, - "wilsonPrecisionLower95": 0.8845 - }, - "threshold": 0.935 - }, - "question": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8445, - "falseNegative": 76, - "falsePositive": 12, - "precision": 0.9522, - "predictedPositive": 251, - "recall": 0.7587, - "total": 1485, - "trueNegative": 1158, - "truePositive": 239, - "wilsonPrecisionLower95": 0.9183 - }, - "threshold": 0.725 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8536, - "falseNegative": 76, - "falsePositive": 6, - "precision": 0.9755, - "predictedPositive": 245, - "recall": 0.7587, - "total": 1485, - "trueNegative": 1164, - "truePositive": 239, - "wilsonPrecisionLower95": 0.9476 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.8729, - "falseNegative": 122, - "falsePositive": 26, - "precision": 0.9513, - "predictedPositive": 534, - "recall": 0.8063, - "total": 2970, - "trueNegative": 2314, - "truePositive": 508, - "wilsonPrecisionLower95": 0.9296 - }, - "threshold": 0.515 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.948, - "falseNegative": 52, - "falsePositive": 46, - "precision": 0.951, - "predictedPositive": 939, - "recall": 0.945, - "total": 1485, - "trueNegative": 494, - "truePositive": 893, - "wilsonPrecisionLower95": 0.9353 - }, - "threshold": 0.7 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9424, - "falseNegative": 62, - "falsePositive": 46, - "precision": 0.9505, - "predictedPositive": 929, - "recall": 0.9344, - "total": 1485, - "trueNegative": 494, - "truePositive": 883, - "wilsonPrecisionLower95": 0.9346 - }, - "threshold": 0.695 - } - }, - "metrics": { - "f1": 0.9454, - "falseNegative": 115, - "falsePositive": 90, - "precision": 0.9517, - "predictedPositive": 1865, - "recall": 0.9392, - "total": 2970, - "trueNegative": 990, - "truePositive": 1775, - "wilsonPrecisionLower95": 0.9411 - }, - "threshold": 0.7 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 45, - "recall": 1.0, - "total": 1485, - "trueNegative": 1440, - "truePositive": 45, - "wilsonPrecisionLower95": 0.9213 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.967, - "falseNegative": 1, - "falsePositive": 2, - "precision": 0.9565, - "predictedPositive": 46, - "recall": 0.9778, - "total": 1485, - "trueNegative": 1438, - "truePositive": 44, - "wilsonPrecisionLower95": 0.8547 - }, - "threshold": 0.93 - } - }, - "metrics": { - "f1": 0.978, - "falseNegative": 1, - "falsePositive": 3, - "precision": 0.9674, - "predictedPositive": 92, - "recall": 0.9889, - "total": 2970, - "trueNegative": 2877, - "truePositive": 89, - "wilsonPrecisionLower95": 0.9085 - }, - "threshold": 0.91 - }, - "task": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8403, - "falseNegative": 89, - "falsePositive": 14, - "precision": 0.9509, - "predictedPositive": 285, - "recall": 0.7528, - "total": 1485, - "trueNegative": 1111, - "truePositive": 271, - "wilsonPrecisionLower95": 0.9192 - }, - "threshold": 0.835 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8278, - "falseNegative": 98, - "falsePositive": 11, - "precision": 0.9597, - "predictedPositive": 273, - "recall": 0.7278, - "total": 1485, - "trueNegative": 1114, - "truePositive": 262, - "wilsonPrecisionLower95": 0.9293 - }, - "threshold": 0.745 - } - }, - "metrics": { - "f1": 0.8637, - "falseNegative": 153, - "falsePositive": 26, - "precision": 0.9562, - "predictedPositive": 593, - "recall": 0.7875, - "total": 2970, - "trueNegative": 2224, - "truePositive": 567, - "wilsonPrecisionLower95": 0.9365 - }, - "threshold": 0.755 - } - }, - "trainingRecords": 15345, - "validation": { - "intentsMeeting95Precision": 9, - "macroF1": 0.822, - "macroPrecision": 0.9713, - "macroRecall": 0.7487, - "minimumPrecision": 0.9507, - "perIntent": { - "blessing": { - "f1": 0.3636, - "falseNegative": 70, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 20, - "recall": 0.2222, - "total": 2970, - "trueNegative": 2880, - "truePositive": 20, - "wilsonPrecisionLower95": 0.8389 - }, - "complaint": { - "f1": 0.9764, - "falseNegative": 9, - "falsePositive": 8, - "precision": 0.9777, - "predictedPositive": 359, - "recall": 0.975, - "total": 2970, - "trueNegative": 2602, - "truePositive": 351, - "wilsonPrecisionLower95": 0.9567 - }, - "confirmationDecision": { - "f1": 0.8418, - "falseNegative": 47, - "falsePositive": 3, - "precision": 0.9779, - "predictedPositive": 136, - "recall": 0.7389, - "total": 2970, - "trueNegative": 2787, - "truePositive": 133, - "wilsonPrecisionLower95": 0.9372 - }, - "followUpReminder": { - "f1": 0.8456, - "falseNegative": 89, - "falsePositive": 10, - "precision": 0.9644, - "predictedPositive": 281, - "recall": 0.7528, - "total": 2970, - "trueNegative": 2600, - "truePositive": 271, - "wilsonPrecisionLower95": 0.9357 - }, - "invitation": { - "f1": 0.7593, - "falseNegative": 68, - "falsePositive": 3, - "precision": 0.9739, - "predictedPositive": 115, - "recall": 0.6222, - "total": 2970, - "trueNegative": 2787, - "truePositive": 112, - "wilsonPrecisionLower95": 0.9261 - }, - "question": { - "f1": 0.849, - "falseNegative": 152, - "falsePositive": 18, - "precision": 0.9637, - "predictedPositive": 496, - "recall": 0.7587, - "total": 2970, - "trueNegative": 2322, - "truePositive": 478, - "wilsonPrecisionLower95": 0.9434 - }, - "replyableMessage": { - "f1": 0.9452, - "falseNegative": 114, - "falsePositive": 92, - "precision": 0.9507, - "predictedPositive": 1868, - "recall": 0.9397, - "total": 2970, - "trueNegative": 988, - "truePositive": 1776, - "wilsonPrecisionLower95": 0.94 - }, - "scheduleNegotiation": { - "f1": 0.9834, - "falseNegative": 1, - "falsePositive": 2, - "precision": 0.978, - "predictedPositive": 91, - "recall": 0.9889, - "total": 2970, - "trueNegative": 2878, - "truePositive": 89, - "wilsonPrecisionLower95": 0.9234 - }, - "task": { - "f1": 0.8341, - "falseNegative": 187, - "falsePositive": 25, - "precision": 0.9552, - "predictedPositive": 558, - "recall": 0.7403, - "total": 2970, - "trueNegative": 2225, - "truePositive": 533, - "wilsonPrecisionLower95": 0.9347 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 1.5e-05, - "augmentation": "none", - "char_max": 5, - "char_min": 3, - "external_weight": 0.5900000000000001, - "hard_example_weight": 1.0, - "l1_ratio": 0.0, - "max_char_features": 50000, - "max_word_features": 25000, - "min_df": 2, - "pseudo_weight": 0.18, - "round": 2, - "word_max": 2 - }, - "featureCount": 61451, - "pseudoLabelCounts": {}, - "round": 2, - "selectionObjective": 0.8228, - "syntheticRegression": { - "intentsMeeting95Precision": 7, - "macroF1": 0.751, - "macroPrecision": 0.9667, - "macroRecall": 0.6602, - "minimumPrecision": 0.8606, - "perIntent": { - "blessing": { - "f1": 0.2178, - "falseNegative": 79, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 11, - "recall": 0.1222, - "total": 2970, - "trueNegative": 2880, - "truePositive": 11, - "wilsonPrecisionLower95": 0.7412 - }, - "complaint": { - "f1": 0.9595, - "falseNegative": 28, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 332, - "recall": 0.9222, - "total": 2970, - "trueNegative": 2610, - "truePositive": 332, - "wilsonPrecisionLower95": 0.9886 - }, - "confirmationDecision": { - "f1": 0.8232, - "falseNegative": 38, - "falsePositive": 23, - "precision": 0.8606, - "predictedPositive": 165, - "recall": 0.7889, - "total": 2970, - "trueNegative": 2767, - "truePositive": 142, - "wilsonPrecisionLower95": 0.7995 - }, - "followUpReminder": { - "f1": 0.7565, - "falseNegative": 141, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 219, - "recall": 0.6083, - "total": 2970, - "trueNegative": 2610, - "truePositive": 219, - "wilsonPrecisionLower95": 0.9828 - }, - "invitation": { - "f1": 0.6394, - "falseNegative": 94, - "falsePositive": 3, - "precision": 0.9663, - "predictedPositive": 89, - "recall": 0.4778, - "total": 2970, - "trueNegative": 2787, - "truePositive": 86, - "wilsonPrecisionLower95": 0.9055 - }, - "question": { - "f1": 0.86, - "falseNegative": 151, - "falsePositive": 5, - "precision": 0.9897, - "predictedPositive": 484, - "recall": 0.7603, - "total": 2970, - "trueNegative": 2335, - "truePositive": 479, - "wilsonPrecisionLower95": 0.976 - }, - "replyableMessage": { - "f1": 0.9038, - "falseNegative": 293, - "falsePositive": 47, - "precision": 0.9714, - "predictedPositive": 1644, - "recall": 0.845, - "total": 2970, - "trueNegative": 1033, - "truePositive": 1597, - "wilsonPrecisionLower95": 0.9622 - }, - "scheduleNegotiation": { - "f1": 0.9425, - "falseNegative": 8, - "falsePositive": 2, - "precision": 0.9762, - "predictedPositive": 84, - "recall": 0.9111, - "total": 2970, - "trueNegative": 2878, - "truePositive": 82, - "wilsonPrecisionLower95": 0.9173 - }, - "task": { - "f1": 0.6564, - "falseNegative": 356, - "falsePositive": 25, - "precision": 0.9357, - "predictedPositive": 389, - "recall": 0.5056, - "total": 2970, - "trueNegative": 2225, - "truePositive": 364, - "wilsonPrecisionLower95": 0.9069 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9048, - "falseNegative": 7, - "falsePositive": 1, - "precision": 0.9744, - "predictedPositive": 39, - "recall": 0.8444, - "total": 1485, - "trueNegative": 1439, - "truePositive": 38, - "wilsonPrecisionLower95": 0.8682 - }, - "threshold": 0.59 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 180, - "recall": 1.0, - "total": 1485, - "trueNegative": 1305, - "truePositive": 180, - "wilsonPrecisionLower95": 0.9791 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9226, - "falseNegative": 19, - "falsePositive": 8, - "precision": 0.9527, - "predictedPositive": 169, - "recall": 0.8944, - "total": 1485, - "trueNegative": 1297, - "truePositive": 161, - "wilsonPrecisionLower95": 0.9094 - }, - "threshold": 0.555 - } - }, - "metrics": { - "f1": 0.9649, - "falseNegative": 16, - "falsePositive": 9, - "precision": 0.9745, - "predictedPositive": 353, - "recall": 0.9556, - "total": 2970, - "trueNegative": 2601, - "truePositive": 344, - "wilsonPrecisionLower95": 0.9523 - }, - "threshold": 0.5 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.954, - "falseNegative": 7, - "falsePositive": 1, - "precision": 0.9881, - "predictedPositive": 84, - "recall": 0.9222, - "total": 1485, - "trueNegative": 1394, - "truePositive": 83, - "wilsonPrecisionLower95": 0.9356 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7183, - "falseNegative": 39, - "falsePositive": 1, - "precision": 0.9808, - "predictedPositive": 52, - "recall": 0.5667, - "total": 1485, - "trueNegative": 1394, - "truePositive": 51, - "wilsonPrecisionLower95": 0.8988 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.8481, - "falseNegative": 46, - "falsePositive": 2, - "precision": 0.9853, - "predictedPositive": 136, - "recall": 0.7444, - "total": 2970, - "trueNegative": 2788, - "truePositive": 134, - "wilsonPrecisionLower95": 0.948 - }, - "threshold": 0.5 - }, - "followUpReminder": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8328, - "falseNegative": 48, - "falsePositive": 5, - "precision": 0.9635, - "predictedPositive": 137, - "recall": 0.7333, - "total": 1485, - "trueNegative": 1300, - "truePositive": 132, - "wilsonPrecisionLower95": 0.9174 - }, - "threshold": 0.91 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6879, - "falseNegative": 83, - "falsePositive": 5, - "precision": 0.951, - "predictedPositive": 102, - "recall": 0.5389, - "total": 1485, - "trueNegative": 1300, - "truePositive": 97, - "wilsonPrecisionLower95": 0.8903 - }, - "threshold": 0.765 - } - }, - "metrics": { - "f1": 0.6243, - "falseNegative": 193, - "falsePositive": 8, - "precision": 0.9543, - "predictedPositive": 175, - "recall": 0.4639, - "total": 2970, - "trueNegative": 2602, - "truePositive": 167, - "wilsonPrecisionLower95": 0.9124 - }, - "threshold": 0.905 - }, - "invitation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9059, - "falseNegative": 13, - "falsePositive": 3, - "precision": 0.9625, - "predictedPositive": 80, - "recall": 0.8556, - "total": 1485, - "trueNegative": 1392, - "truePositive": 77, - "wilsonPrecisionLower95": 0.8955 - }, - "threshold": 0.86 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.6113, - "falseNegative": 99, - "falsePositive": 4, - "precision": 0.9529, - "predictedPositive": 85, - "recall": 0.45, - "total": 2970, - "trueNegative": 2786, - "truePositive": 81, - "wilsonPrecisionLower95": 0.8852 - }, - "threshold": 0.855 - }, - "question": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8505, - "falseNegative": 76, - "falsePositive": 8, - "precision": 0.9676, - "predictedPositive": 247, - "recall": 0.7587, - "total": 1485, - "trueNegative": 1162, - "truePositive": 239, - "wilsonPrecisionLower95": 0.9374 - }, - "threshold": 0.705 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7308, - "falseNegative": 129, - "falsePositive": 8, - "precision": 0.9588, - "predictedPositive": 194, - "recall": 0.5905, - "total": 1485, - "trueNegative": 1162, - "truePositive": 186, - "wilsonPrecisionLower95": 0.9208 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.7813, - "falseNegative": 212, - "falsePositive": 22, - "precision": 0.95, - "predictedPositive": 440, - "recall": 0.6635, - "total": 2970, - "trueNegative": 2318, - "truePositive": 418, - "wilsonPrecisionLower95": 0.9255 - }, - "threshold": 0.57 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9629, - "falseNegative": 23, - "falsePositive": 48, - "precision": 0.9505, - "predictedPositive": 970, - "recall": 0.9757, - "total": 1485, - "trueNegative": 492, - "truePositive": 922, - "wilsonPrecisionLower95": 0.935 - }, - "threshold": 0.62 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8526, - "falseNegative": 216, - "falsePositive": 36, - "precision": 0.9529, - "predictedPositive": 765, - "recall": 0.7714, - "total": 1485, - "trueNegative": 504, - "truePositive": 729, - "wilsonPrecisionLower95": 0.9355 - }, - "threshold": 0.84 - } - }, - "metrics": { - "f1": 0.9224, - "falseNegative": 197, - "falsePositive": 88, - "precision": 0.9506, - "predictedPositive": 1781, - "recall": 0.8958, - "total": 2970, - "trueNegative": 992, - "truePositive": 1693, - "wilsonPrecisionLower95": 0.9395 - }, - "threshold": 0.775 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9783, - "falseNegative": 0, - "falsePositive": 2, - "precision": 0.9574, - "predictedPositive": 47, - "recall": 1.0, - "total": 1485, - "trueNegative": 1438, - "truePositive": 45, - "wilsonPrecisionLower95": 0.8575 - }, - "threshold": 0.765 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9778, - "falseNegative": 1, - "falsePositive": 1, - "precision": 0.9778, - "predictedPositive": 45, - "recall": 0.9778, - "total": 1485, - "trueNegative": 1439, - "truePositive": 44, - "wilsonPrecisionLower95": 0.8843 - }, - "threshold": 0.855 - } - }, - "metrics": { - "f1": 0.978, - "falseNegative": 1, - "falsePositive": 3, - "precision": 0.9674, - "predictedPositive": 92, - "recall": 0.9889, - "total": 2970, - "trueNegative": 2877, - "truePositive": 89, - "wilsonPrecisionLower95": 0.9085 - }, - "threshold": 0.855 - }, - "task": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8541, - "falseNegative": 82, - "falsePositive": 13, - "precision": 0.9553, - "predictedPositive": 291, - "recall": 0.7722, - "total": 1485, - "trueNegative": 1112, - "truePositive": 278, - "wilsonPrecisionLower95": 0.9251 - }, - "threshold": 0.84 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.5418, - "falseNegative": 224, - "falsePositive": 6, - "precision": 0.9577, - "predictedPositive": 142, - "recall": 0.3778, - "total": 1485, - "trueNegative": 1119, - "truePositive": 136, - "wilsonPrecisionLower95": 0.9109 - }, - "threshold": 0.91 - } - }, - "metrics": { - "f1": 0.7246, - "falseNegative": 299, - "falsePositive": 21, - "precision": 0.9525, - "predictedPositive": 442, - "recall": 0.5847, - "total": 2970, - "trueNegative": 2229, - "truePositive": 421, - "wilsonPrecisionLower95": 0.9285 - }, - "threshold": 0.88 - } - }, - "trainingRecords": 15345, - "validation": { - "intentsMeeting95Precision": 9, - "macroF1": 0.7578, - "macroPrecision": 0.9691, - "macroRecall": 0.668, - "minimumPrecision": 0.9516, - "perIntent": { - "blessing": { - "f1": 0.2524, - "falseNegative": 77, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 13, - "recall": 0.1444, - "total": 2970, - "trueNegative": 2880, - "truePositive": 13, - "wilsonPrecisionLower95": 0.7719 - }, - "complaint": { - "f1": 0.9619, - "falseNegative": 19, - "falsePositive": 8, - "precision": 0.9771, - "predictedPositive": 349, - "recall": 0.9472, - "total": 2970, - "trueNegative": 2602, - "truePositive": 341, - "wilsonPrecisionLower95": 0.9554 - }, - "confirmationDecision": { - "f1": 0.8481, - "falseNegative": 46, - "falsePositive": 2, - "precision": 0.9853, - "predictedPositive": 136, - "recall": 0.7444, - "total": 2970, - "trueNegative": 2788, - "truePositive": 134, - "wilsonPrecisionLower95": 0.948 - }, - "followUpReminder": { - "f1": 0.7646, - "falseNegative": 131, - "falsePositive": 10, - "precision": 0.9582, - "predictedPositive": 239, - "recall": 0.6361, - "total": 2970, - "trueNegative": 2600, - "truePositive": 229, - "wilsonPrecisionLower95": 0.9247 - }, - "invitation": { - "f1": 0.5923, - "falseNegative": 103, - "falsePositive": 3, - "precision": 0.9625, - "predictedPositive": 80, - "recall": 0.4278, - "total": 2970, - "trueNegative": 2787, - "truePositive": 77, - "wilsonPrecisionLower95": 0.8955 - }, - "question": { - "f1": 0.7937, - "falseNegative": 205, - "falsePositive": 16, - "precision": 0.9637, - "predictedPositive": 441, - "recall": 0.6746, - "total": 2970, - "trueNegative": 2324, - "truePositive": 425, - "wilsonPrecisionLower95": 0.9419 - }, - "replyableMessage": { - "f1": 0.9109, - "falseNegative": 239, - "falsePositive": 84, - "precision": 0.9516, - "predictedPositive": 1735, - "recall": 0.8735, - "total": 2970, - "trueNegative": 996, - "truePositive": 1651, - "wilsonPrecisionLower95": 0.9404 - }, - "scheduleNegotiation": { - "f1": 0.978, - "falseNegative": 1, - "falsePositive": 3, - "precision": 0.9674, - "predictedPositive": 92, - "recall": 0.9889, - "total": 2970, - "trueNegative": 2877, - "truePositive": 89, - "wilsonPrecisionLower95": 0.9085 - }, - "task": { - "f1": 0.7181, - "falseNegative": 306, - "falsePositive": 19, - "precision": 0.9561, - "predictedPositive": 433, - "recall": 0.575, - "total": 2970, - "trueNegative": 2231, - "truePositive": 414, - "wilsonPrecisionLower95": 0.9325 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 1e-05, - "augmentation": "none", - "char_max": 6, - "char_min": 2, - "external_weight": 0.6100000000000001, - "hard_example_weight": 1.2, - "l1_ratio": 0.05, - "max_char_features": 55000, - "max_word_features": 25000, - "min_df": 2, - "pseudo_weight": 0.195, - "round": 3, - "word_max": 2 - }, - "featureCount": 66451, - "pseudoLabelCounts": {}, - "round": 3, - "selectionObjective": 0.8348, - "syntheticRegression": { - "intentsMeeting95Precision": 4, - "macroF1": 0.7343, - "macroPrecision": 0.9403, - "macroRecall": 0.6672, - "minimumPrecision": 0.809, - "perIntent": { - "blessing": { - "f1": 0.0645, - "falseNegative": 87, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 3, - "recall": 0.0333, - "total": 2970, - "trueNegative": 2880, - "truePositive": 3, - "wilsonPrecisionLower95": 0.4385 - }, - "complaint": { - "f1": 0.9816, - "falseNegative": 13, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 347, - "recall": 0.9639, - "total": 2970, - "trueNegative": 2610, - "truePositive": 347, - "wilsonPrecisionLower95": 0.9891 - }, - "confirmationDecision": { - "f1": 0.8045, - "falseNegative": 36, - "falsePositive": 34, - "precision": 0.809, - "predictedPositive": 178, - "recall": 0.8, - "total": 2970, - "trueNegative": 2756, - "truePositive": 144, - "wilsonPrecisionLower95": 0.745 - }, - "followUpReminder": { - "f1": 0.7967, - "falseNegative": 121, - "falsePositive": 1, - "precision": 0.9958, - "predictedPositive": 240, - "recall": 0.6639, - "total": 2970, - "trueNegative": 2609, - "truePositive": 239, - "wilsonPrecisionLower95": 0.9768 - }, - "invitation": { - "f1": 0.5985, - "falseNegative": 101, - "falsePositive": 5, - "precision": 0.9405, - "predictedPositive": 84, - "recall": 0.4389, - "total": 2970, - "trueNegative": 2785, - "truePositive": 79, - "wilsonPrecisionLower95": 0.8681 - }, - "question": { - "f1": 0.8643, - "falseNegative": 149, - "falsePositive": 2, - "precision": 0.9959, - "predictedPositive": 483, - "recall": 0.7635, - "total": 2970, - "trueNegative": 2338, - "truePositive": 481, - "wilsonPrecisionLower95": 0.985 - }, - "replyableMessage": { - "f1": 0.931, - "falseNegative": 189, - "falsePositive": 63, - "precision": 0.9643, - "predictedPositive": 1764, - "recall": 0.9, - "total": 2970, - "trueNegative": 1017, - "truePositive": 1701, - "wilsonPrecisionLower95": 0.9546 - }, - "scheduleNegotiation": { - "f1": 0.8222, - "falseNegative": 16, - "falsePositive": 16, - "precision": 0.8222, - "predictedPositive": 90, - "recall": 0.8222, - "total": 2970, - "trueNegative": 2864, - "truePositive": 74, - "wilsonPrecisionLower95": 0.7306 - }, - "task": { - "f1": 0.7452, - "falseNegative": 274, - "falsePositive": 31, - "precision": 0.935, - "predictedPositive": 477, - "recall": 0.6194, - "total": 2970, - "trueNegative": 2219, - "truePositive": 446, - "wilsonPrecisionLower95": 0.9092 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 180, - "recall": 1.0, - "total": 1485, - "trueNegative": 1305, - "truePositive": 180, - "wilsonPrecisionLower95": 0.9791 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9371, - "falseNegative": 16, - "falsePositive": 6, - "precision": 0.9647, - "predictedPositive": 170, - "recall": 0.9111, - "total": 1485, - "trueNegative": 1299, - "truePositive": 164, - "wilsonPrecisionLower95": 0.9251 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.969, - "falseNegative": 16, - "falsePositive": 6, - "precision": 0.9829, - "predictedPositive": 350, - "recall": 0.9556, - "total": 2970, - "trueNegative": 2604, - "truePositive": 344, - "wilsonPrecisionLower95": 0.9631 - }, - "threshold": 0.5 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9727, - "falseNegative": 1, - "falsePositive": 4, - "precision": 0.957, - "predictedPositive": 93, - "recall": 0.9889, - "total": 1485, - "trueNegative": 1391, - "truePositive": 89, - "wilsonPrecisionLower95": 0.8946 - }, - "threshold": 0.585 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7815, - "falseNegative": 31, - "falsePositive": 2, - "precision": 0.9672, - "predictedPositive": 61, - "recall": 0.6556, - "total": 1485, - "trueNegative": 1393, - "truePositive": 59, - "wilsonPrecisionLower95": 0.8881 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.8701, - "falseNegative": 36, - "falsePositive": 7, - "precision": 0.9536, - "predictedPositive": 151, - "recall": 0.8, - "total": 2970, - "trueNegative": 2783, - "truePositive": 144, - "wilsonPrecisionLower95": 0.9074 - }, - "threshold": 0.555 - }, - "followUpReminder": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8454, - "falseNegative": 46, - "falsePositive": 3, - "precision": 0.9781, - "predictedPositive": 137, - "recall": 0.7444, - "total": 1485, - "trueNegative": 1302, - "truePositive": 134, - "wilsonPrecisionLower95": 0.9376 - }, - "threshold": 0.97 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7525, - "falseNegative": 69, - "falsePositive": 4, - "precision": 0.9652, - "predictedPositive": 115, - "recall": 0.6167, - "total": 1485, - "trueNegative": 1301, - "truePositive": 111, - "wilsonPrecisionLower95": 0.914 - }, - "threshold": 0.785 - } - }, - "metrics": { - "f1": 0.703, - "falseNegative": 160, - "falsePositive": 9, - "precision": 0.9569, - "predictedPositive": 209, - "recall": 0.5556, - "total": 2970, - "trueNegative": 2601, - "truePositive": 200, - "wilsonPrecisionLower95": 0.9202 - }, - "threshold": 0.955 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7974, - "falseNegative": 29, - "falsePositive": 2, - "precision": 0.9683, - "predictedPositive": 63, - "recall": 0.6778, - "total": 1485, - "trueNegative": 1393, - "truePositive": 61, - "wilsonPrecisionLower95": 0.8914 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8259, - "falseNegative": 85, - "falsePositive": 12, - "precision": 0.9504, - "predictedPositive": 242, - "recall": 0.7302, - "total": 1485, - "trueNegative": 1158, - "truePositive": 230, - "wilsonPrecisionLower95": 0.9153 - }, - "threshold": 0.865 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8194, - "falseNegative": 95, - "falsePositive": 2, - "precision": 0.991, - "predictedPositive": 222, - "recall": 0.6984, - "total": 1485, - "trueNegative": 1168, - "truePositive": 220, - "wilsonPrecisionLower95": 0.9678 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.7959, - "falseNegative": 199, - "falsePositive": 22, - "precision": 0.9514, - "predictedPositive": 453, - "recall": 0.6841, - "total": 2970, - "trueNegative": 2318, - "truePositive": 431, - "wilsonPrecisionLower95": 0.9276 - }, - "threshold": 0.74 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9496, - "falseNegative": 50, - "falsePositive": 45, - "precision": 0.9521, - "predictedPositive": 940, - "recall": 0.9471, - "total": 1485, - "trueNegative": 495, - "truePositive": 895, - "wilsonPrecisionLower95": 0.9365 - }, - "threshold": 0.705 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9209, - "falseNegative": 101, - "falsePositive": 44, - "precision": 0.9505, - "predictedPositive": 888, - "recall": 0.8931, - "total": 1485, - "trueNegative": 496, - "truePositive": 844, - "wilsonPrecisionLower95": 0.9341 - }, - "threshold": 0.78 - } - }, - "metrics": { - "f1": 0.9318, - "falseNegative": 163, - "falsePositive": 90, - "precision": 0.9505, - "predictedPositive": 1817, - "recall": 0.9138, - "total": 2970, - "trueNegative": 990, - "truePositive": 1727, - "wilsonPrecisionLower95": 0.9395 - }, - "threshold": 0.77 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 45, - "recall": 1.0, - "total": 1485, - "trueNegative": 1440, - "truePositive": 45, - "wilsonPrecisionLower95": 0.9213 - }, - "threshold": 0.745 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9783, - "falseNegative": 0, - "falsePositive": 2, - "precision": 0.9574, - "predictedPositive": 47, - "recall": 1.0, - "total": 1485, - "trueNegative": 1438, - "truePositive": 45, - "wilsonPrecisionLower95": 0.8575 - }, - "threshold": 0.93 - } - }, - "metrics": { - "f1": 0.9778, - "falseNegative": 2, - "falsePositive": 2, - "precision": 0.9778, - "predictedPositive": 90, - "recall": 0.9778, - "total": 2970, - "trueNegative": 2878, - "truePositive": 88, - "wilsonPrecisionLower95": 0.9226 - }, - "threshold": 0.93 - }, - "task": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8488, - "falseNegative": 85, - "falsePositive": 13, - "precision": 0.9549, - "predictedPositive": 288, - "recall": 0.7639, - "total": 1485, - "trueNegative": 1112, - "truePositive": 275, - "wilsonPrecisionLower95": 0.9243 - }, - "threshold": 0.865 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8006, - "falseNegative": 111, - "falsePositive": 13, - "precision": 0.9504, - "predictedPositive": 262, - "recall": 0.6917, - "total": 1485, - "trueNegative": 1112, - "truePositive": 249, - "wilsonPrecisionLower95": 0.917 - }, - "threshold": 0.8 - } - }, - "metrics": { - "f1": 0.8301, - "falseNegative": 190, - "falsePositive": 27, - "precision": 0.9515, - "predictedPositive": 557, - "recall": 0.7361, - "total": 2970, - "trueNegative": 2223, - "truePositive": 530, - "wilsonPrecisionLower95": 0.9304 - }, - "threshold": 0.82 - } - }, - "trainingRecords": 15345, - "validation": { - "intentsMeeting95Precision": 8, - "macroF1": 0.7723, - "macroPrecision": 0.9713, - "macroRecall": 0.7011, - "minimumPrecision": 0.9513, - "perIntent": { - "blessing": { - "f1": 0.1443, - "falseNegative": 83, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 7, - "recall": 0.0778, - "total": 2970, - "trueNegative": 2880, - "truePositive": 7, - "wilsonPrecisionLower95": 0.6457 - }, - "complaint": { - "f1": 0.969, - "falseNegative": 16, - "falsePositive": 6, - "precision": 0.9829, - "predictedPositive": 350, - "recall": 0.9556, - "total": 2970, - "trueNegative": 2604, - "truePositive": 344, - "wilsonPrecisionLower95": 0.9631 - }, - "confirmationDecision": { - "f1": 0.8862, - "falseNegative": 32, - "falsePositive": 6, - "precision": 0.961, - "predictedPositive": 154, - "recall": 0.8222, - "total": 2970, - "trueNegative": 2784, - "truePositive": 148, - "wilsonPrecisionLower95": 0.9176 - }, - "followUpReminder": { - "f1": 0.8007, - "falseNegative": 115, - "falsePositive": 7, - "precision": 0.9722, - "predictedPositive": 252, - "recall": 0.6806, - "total": 2970, - "trueNegative": 2603, - "truePositive": 245, - "wilsonPrecisionLower95": 0.9438 - }, - "invitation": { - "f1": 0.5781, - "falseNegative": 106, - "falsePositive": 2, - "precision": 0.9737, - "predictedPositive": 76, - "recall": 0.4111, - "total": 2970, - "trueNegative": 2788, - "truePositive": 74, - "wilsonPrecisionLower95": 0.909 - }, - "question": { - "f1": 0.8227, - "falseNegative": 180, - "falsePositive": 14, - "precision": 0.9698, - "predictedPositive": 464, - "recall": 0.7143, - "total": 2970, - "trueNegative": 2326, - "truePositive": 450, - "wilsonPrecisionLower95": 0.95 - }, - "replyableMessage": { - "f1": 0.9354, - "falseNegative": 151, - "falsePositive": 89, - "precision": 0.9513, - "predictedPositive": 1828, - "recall": 0.9201, - "total": 2970, - "trueNegative": 991, - "truePositive": 1739, - "wilsonPrecisionLower95": 0.9405 - }, - "scheduleNegotiation": { - "f1": 0.989, - "falseNegative": 0, - "falsePositive": 2, - "precision": 0.9783, - "predictedPositive": 92, - "recall": 1.0, - "total": 2970, - "trueNegative": 2878, - "truePositive": 90, - "wilsonPrecisionLower95": 0.9242 - }, - "task": { - "f1": 0.8252, - "falseNegative": 196, - "falsePositive": 26, - "precision": 0.9527, - "predictedPositive": 550, - "recall": 0.7278, - "total": 2970, - "trueNegative": 2224, - "truePositive": 524, - "wilsonPrecisionLower95": 0.9316 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 8e-06, - "augmentation": "none", - "char_max": 6, - "char_min": 3, - "external_weight": 0.63, - "hard_example_weight": 1.2, - "l1_ratio": 0.05, - "max_char_features": 60000, - "max_word_features": 30000, - "min_df": 2, - "pseudo_weight": 0.21, - "round": 4, - "word_max": 3 - }, - "featureCount": 79458, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 0 - }, - "complaint": { - "negative": 97, - "positive": 0 - }, - "confirmationDecision": { - "negative": 51, - "positive": 2 - }, - "followUpReminder": { - "negative": 250, - "positive": 0 - }, - "invitation": { - "negative": 250, - "positive": 0 - }, - "question": { - "negative": 36, - "positive": 1 - }, - "replyableMessage": { - "negative": 3, - "positive": 130 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 0 - }, - "task": { - "negative": 85, - "positive": 0 - } - }, - "round": 4, - "selectionObjective": 0.796, - "syntheticRegression": { - "intentsMeeting95Precision": 6, - "macroF1": 0.6978, - "macroPrecision": 0.9611, - "macroRecall": 0.5994, - "minimumPrecision": 0.8074, - "perIntent": { - "blessing": { - "f1": 0.0645, - "falseNegative": 87, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 3, - "recall": 0.0333, - "total": 2970, - "trueNegative": 2880, - "truePositive": 3, - "wilsonPrecisionLower95": 0.4385 - }, - "complaint": { - "f1": 0.967, - "falseNegative": 23, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 337, - "recall": 0.9361, - "total": 2970, - "trueNegative": 2610, - "truePositive": 337, - "wilsonPrecisionLower95": 0.9887 - }, - "confirmationDecision": { - "f1": 0.6921, - "falseNegative": 71, - "falsePositive": 26, - "precision": 0.8074, - "predictedPositive": 135, - "recall": 0.6056, - "total": 2970, - "trueNegative": 2764, - "truePositive": 109, - "wilsonPrecisionLower95": 0.7328 - }, - "followUpReminder": { - "f1": 0.7734, - "falseNegative": 133, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 227, - "recall": 0.6306, - "total": 2970, - "trueNegative": 2610, - "truePositive": 227, - "wilsonPrecisionLower95": 0.9834 - }, - "invitation": { - "f1": 0.5837, - "falseNegative": 105, - "falsePositive": 2, - "precision": 0.974, - "predictedPositive": 77, - "recall": 0.4167, - "total": 2970, - "trueNegative": 2788, - "truePositive": 75, - "wilsonPrecisionLower95": 0.9102 - }, - "question": { - "f1": 0.7613, - "falseNegative": 241, - "falsePositive": 3, - "precision": 0.9923, - "predictedPositive": 392, - "recall": 0.6175, - "total": 2970, - "trueNegative": 2337, - "truePositive": 389, - "wilsonPrecisionLower95": 0.9777 - }, - "replyableMessage": { - "f1": 0.8816, - "falseNegative": 353, - "falsePositive": 60, - "precision": 0.9624, - "predictedPositive": 1597, - "recall": 0.8132, - "total": 2970, - "trueNegative": 1020, - "truePositive": 1537, - "wilsonPrecisionLower95": 0.9519 - }, - "scheduleNegotiation": { - "f1": 0.9222, - "falseNegative": 13, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 77, - "recall": 0.8556, - "total": 2970, - "trueNegative": 2880, - "truePositive": 77, - "wilsonPrecisionLower95": 0.9525 - }, - "task": { - "f1": 0.6346, - "falseNegative": 370, - "falsePositive": 33, - "precision": 0.9138, - "predictedPositive": 383, - "recall": 0.4861, - "total": 2970, - "trueNegative": 2217, - "truePositive": 350, - "wilsonPrecisionLower95": 0.8815 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9972, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 179, - "recall": 0.9944, - "total": 1485, - "trueNegative": 1305, - "truePositive": 179, - "wilsonPrecisionLower95": 0.979 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9253, - "falseNegative": 19, - "falsePositive": 7, - "precision": 0.9583, - "predictedPositive": 168, - "recall": 0.8944, - "total": 1485, - "trueNegative": 1298, - "truePositive": 161, - "wilsonPrecisionLower95": 0.9165 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.9618, - "falseNegative": 20, - "falsePositive": 7, - "precision": 0.9798, - "predictedPositive": 347, - "recall": 0.9444, - "total": 2970, - "trueNegative": 2603, - "truePositive": 340, - "wilsonPrecisionLower95": 0.959 - }, - "threshold": 0.5 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8765, - "falseNegative": 19, - "falsePositive": 1, - "precision": 0.9861, - "predictedPositive": 72, - "recall": 0.7889, - "total": 1485, - "trueNegative": 1394, - "truePositive": 71, - "wilsonPrecisionLower95": 0.9254 - }, - "threshold": 0.945 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6569, - "falseNegative": 45, - "falsePositive": 2, - "precision": 0.9574, - "predictedPositive": 47, - "recall": 0.5, - "total": 1485, - "trueNegative": 1393, - "truePositive": 45, - "wilsonPrecisionLower95": 0.8575 - }, - "threshold": 0.64 - } - }, - "metrics": { - "f1": 0.7108, - "falseNegative": 78, - "falsePositive": 5, - "precision": 0.9533, - "predictedPositive": 107, - "recall": 0.5667, - "total": 2970, - "trueNegative": 2785, - "truePositive": 102, - "wilsonPrecisionLower95": 0.8952 - }, - "threshold": 0.91 - }, - "followUpReminder": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8, - "falseNegative": 56, - "falsePositive": 6, - "precision": 0.9538, - "predictedPositive": 130, - "recall": 0.6889, - "total": 1485, - "trueNegative": 1299, - "truePositive": 124, - "wilsonPrecisionLower95": 0.903 - }, - "threshold": 0.98 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6165, - "falseNegative": 98, - "falsePositive": 4, - "precision": 0.9535, - "predictedPositive": 86, - "recall": 0.4556, - "total": 1485, - "trueNegative": 1301, - "truePositive": 82, - "wilsonPrecisionLower95": 0.8864 - }, - "threshold": 0.825 - } - }, - "metrics": { - "f1": 0.5912, - "falseNegative": 206, - "falsePositive": 7, - "precision": 0.9565, - "predictedPositive": 161, - "recall": 0.4278, - "total": 2970, - "trueNegative": 2603, - "truePositive": 154, - "wilsonPrecisionLower95": 0.913 - }, - "threshold": 0.98 - }, - "invitation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8302, - "falseNegative": 24, - "falsePositive": 3, - "precision": 0.9565, - "predictedPositive": 69, - "recall": 0.7333, - "total": 1485, - "trueNegative": 1392, - "truePositive": 66, - "wilsonPrecisionLower95": 0.8798 - }, - "threshold": 0.98 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.536, - "falseNegative": 113, - "falsePositive": 3, - "precision": 0.9571, - "predictedPositive": 70, - "recall": 0.3722, - "total": 2970, - "trueNegative": 2787, - "truePositive": 67, - "wilsonPrecisionLower95": 0.8814 - }, - "threshold": 0.995 - }, - "question": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.7813, - "falseNegative": 106, - "falsePositive": 11, - "precision": 0.95, - "predictedPositive": 220, - "recall": 0.6635, - "total": 1485, - "trueNegative": 1159, - "truePositive": 209, - "wilsonPrecisionLower95": 0.9127 - }, - "threshold": 0.865 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6844, - "falseNegative": 148, - "falsePositive": 6, - "precision": 0.9653, - "predictedPositive": 173, - "recall": 0.5302, - "total": 1485, - "trueNegative": 1164, - "truePositive": 167, - "wilsonPrecisionLower95": 0.9264 - }, - "threshold": 0.63 - } - }, - "metrics": { - "f1": 0.75, - "falseNegative": 240, - "falsePositive": 20, - "precision": 0.9512, - "predictedPositive": 410, - "recall": 0.619, - "total": 2970, - "trueNegative": 2320, - "truePositive": 390, - "wilsonPrecisionLower95": 0.9259 - }, - "threshold": 0.74 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9441, - "falseNegative": 59, - "falsePositive": 46, - "precision": 0.9506, - "predictedPositive": 932, - "recall": 0.9376, - "total": 1485, - "trueNegative": 494, - "truePositive": 886, - "wilsonPrecisionLower95": 0.9348 - }, - "threshold": 0.76 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8342, - "falseNegative": 243, - "falsePositive": 36, - "precision": 0.9512, - "predictedPositive": 738, - "recall": 0.7429, - "total": 1485, - "trueNegative": 504, - "truePositive": 702, - "wilsonPrecisionLower95": 0.9332 - }, - "threshold": 0.885 - } - }, - "metrics": { - "f1": 0.9056, - "falseNegative": 255, - "falsePositive": 86, - "precision": 0.95, - "predictedPositive": 1721, - "recall": 0.8651, - "total": 2970, - "trueNegative": 994, - "truePositive": 1635, - "wilsonPrecisionLower95": 0.9387 - }, - "threshold": 0.855 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 45, - "recall": 1.0, - "total": 1485, - "trueNegative": 1440, - "truePositive": 45, - "wilsonPrecisionLower95": 0.9213 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9318, - "falseNegative": 4, - "falsePositive": 2, - "precision": 0.9535, - "predictedPositive": 43, - "recall": 0.9111, - "total": 1485, - "trueNegative": 1438, - "truePositive": 41, - "wilsonPrecisionLower95": 0.8454 - }, - "threshold": 0.945 - } - }, - "metrics": { - "f1": 0.9727, - "falseNegative": 1, - "falsePositive": 4, - "precision": 0.957, - "predictedPositive": 93, - "recall": 0.9889, - "total": 2970, - "trueNegative": 2876, - "truePositive": 89, - "wilsonPrecisionLower95": 0.8946 - }, - "threshold": 0.62 - }, - "task": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.851, - "falseNegative": 83, - "falsePositive": 14, - "precision": 0.9519, - "predictedPositive": 291, - "recall": 0.7694, - "total": 1485, - "trueNegative": 1111, - "truePositive": 277, - "wilsonPrecisionLower95": 0.9209 - }, - "threshold": 0.87 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.4886, - "falseNegative": 242, - "falsePositive": 5, - "precision": 0.9593, - "predictedPositive": 123, - "recall": 0.3278, - "total": 1485, - "trueNegative": 1120, - "truePositive": 118, - "wilsonPrecisionLower95": 0.9084 - }, - "threshold": 0.935 - } - }, - "metrics": { - "f1": 0.6978, - "falseNegative": 324, - "falsePositive": 19, - "precision": 0.9542, - "predictedPositive": 415, - "recall": 0.55, - "total": 2970, - "trueNegative": 2231, - "truePositive": 396, - "wilsonPrecisionLower95": 0.9296 - }, - "threshold": 0.915 - } - }, - "trainingRecords": 15345, - "validation": { - "intentsMeeting95Precision": 9, - "macroF1": 0.7192, - "macroPrecision": 0.9671, - "macroRecall": 0.62, - "minimumPrecision": 0.9509, - "perIntent": { - "blessing": { - "f1": 0.2, - "falseNegative": 80, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 10, - "recall": 0.1111, - "total": 2970, - "trueNegative": 2880, - "truePositive": 10, - "wilsonPrecisionLower95": 0.7225 - }, - "complaint": { - "f1": 0.9618, - "falseNegative": 20, - "falsePositive": 7, - "precision": 0.9798, - "predictedPositive": 347, - "recall": 0.9444, - "total": 2970, - "trueNegative": 2603, - "truePositive": 340, - "wilsonPrecisionLower95": 0.959 - }, - "confirmationDecision": { - "f1": 0.7759, - "falseNegative": 64, - "falsePositive": 3, - "precision": 0.9748, - "predictedPositive": 119, - "recall": 0.6444, - "total": 2970, - "trueNegative": 2787, - "truePositive": 116, - "wilsonPrecisionLower95": 0.9285 - }, - "followUpReminder": { - "f1": 0.7153, - "falseNegative": 154, - "falsePositive": 10, - "precision": 0.9537, - "predictedPositive": 216, - "recall": 0.5722, - "total": 2970, - "trueNegative": 2600, - "truePositive": 206, - "wilsonPrecisionLower95": 0.9169 - }, - "invitation": { - "f1": 0.5301, - "falseNegative": 114, - "falsePositive": 3, - "precision": 0.9565, - "predictedPositive": 69, - "recall": 0.3667, - "total": 2970, - "trueNegative": 2787, - "truePositive": 66, - "wilsonPrecisionLower95": 0.8798 - }, - "question": { - "f1": 0.7351, - "falseNegative": 254, - "falsePositive": 17, - "precision": 0.9567, - "predictedPositive": 393, - "recall": 0.5968, - "total": 2970, - "trueNegative": 2323, - "truePositive": 376, - "wilsonPrecisionLower95": 0.9318 - }, - "replyableMessage": { - "f1": 0.8921, - "falseNegative": 302, - "falsePositive": 82, - "precision": 0.9509, - "predictedPositive": 1670, - "recall": 0.8402, - "total": 2970, - "trueNegative": 998, - "truePositive": 1588, - "wilsonPrecisionLower95": 0.9395 - }, - "scheduleNegotiation": { - "f1": 0.9663, - "falseNegative": 4, - "falsePositive": 2, - "precision": 0.9773, - "predictedPositive": 88, - "recall": 0.9556, - "total": 2970, - "trueNegative": 2878, - "truePositive": 86, - "wilsonPrecisionLower95": 0.9209 - }, - "task": { - "f1": 0.6966, - "falseNegative": 325, - "falsePositive": 19, - "precision": 0.9541, - "predictedPositive": 414, - "recall": 0.5486, - "total": 2970, - "trueNegative": 2231, - "truePositive": 395, - "wilsonPrecisionLower95": 0.9294 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 1.2e-05, - "augmentation": "none", - "char_max": 5, - "char_min": 2, - "external_weight": 0.65, - "hard_example_weight": 1.4, - "l1_ratio": 0.1, - "max_char_features": 60000, - "max_word_features": 32000, - "min_df": 1, - "pseudo_weight": 0.22499999999999998, - "round": 5, - "word_max": 3 - }, - "featureCount": 92000, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 0 - }, - "complaint": { - "negative": 88, - "positive": 0 - }, - "confirmationDecision": { - "negative": 96, - "positive": 3 - }, - "followUpReminder": { - "negative": 250, - "positive": 0 - }, - "invitation": { - "negative": 250, - "positive": 0 - }, - "question": { - "negative": 40, - "positive": 1 - }, - "replyableMessage": { - "negative": 3, - "positive": 150 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 0 - }, - "task": { - "negative": 97, - "positive": 0 - } - }, - "round": 5, - "selectionObjective": 0.7771, - "syntheticRegression": { - "intentsMeeting95Precision": 5, - "macroF1": 0.6709, - "macroPrecision": 0.9586, - "macroRecall": 0.5803, - "minimumPrecision": 0.8588, - "perIntent": { - "blessing": { - "f1": 0.022, - "falseNegative": 89, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 1, - "recall": 0.0111, - "total": 2970, - "trueNegative": 2880, - "truePositive": 1, - "wilsonPrecisionLower95": 0.2065 - }, - "complaint": { - "f1": 0.967, - "falseNegative": 23, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 337, - "recall": 0.9361, - "total": 2970, - "trueNegative": 2610, - "truePositive": 337, - "wilsonPrecisionLower95": 0.9887 - }, - "confirmationDecision": { - "f1": 0.8343, - "falseNegative": 34, - "falsePositive": 24, - "precision": 0.8588, - "predictedPositive": 170, - "recall": 0.8111, - "total": 2970, - "trueNegative": 2766, - "truePositive": 146, - "wilsonPrecisionLower95": 0.7985 - }, - "followUpReminder": { - "f1": 0.7173, - "falseNegative": 157, - "falsePositive": 3, - "precision": 0.9854, - "predictedPositive": 206, - "recall": 0.5639, - "total": 2970, - "trueNegative": 2607, - "truePositive": 203, - "wilsonPrecisionLower95": 0.9581 - }, - "invitation": { - "f1": 0.3612, - "falseNegative": 139, - "falsePositive": 6, - "precision": 0.8723, - "predictedPositive": 47, - "recall": 0.2278, - "total": 2970, - "trueNegative": 2784, - "truePositive": 41, - "wilsonPrecisionLower95": 0.7483 - }, - "question": { - "f1": 0.8336, - "falseNegative": 179, - "falsePositive": 1, - "precision": 0.9978, - "predictedPositive": 452, - "recall": 0.7159, - "total": 2970, - "trueNegative": 2339, - "truePositive": 451, - "wilsonPrecisionLower95": 0.9876 - }, - "replyableMessage": { - "f1": 0.912, - "falseNegative": 257, - "falsePositive": 58, - "precision": 0.9657, - "predictedPositive": 1691, - "recall": 0.864, - "total": 2970, - "trueNegative": 1022, - "truePositive": 1633, - "wilsonPrecisionLower95": 0.9559 - }, - "scheduleNegotiation": { - "f1": 0.6364, - "falseNegative": 48, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 42, - "recall": 0.4667, - "total": 2970, - "trueNegative": 2880, - "truePositive": 42, - "wilsonPrecisionLower95": 0.9162 - }, - "task": { - "f1": 0.7542, - "falseNegative": 269, - "falsePositive": 25, - "precision": 0.9475, - "predictedPositive": 476, - "recall": 0.6264, - "total": 2970, - "trueNegative": 2225, - "truePositive": 451, - "wilsonPrecisionLower95": 0.9236 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 180, - "recall": 1.0, - "total": 1485, - "trueNegative": 1305, - "truePositive": 180, - "wilsonPrecisionLower95": 0.9791 - }, - "threshold": 0.69 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9288, - "falseNegative": 17, - "falsePositive": 8, - "precision": 0.9532, - "predictedPositive": 171, - "recall": 0.9056, - "total": 1485, - "trueNegative": 1297, - "truePositive": 163, - "wilsonPrecisionLower95": 0.9104 - }, - "threshold": 0.51 - } - }, - "metrics": { - "f1": 0.9569, - "falseNegative": 16, - "falsePositive": 15, - "precision": 0.9582, - "predictedPositive": 359, - "recall": 0.9556, - "total": 2970, - "trueNegative": 2595, - "truePositive": 344, - "wilsonPrecisionLower95": 0.9322 - }, - "threshold": 0.5 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8589, - "falseNegative": 20, - "falsePositive": 3, - "precision": 0.9589, - "predictedPositive": 73, - "recall": 0.7778, - "total": 1485, - "trueNegative": 1392, - "truePositive": 70, - "wilsonPrecisionLower95": 0.886 - }, - "threshold": 0.57 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6763, - "falseNegative": 43, - "falsePositive": 2, - "precision": 0.9592, - "predictedPositive": 49, - "recall": 0.5222, - "total": 1485, - "trueNegative": 1393, - "truePositive": 47, - "wilsonPrecisionLower95": 0.8629 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.76, - "falseNegative": 66, - "falsePositive": 6, - "precision": 0.95, - "predictedPositive": 120, - "recall": 0.6333, - "total": 2970, - "trueNegative": 2784, - "truePositive": 114, - "wilsonPrecisionLower95": 0.8952 - }, - "threshold": 0.56 - }, - "followUpReminder": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8, - "falseNegative": 56, - "falsePositive": 6, - "precision": 0.9538, - "predictedPositive": 130, - "recall": 0.6889, - "total": 1485, - "trueNegative": 1299, - "truePositive": 124, - "wilsonPrecisionLower95": 0.903 - }, - "threshold": 0.96 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7708, - "falseNegative": 64, - "falsePositive": 5, - "precision": 0.9587, - "predictedPositive": 121, - "recall": 0.6444, - "total": 1485, - "trueNegative": 1300, - "truePositive": 116, - "wilsonPrecisionLower95": 0.9069 - }, - "threshold": 0.58 - } - }, - "metrics": { - "f1": 0.6892, - "falseNegative": 166, - "falsePositive": 9, - "precision": 0.9557, - "predictedPositive": 203, - "recall": 0.5389, - "total": 2970, - "trueNegative": 2601, - "truePositive": 194, - "wilsonPrecisionLower95": 0.9179 - }, - "threshold": 0.915 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.5203, - "falseNegative": 58, - "falsePositive": 1, - "precision": 0.9697, - "predictedPositive": 33, - "recall": 0.3556, - "total": 1485, - "trueNegative": 1394, - "truePositive": 32, - "wilsonPrecisionLower95": 0.8468 - }, - "threshold": 0.88 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8225, - "falseNegative": 88, - "falsePositive": 10, - "precision": 0.9578, - "predictedPositive": 237, - "recall": 0.7206, - "total": 1485, - "trueNegative": 1160, - "truePositive": 227, - "wilsonPrecisionLower95": 0.9241 - }, - "threshold": 0.84 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8388, - "falseNegative": 86, - "falsePositive": 2, - "precision": 0.9913, - "predictedPositive": 231, - "recall": 0.727, - "total": 1485, - "trueNegative": 1168, - "truePositive": 229, - "wilsonPrecisionLower95": 0.969 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.8591, - "falseNegative": 136, - "falsePositive": 26, - "precision": 0.95, - "predictedPositive": 520, - "recall": 0.7841, - "total": 2970, - "trueNegative": 2314, - "truePositive": 494, - "wilsonPrecisionLower95": 0.9277 - }, - "threshold": 0.505 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9541, - "falseNegative": 40, - "falsePositive": 47, - "precision": 0.9506, - "predictedPositive": 952, - "recall": 0.9577, - "total": 1485, - "trueNegative": 493, - "truePositive": 905, - "wilsonPrecisionLower95": 0.935 - }, - "threshold": 0.69 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8807, - "falseNegative": 170, - "falsePositive": 40, - "precision": 0.9509, - "predictedPositive": 815, - "recall": 0.8201, - "total": 1485, - "trueNegative": 500, - "truePositive": 775, - "wilsonPrecisionLower95": 0.9339 - }, - "threshold": 0.8 - } - }, - "metrics": { - "f1": 0.914, - "falseNegative": 227, - "falsePositive": 86, - "precision": 0.9508, - "predictedPositive": 1749, - "recall": 0.8799, - "total": 2970, - "trueNegative": 994, - "truePositive": 1663, - "wilsonPrecisionLower95": 0.9397 - }, - "threshold": 0.78 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9888, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 44, - "recall": 0.9778, - "total": 1485, - "trueNegative": 1440, - "truePositive": 44, - "wilsonPrecisionLower95": 0.9197 - }, - "threshold": 0.635 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.8129, - "falseNegative": 27, - "falsePositive": 2, - "precision": 0.9692, - "predictedPositive": 65, - "recall": 0.7, - "total": 2970, - "trueNegative": 2878, - "truePositive": 63, - "wilsonPrecisionLower95": 0.8946 - }, - "threshold": 0.995 - }, - "task": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8185, - "falseNegative": 103, - "falsePositive": 11, - "precision": 0.959, - "predictedPositive": 268, - "recall": 0.7139, - "total": 1485, - "trueNegative": 1114, - "truePositive": 257, - "wilsonPrecisionLower95": 0.928 - }, - "threshold": 0.83 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7702, - "falseNegative": 127, - "falsePositive": 12, - "precision": 0.951, - "predictedPositive": 245, - "recall": 0.6472, - "total": 1485, - "trueNegative": 1113, - "truePositive": 233, - "wilsonPrecisionLower95": 0.9164 - }, - "threshold": 0.765 - } - }, - "metrics": { - "f1": 0.7816, - "falseNegative": 244, - "falsePositive": 22, - "precision": 0.9558, - "predictedPositive": 498, - "recall": 0.6611, - "total": 2970, - "trueNegative": 2228, - "truePositive": 476, - "wilsonPrecisionLower95": 0.934 - }, - "threshold": 0.805 - } - }, - "trainingRecords": 15345, - "validation": { - "intentsMeeting95Precision": 8, - "macroF1": 0.6867, - "macroPrecision": 0.9721, - "macroRecall": 0.5915, - "minimumPrecision": 0.9508, - "perIntent": { - "blessing": { - "f1": 0.0851, - "falseNegative": 86, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 4, - "recall": 0.0444, - "total": 2970, - "trueNegative": 2880, - "truePositive": 4, - "wilsonPrecisionLower95": 0.5101 - }, - "complaint": { - "f1": 0.9648, - "falseNegative": 17, - "falsePositive": 8, - "precision": 0.9772, - "predictedPositive": 351, - "recall": 0.9528, - "total": 2970, - "trueNegative": 2602, - "truePositive": 343, - "wilsonPrecisionLower95": 0.9557 - }, - "confirmationDecision": { - "f1": 0.7748, - "falseNegative": 63, - "falsePositive": 5, - "precision": 0.959, - "predictedPositive": 122, - "recall": 0.65, - "total": 2970, - "trueNegative": 2785, - "truePositive": 117, - "wilsonPrecisionLower95": 0.9076 - }, - "followUpReminder": { - "f1": 0.7856, - "falseNegative": 120, - "falsePositive": 11, - "precision": 0.9562, - "predictedPositive": 251, - "recall": 0.6667, - "total": 2970, - "trueNegative": 2599, - "truePositive": 240, - "wilsonPrecisionLower95": 0.9232 - }, - "invitation": { - "f1": 0.3694, - "falseNegative": 139, - "falsePositive": 1, - "precision": 0.9762, - "predictedPositive": 42, - "recall": 0.2278, - "total": 2970, - "trueNegative": 2789, - "truePositive": 41, - "wilsonPrecisionLower95": 0.8768 - }, - "question": { - "f1": 0.8306, - "falseNegative": 174, - "falsePositive": 12, - "precision": 0.9744, - "predictedPositive": 468, - "recall": 0.7238, - "total": 2970, - "trueNegative": 2328, - "truePositive": 456, - "wilsonPrecisionLower95": 0.9557 - }, - "replyableMessage": { - "f1": 0.9188, - "falseNegative": 210, - "falsePositive": 87, - "precision": 0.9508, - "predictedPositive": 1767, - "recall": 0.8889, - "total": 2970, - "trueNegative": 993, - "truePositive": 1680, - "wilsonPrecisionLower95": 0.9397 - }, - "scheduleNegotiation": { - "f1": 0.6567, - "falseNegative": 46, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 44, - "recall": 0.4889, - "total": 2970, - "trueNegative": 2880, - "truePositive": 44, - "wilsonPrecisionLower95": 0.9197 - }, - "task": { - "f1": 0.7948, - "falseNegative": 230, - "falsePositive": 23, - "precision": 0.9552, - "predictedPositive": 513, - "recall": 0.6806, - "total": 2970, - "trueNegative": 2227, - "truePositive": 490, - "wilsonPrecisionLower95": 0.9336 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 8e-06, - "augmentation": "punctuation", - "char_max": 6, - "char_min": 2, - "external_weight": 0.67, - "hard_example_weight": 1.5, - "l1_ratio": 0.1, - "max_char_features": 65000, - "max_word_features": 28000, - "min_df": 2, - "pseudo_weight": 0.24, - "round": 6, - "word_max": 2 - }, - "featureCount": 78354, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 0 - }, - "complaint": { - "negative": 146, - "positive": 0 - }, - "confirmationDecision": { - "negative": 88, - "positive": 2 - }, - "followUpReminder": { - "negative": 250, - "positive": 0 - }, - "invitation": { - "negative": 250, - "positive": 0 - }, - "question": { - "negative": 36, - "positive": 5 - }, - "replyableMessage": { - "negative": 2, - "positive": 130 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 0 - }, - "task": { - "negative": 83, - "positive": 0 - } - }, - "round": 6, - "selectionObjective": 0.811, - "syntheticRegression": { - "intentsMeeting95Precision": 5, - "macroF1": 0.7064, - "macroPrecision": 0.9443, - "macroRecall": 0.623, - "minimumPrecision": 0.7975, - "perIntent": { - "blessing": { - "f1": 0.0435, - "falseNegative": 88, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 2, - "recall": 0.0222, - "total": 2970, - "trueNegative": 2880, - "truePositive": 2, - "wilsonPrecisionLower95": 0.3424 - }, - "complaint": { - "f1": 0.967, - "falseNegative": 23, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 337, - "recall": 0.9361, - "total": 2970, - "trueNegative": 2610, - "truePositive": 337, - "wilsonPrecisionLower95": 0.9887 - }, - "confirmationDecision": { - "f1": 0.7215, - "falseNegative": 66, - "falsePositive": 22, - "precision": 0.8382, - "predictedPositive": 136, - "recall": 0.6333, - "total": 2970, - "trueNegative": 2768, - "truePositive": 114, - "wilsonPrecisionLower95": 0.7672 - }, - "followUpReminder": { - "f1": 0.6847, - "falseNegative": 170, - "falsePositive": 5, - "precision": 0.9744, - "predictedPositive": 195, - "recall": 0.5278, - "total": 2970, - "trueNegative": 2605, - "truePositive": 190, - "wilsonPrecisionLower95": 0.9414 - }, - "invitation": { - "f1": 0.4865, - "falseNegative": 117, - "falsePositive": 16, - "precision": 0.7975, - "predictedPositive": 79, - "recall": 0.35, - "total": 2970, - "trueNegative": 2774, - "truePositive": 63, - "wilsonPrecisionLower95": 0.696 - }, - "question": { - "f1": 0.8556, - "falseNegative": 156, - "falsePositive": 4, - "precision": 0.9916, - "predictedPositive": 478, - "recall": 0.7524, - "total": 2970, - "trueNegative": 2336, - "truePositive": 474, - "wilsonPrecisionLower95": 0.9787 - }, - "replyableMessage": { - "f1": 0.8888, - "falseNegative": 335, - "falsePositive": 54, - "precision": 0.9664, - "predictedPositive": 1609, - "recall": 0.8228, - "total": 2970, - "trueNegative": 1026, - "truePositive": 1555, - "wilsonPrecisionLower95": 0.9565 - }, - "scheduleNegotiation": { - "f1": 0.9831, - "falseNegative": 3, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 87, - "recall": 0.9667, - "total": 2970, - "trueNegative": 2880, - "truePositive": 87, - "wilsonPrecisionLower95": 0.9577 - }, - "task": { - "f1": 0.7265, - "falseNegative": 291, - "falsePositive": 32, - "precision": 0.9306, - "predictedPositive": 461, - "recall": 0.5958, - "total": 2970, - "trueNegative": 2218, - "truePositive": 429, - "wilsonPrecisionLower95": 0.9037 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9834, - "falseNegative": 2, - "falsePositive": 4, - "precision": 0.978, - "predictedPositive": 182, - "recall": 0.9889, - "total": 1485, - "trueNegative": 1301, - "truePositive": 178, - "wilsonPrecisionLower95": 0.9449 - }, - "threshold": 0.555 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9494, - "falseNegative": 11, - "falsePositive": 7, - "precision": 0.9602, - "predictedPositive": 176, - "recall": 0.9389, - "total": 1485, - "trueNegative": 1298, - "truePositive": 169, - "wilsonPrecisionLower95": 0.9202 - }, - "threshold": 0.535 - } - }, - "metrics": { - "f1": 0.9652, - "falseNegative": 13, - "falsePositive": 12, - "precision": 0.9666, - "predictedPositive": 359, - "recall": 0.9639, - "total": 2970, - "trueNegative": 2598, - "truePositive": 347, - "wilsonPrecisionLower95": 0.9425 - }, - "threshold": 0.535 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8, - "falseNegative": 28, - "falsePositive": 3, - "precision": 0.9538, - "predictedPositive": 65, - "recall": 0.6889, - "total": 1485, - "trueNegative": 1392, - "truePositive": 62, - "wilsonPrecisionLower95": 0.8729 - }, - "threshold": 0.695 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.731, - "falseNegative": 37, - "falsePositive": 2, - "precision": 0.9636, - "predictedPositive": 55, - "recall": 0.5889, - "total": 1485, - "trueNegative": 1393, - "truePositive": 53, - "wilsonPrecisionLower95": 0.8768 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.7354, - "falseNegative": 73, - "falsePositive": 4, - "precision": 0.964, - "predictedPositive": 111, - "recall": 0.5944, - "total": 2970, - "trueNegative": 2786, - "truePositive": 107, - "wilsonPrecisionLower95": 0.911 - }, - "threshold": 0.695 - }, - "followUpReminder": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8632, - "falseNegative": 38, - "falsePositive": 7, - "precision": 0.953, - "predictedPositive": 149, - "recall": 0.7889, - "total": 1485, - "trueNegative": 1298, - "truePositive": 142, - "wilsonPrecisionLower95": 0.9062 - }, - "threshold": 0.9 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.4938, - "falseNegative": 120, - "falsePositive": 3, - "precision": 0.9524, - "predictedPositive": 63, - "recall": 0.3333, - "total": 1485, - "trueNegative": 1302, - "truePositive": 60, - "wilsonPrecisionLower95": 0.8691 - }, - "threshold": 0.845 - } - }, - "metrics": { - "f1": 0.7018, - "falseNegative": 160, - "falsePositive": 10, - "precision": 0.9524, - "predictedPositive": 210, - "recall": 0.5556, - "total": 2970, - "trueNegative": 2600, - "truePositive": 200, - "wilsonPrecisionLower95": 0.9146 - }, - "threshold": 0.885 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7222, - "falseNegative": 38, - "falsePositive": 2, - "precision": 0.963, - "predictedPositive": 54, - "recall": 0.5778, - "total": 1485, - "trueNegative": 1393, - "truePositive": 52, - "wilsonPrecisionLower95": 0.8746 - }, - "threshold": 0.665 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8167, - "falseNegative": 90, - "falsePositive": 11, - "precision": 0.9534, - "predictedPositive": 236, - "recall": 0.7143, - "total": 1485, - "trueNegative": 1159, - "truePositive": 225, - "wilsonPrecisionLower95": 0.9185 - }, - "threshold": 0.865 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8205, - "falseNegative": 91, - "falsePositive": 7, - "precision": 0.9697, - "predictedPositive": 231, - "recall": 0.7111, - "total": 1485, - "trueNegative": 1163, - "truePositive": 224, - "wilsonPrecisionLower95": 0.9388 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.804, - "falseNegative": 191, - "falsePositive": 23, - "precision": 0.9502, - "predictedPositive": 462, - "recall": 0.6968, - "total": 2970, - "trueNegative": 2317, - "truePositive": 439, - "wilsonPrecisionLower95": 0.9264 - }, - "threshold": 0.68 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9268, - "falseNegative": 90, - "falsePositive": 45, - "precision": 0.95, - "predictedPositive": 900, - "recall": 0.9048, - "total": 1485, - "trueNegative": 495, - "truePositive": 855, - "wilsonPrecisionLower95": 0.9338 - }, - "threshold": 0.86 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8789, - "falseNegative": 176, - "falsePositive": 36, - "precision": 0.9553, - "predictedPositive": 805, - "recall": 0.8138, - "total": 1485, - "trueNegative": 504, - "truePositive": 769, - "wilsonPrecisionLower95": 0.9387 - }, - "threshold": 0.85 - } - }, - "metrics": { - "f1": 0.9046, - "falseNegative": 259, - "falsePositive": 85, - "precision": 0.9505, - "predictedPositive": 1716, - "recall": 0.863, - "total": 2970, - "trueNegative": 995, - "truePositive": 1631, - "wilsonPrecisionLower95": 0.9392 - }, - "threshold": 0.85 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9773, - "falseNegative": 2, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 43, - "recall": 0.9556, - "total": 1485, - "trueNegative": 1440, - "truePositive": 43, - "wilsonPrecisionLower95": 0.918 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9888, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 44, - "recall": 0.9778, - "total": 1485, - "trueNegative": 1440, - "truePositive": 44, - "wilsonPrecisionLower95": 0.9197 - }, - "threshold": 0.97 - } - }, - "metrics": { - "f1": 0.9497, - "falseNegative": 5, - "falsePositive": 4, - "precision": 0.9551, - "predictedPositive": 89, - "recall": 0.9444, - "total": 2970, - "trueNegative": 2876, - "truePositive": 85, - "wilsonPrecisionLower95": 0.8901 - }, - "threshold": 0.91 - }, - "task": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.827, - "falseNegative": 97, - "falsePositive": 13, - "precision": 0.9529, - "predictedPositive": 276, - "recall": 0.7306, - "total": 1485, - "trueNegative": 1112, - "truePositive": 263, - "wilsonPrecisionLower95": 0.9211 - }, - "threshold": 0.89 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7902, - "falseNegative": 117, - "falsePositive": 12, - "precision": 0.9529, - "predictedPositive": 255, - "recall": 0.675, - "total": 1485, - "trueNegative": 1113, - "truePositive": 243, - "wilsonPrecisionLower95": 0.9196 - }, - "threshold": 0.785 - } - }, - "metrics": { - "f1": 0.7783, - "falseNegative": 246, - "falsePositive": 24, - "precision": 0.9518, - "predictedPositive": 498, - "recall": 0.6583, - "total": 2970, - "trueNegative": 2226, - "truePositive": 474, - "wilsonPrecisionLower95": 0.9293 - }, - "threshold": 0.85 - } - }, - "trainingRecords": 17767, - "validation": { - "intentsMeeting95Precision": 8, - "macroF1": 0.7395, - "macroPrecision": 0.9687, - "macroRecall": 0.6518, - "minimumPrecision": 0.9525, - "perIntent": { - "blessing": { - "f1": 0.1633, - "falseNegative": 82, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 8, - "recall": 0.0889, - "total": 2970, - "trueNegative": 2880, - "truePositive": 8, - "wilsonPrecisionLower95": 0.6756 - }, - "complaint": { - "f1": 0.9666, - "falseNegative": 13, - "falsePositive": 11, - "precision": 0.9693, - "predictedPositive": 358, - "recall": 0.9639, - "total": 2970, - "trueNegative": 2599, - "truePositive": 347, - "wilsonPrecisionLower95": 0.9458 - }, - "confirmationDecision": { - "f1": 0.7667, - "falseNegative": 65, - "falsePositive": 5, - "precision": 0.9583, - "predictedPositive": 120, - "recall": 0.6389, - "total": 2970, - "trueNegative": 2785, - "truePositive": 115, - "wilsonPrecisionLower95": 0.9062 - }, - "followUpReminder": { - "f1": 0.7063, - "falseNegative": 158, - "falsePositive": 10, - "precision": 0.9528, - "predictedPositive": 212, - "recall": 0.5611, - "total": 2970, - "trueNegative": 2600, - "truePositive": 202, - "wilsonPrecisionLower95": 0.9154 - }, - "invitation": { - "f1": 0.5382, - "falseNegative": 113, - "falsePositive": 2, - "precision": 0.971, - "predictedPositive": 69, - "recall": 0.3722, - "total": 2970, - "trueNegative": 2788, - "truePositive": 67, - "wilsonPrecisionLower95": 0.9003 - }, - "question": { - "f1": 0.8186, - "falseNegative": 181, - "falsePositive": 18, - "precision": 0.9615, - "predictedPositive": 467, - "recall": 0.7127, - "total": 2970, - "trueNegative": 2322, - "truePositive": 449, - "wilsonPrecisionLower95": 0.9399 - }, - "replyableMessage": { - "f1": 0.9035, - "falseNegative": 266, - "falsePositive": 81, - "precision": 0.9525, - "predictedPositive": 1705, - "recall": 0.8593, - "total": 2970, - "trueNegative": 999, - "truePositive": 1624, - "wilsonPrecisionLower95": 0.9413 - }, - "scheduleNegotiation": { - "f1": 0.9831, - "falseNegative": 3, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 87, - "recall": 0.9667, - "total": 2970, - "trueNegative": 2880, - "truePositive": 87, - "wilsonPrecisionLower95": 0.9577 - }, - "task": { - "f1": 0.809, - "falseNegative": 214, - "falsePositive": 25, - "precision": 0.9529, - "predictedPositive": 531, - "recall": 0.7028, - "total": 2970, - "trueNegative": 2225, - "truePositive": 506, - "wilsonPrecisionLower95": 0.9314 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 6e-06, - "augmentation": "punctuation", - "char_max": 6, - "char_min": 3, - "external_weight": 0.6900000000000001, - "hard_example_weight": 1.5, - "l1_ratio": 0.1, - "max_char_features": 65000, - "max_word_features": 32000, - "min_df": 2, - "pseudo_weight": 0.255, - "round": 7, - "word_max": 3 - }, - "featureCount": 87507, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 0 - }, - "complaint": { - "negative": 141, - "positive": 0 - }, - "confirmationDecision": { - "negative": 95, - "positive": 2 - }, - "followUpReminder": { - "negative": 250, - "positive": 0 - }, - "invitation": { - "negative": 250, - "positive": 0 - }, - "question": { - "negative": 36, - "positive": 5 - }, - "replyableMessage": { - "negative": 2, - "positive": 129 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 0 - }, - "task": { - "negative": 87, - "positive": 0 - } - }, - "round": 7, - "selectionObjective": 0.7501, - "syntheticRegression": { - "intentsMeeting95Precision": 5, - "macroF1": 0.587, - "macroPrecision": 0.9374, - "macroRecall": 0.4917, - "minimumPrecision": 0.7548, - "perIntent": { - "blessing": { - "f1": 0.0851, - "falseNegative": 86, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 4, - "recall": 0.0444, - "total": 2970, - "trueNegative": 2880, - "truePositive": 4, - "wilsonPrecisionLower95": 0.5101 - }, - "complaint": { - "f1": 0.9304, - "falseNegative": 39, - "falsePositive": 9, - "precision": 0.9727, - "predictedPositive": 330, - "recall": 0.8917, - "total": 2970, - "trueNegative": 2601, - "truePositive": 321, - "wilsonPrecisionLower95": 0.949 - }, - "confirmationDecision": { - "f1": 0.6985, - "falseNegative": 63, - "falsePositive": 38, - "precision": 0.7548, - "predictedPositive": 155, - "recall": 0.65, - "total": 2970, - "trueNegative": 2752, - "truePositive": 117, - "wilsonPrecisionLower95": 0.6815 - }, - "followUpReminder": { - "f1": 0.3974, - "falseNegative": 269, - "falsePositive": 7, - "precision": 0.9286, - "predictedPositive": 98, - "recall": 0.2528, - "total": 2970, - "trueNegative": 2603, - "truePositive": 91, - "wilsonPrecisionLower95": 0.8598 - }, - "invitation": { - "f1": 0.2609, - "falseNegative": 153, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 27, - "recall": 0.15, - "total": 2970, - "trueNegative": 2790, - "truePositive": 27, - "wilsonPrecisionLower95": 0.8754 - }, - "question": { - "f1": 0.697, - "falseNegative": 285, - "falsePositive": 15, - "precision": 0.9583, - "predictedPositive": 360, - "recall": 0.5476, - "total": 2970, - "trueNegative": 2325, - "truePositive": 345, - "wilsonPrecisionLower95": 0.9324 - }, - "replyableMessage": { - "f1": 0.8083, - "falseNegative": 572, - "falsePositive": 53, - "precision": 0.9613, - "predictedPositive": 1371, - "recall": 0.6974, - "total": 2970, - "trueNegative": 1027, - "truePositive": 1318, - "wilsonPrecisionLower95": 0.9498 - }, - "scheduleNegotiation": { - "f1": 0.8929, - "falseNegative": 15, - "falsePositive": 3, - "precision": 0.9615, - "predictedPositive": 78, - "recall": 0.8333, - "total": 2970, - "trueNegative": 2877, - "truePositive": 75, - "wilsonPrecisionLower95": 0.8929 - }, - "task": { - "f1": 0.5124, - "falseNegative": 462, - "falsePositive": 29, - "precision": 0.899, - "predictedPositive": 287, - "recall": 0.3583, - "total": 2970, - "trueNegative": 2221, - "truePositive": 258, - "wilsonPrecisionLower95": 0.8586 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9809, - "falseNegative": 0, - "falsePositive": 7, - "precision": 0.9626, - "predictedPositive": 187, - "recall": 1.0, - "total": 1485, - "trueNegative": 1298, - "truePositive": 180, - "wilsonPrecisionLower95": 0.9248 - }, - "threshold": 0.74 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8974, - "falseNegative": 27, - "falsePositive": 8, - "precision": 0.9503, - "predictedPositive": 161, - "recall": 0.85, - "total": 1485, - "trueNegative": 1297, - "truePositive": 153, - "wilsonPrecisionLower95": 0.905 - }, - "threshold": 0.665 - } - }, - "metrics": { - "f1": 0.9529, - "falseNegative": 16, - "falsePositive": 18, - "precision": 0.9503, - "predictedPositive": 362, - "recall": 0.9556, - "total": 2970, - "trueNegative": 2592, - "truePositive": 344, - "wilsonPrecisionLower95": 0.9228 - }, - "threshold": 0.51 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.7763, - "falseNegative": 31, - "falsePositive": 3, - "precision": 0.9516, - "predictedPositive": 62, - "recall": 0.6556, - "total": 1485, - "trueNegative": 1392, - "truePositive": 59, - "wilsonPrecisionLower95": 0.8671 - }, - "threshold": 0.895 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7568, - "falseNegative": 34, - "falsePositive": 2, - "precision": 0.9655, - "predictedPositive": 58, - "recall": 0.6222, - "total": 1485, - "trueNegative": 1393, - "truePositive": 56, - "wilsonPrecisionLower95": 0.8827 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.6569, - "falseNegative": 90, - "falsePositive": 4, - "precision": 0.9574, - "predictedPositive": 94, - "recall": 0.5, - "total": 2970, - "trueNegative": 2786, - "truePositive": 90, - "wilsonPrecisionLower95": 0.8956 - }, - "threshold": 0.895 - }, - "followUpReminder": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8421, - "falseNegative": 44, - "falsePositive": 7, - "precision": 0.951, - "predictedPositive": 143, - "recall": 0.7556, - "total": 1485, - "trueNegative": 1298, - "truePositive": 136, - "wilsonPrecisionLower95": 0.9024 - }, - "threshold": 0.985 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.5985, - "falseNegative": 205, - "falsePositive": 3, - "precision": 0.981, - "predictedPositive": 158, - "recall": 0.4306, - "total": 2970, - "trueNegative": 2607, - "truePositive": 155, - "wilsonPrecisionLower95": 0.9457 - }, - "threshold": 0.995 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.7143, - "falseNegative": 135, - "falsePositive": 9, - "precision": 0.9524, - "predictedPositive": 189, - "recall": 0.5714, - "total": 1485, - "trueNegative": 1161, - "truePositive": 180, - "wilsonPrecisionLower95": 0.912 - }, - "threshold": 0.965 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7168, - "falseNegative": 134, - "falsePositive": 9, - "precision": 0.9526, - "predictedPositive": 190, - "recall": 0.5746, - "total": 1485, - "trueNegative": 1161, - "truePositive": 181, - "wilsonPrecisionLower95": 0.9124 - }, - "threshold": 0.52 - } - }, - "metrics": { - "f1": 0.7149, - "falseNegative": 269, - "falsePositive": 19, - "precision": 0.95, - "predictedPositive": 380, - "recall": 0.573, - "total": 2970, - "trueNegative": 2321, - "truePositive": 361, - "wilsonPrecisionLower95": 0.9232 - }, - "threshold": 0.795 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8846, - "falseNegative": 163, - "falsePositive": 41, - "precision": 0.9502, - "predictedPositive": 823, - "recall": 0.8275, - "total": 1485, - "trueNegative": 499, - "truePositive": 782, - "wilsonPrecisionLower95": 0.9331 - }, - "threshold": 0.89 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7476, - "falseNegative": 363, - "falsePositive": 30, - "precision": 0.951, - "predictedPositive": 612, - "recall": 0.6159, - "total": 1485, - "trueNegative": 510, - "truePositive": 582, - "wilsonPrecisionLower95": 0.9309 - }, - "threshold": 0.92 - } - }, - "metrics": { - "f1": 0.828, - "falseNegative": 506, - "falsePositive": 69, - "precision": 0.9525, - "predictedPositive": 1453, - "recall": 0.7323, - "total": 2970, - "trueNegative": 1011, - "truePositive": 1384, - "wilsonPrecisionLower95": 0.9403 - }, - "threshold": 0.91 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 45, - "recall": 1.0, - "total": 1485, - "trueNegative": 1440, - "truePositive": 45, - "wilsonPrecisionLower95": 0.9213 - }, - "threshold": 0.66 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 45, - "recall": 1.0, - "total": 1485, - "trueNegative": 1440, - "truePositive": 45, - "wilsonPrecisionLower95": 0.9213 - }, - "threshold": 0.55 - } - }, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 90, - "recall": 1.0, - "total": 2970, - "trueNegative": 2880, - "truePositive": 90, - "wilsonPrecisionLower95": 0.9591 - }, - "threshold": 0.66 - }, - "task": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8615, - "falseNegative": 77, - "falsePositive": 14, - "precision": 0.9529, - "predictedPositive": 297, - "recall": 0.7861, - "total": 1485, - "trueNegative": 1111, - "truePositive": 283, - "wilsonPrecisionLower95": 0.9224 - }, - "threshold": 0.805 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.6868, - "falseNegative": 333, - "falsePositive": 20, - "precision": 0.9509, - "predictedPositive": 407, - "recall": 0.5375, - "total": 2970, - "trueNegative": 2230, - "truePositive": 387, - "wilsonPrecisionLower95": 0.9253 - }, - "threshold": 0.925 - } - }, - "trainingRecords": 17708, - "validation": { - "intentsMeeting95Precision": 8, - "macroF1": 0.6491, - "macroPrecision": 0.967, - "macroRecall": 0.5465, - "minimumPrecision": 0.9315, - "perIntent": { - "blessing": { - "f1": 0.2353, - "falseNegative": 78, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 12, - "recall": 0.1333, - "total": 2970, - "trueNegative": 2880, - "truePositive": 12, - "wilsonPrecisionLower95": 0.7575 - }, - "complaint": { - "f1": 0.9407, - "falseNegative": 27, - "falsePositive": 15, - "precision": 0.9569, - "predictedPositive": 348, - "recall": 0.925, - "total": 2970, - "trueNegative": 2595, - "truePositive": 333, - "wilsonPrecisionLower95": 0.9301 - }, - "confirmationDecision": { - "f1": 0.7667, - "falseNegative": 65, - "falsePositive": 5, - "precision": 0.9583, - "predictedPositive": 120, - "recall": 0.6389, - "total": 2970, - "trueNegative": 2785, - "truePositive": 115, - "wilsonPrecisionLower95": 0.9062 - }, - "followUpReminder": { - "f1": 0.5375, - "falseNegative": 224, - "falsePositive": 10, - "precision": 0.9315, - "predictedPositive": 146, - "recall": 0.3778, - "total": 2970, - "trueNegative": 2600, - "truePositive": 136, - "wilsonPrecisionLower95": 0.8785 - }, - "invitation": { - "f1": 0.2692, - "falseNegative": 152, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 28, - "recall": 0.1556, - "total": 2970, - "trueNegative": 2790, - "truePositive": 28, - "wilsonPrecisionLower95": 0.8794 - }, - "question": { - "f1": 0.7156, - "falseNegative": 269, - "falsePositive": 18, - "precision": 0.9525, - "predictedPositive": 379, - "recall": 0.573, - "total": 2970, - "trueNegative": 2322, - "truePositive": 361, - "wilsonPrecisionLower95": 0.9262 - }, - "replyableMessage": { - "f1": 0.8205, - "falseNegative": 526, - "falsePositive": 71, - "precision": 0.9505, - "predictedPositive": 1435, - "recall": 0.7217, - "total": 2970, - "trueNegative": 1009, - "truePositive": 1364, - "wilsonPrecisionLower95": 0.9381 - }, - "scheduleNegotiation": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 90, - "recall": 1.0, - "total": 2970, - "trueNegative": 2880, - "truePositive": 90, - "wilsonPrecisionLower95": 0.9591 - }, - "task": { - "f1": 0.5565, - "falseNegative": 437, - "falsePositive": 14, - "precision": 0.9529, - "predictedPositive": 297, - "recall": 0.3931, - "total": 2970, - "trueNegative": 2236, - "truePositive": 283, - "wilsonPrecisionLower95": 0.9224 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 5e-06, - "augmentation": "punctuation", - "char_max": 7, - "char_min": 2, - "external_weight": 0.7100000000000001, - "hard_example_weight": 1.7, - "l1_ratio": 0.15, - "max_char_features": 70000, - "max_word_features": 28000, - "min_df": 2, - "pseudo_weight": 0.27, - "round": 8, - "word_max": 2 - }, - "featureCount": 83313, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 0 - }, - "complaint": { - "negative": 187, - "positive": 0 - }, - "confirmationDecision": { - "negative": 107, - "positive": 2 - }, - "followUpReminder": { - "negative": 250, - "positive": 0 - }, - "invitation": { - "negative": 250, - "positive": 0 - }, - "question": { - "negative": 40, - "positive": 5 - }, - "replyableMessage": { - "negative": 2, - "positive": 135 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 0 - }, - "task": { - "negative": 94, - "positive": 0 - } - }, - "round": 8, - "selectionObjective": 0.7837, - "syntheticRegression": { - "intentsMeeting95Precision": 4, - "macroF1": 0.6947, - "macroPrecision": 0.9526, - "macroRecall": 0.6066, - "minimumPrecision": 0.8509, - "perIntent": { - "blessing": { - "f1": 0.125, - "falseNegative": 84, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 6, - "recall": 0.0667, - "total": 2970, - "trueNegative": 2880, - "truePositive": 6, - "wilsonPrecisionLower95": 0.6097 - }, - "complaint": { - "f1": 0.9744, - "falseNegative": 18, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 342, - "recall": 0.95, - "total": 2970, - "trueNegative": 2610, - "truePositive": 342, - "wilsonPrecisionLower95": 0.9889 - }, - "confirmationDecision": { - "f1": 0.8035, - "falseNegative": 43, - "falsePositive": 24, - "precision": 0.8509, - "predictedPositive": 161, - "recall": 0.7611, - "total": 2970, - "trueNegative": 2766, - "truePositive": 137, - "wilsonPrecisionLower95": 0.7878 - }, - "followUpReminder": { - "f1": 0.5422, - "falseNegative": 222, - "falsePositive": 11, - "precision": 0.9262, - "predictedPositive": 149, - "recall": 0.3833, - "total": 2970, - "trueNegative": 2599, - "truePositive": 138, - "wilsonPrecisionLower95": 0.8726 - }, - "invitation": { - "f1": 0.5062, - "falseNegative": 119, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 61, - "recall": 0.3389, - "total": 2970, - "trueNegative": 2790, - "truePositive": 61, - "wilsonPrecisionLower95": 0.9408 - }, - "question": { - "f1": 0.7719, - "falseNegative": 229, - "falsePositive": 8, - "precision": 0.9804, - "predictedPositive": 409, - "recall": 0.6365, - "total": 2970, - "trueNegative": 2332, - "truePositive": 401, - "wilsonPrecisionLower95": 0.9619 - }, - "replyableMessage": { - "f1": 0.8832, - "falseNegative": 366, - "falsePositive": 37, - "precision": 0.9763, - "predictedPositive": 1561, - "recall": 0.8063, - "total": 2970, - "trueNegative": 1043, - "truePositive": 1524, - "wilsonPrecisionLower95": 0.9675 - }, - "scheduleNegotiation": { - "f1": 0.9162, - "falseNegative": 8, - "falsePositive": 7, - "precision": 0.9213, - "predictedPositive": 89, - "recall": 0.9111, - "total": 2970, - "trueNegative": 2873, - "truePositive": 82, - "wilsonPrecisionLower95": 0.8464 - }, - "task": { - "f1": 0.7297, - "falseNegative": 284, - "falsePositive": 39, - "precision": 0.9179, - "predictedPositive": 475, - "recall": 0.6056, - "total": 2970, - "trueNegative": 2211, - "truePositive": 436, - "wilsonPrecisionLower95": 0.8897 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.989, - "falseNegative": 1, - "falsePositive": 3, - "precision": 0.9835, - "predictedPositive": 182, - "recall": 0.9944, - "total": 1485, - "trueNegative": 1302, - "truePositive": 179, - "wilsonPrecisionLower95": 0.9527 - }, - "threshold": 0.97 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9405, - "falseNegative": 14, - "falsePositive": 7, - "precision": 0.9595, - "predictedPositive": 173, - "recall": 0.9222, - "total": 1485, - "trueNegative": 1298, - "truePositive": 166, - "wilsonPrecisionLower95": 0.9188 - }, - "threshold": 0.54 - } - }, - "metrics": { - "f1": 0.9237, - "falseNegative": 39, - "falsePositive": 14, - "precision": 0.9582, - "predictedPositive": 335, - "recall": 0.8917, - "total": 2970, - "trueNegative": 2596, - "truePositive": 321, - "wilsonPrecisionLower95": 0.9311 - }, - "threshold": 0.925 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9613, - "falseNegative": 3, - "falsePositive": 4, - "precision": 0.956, - "predictedPositive": 91, - "recall": 0.9667, - "total": 1485, - "trueNegative": 1391, - "truePositive": 87, - "wilsonPrecisionLower95": 0.8924 - }, - "threshold": 0.72 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.5556, - "falseNegative": 55, - "falsePositive": 1, - "precision": 0.9722, - "predictedPositive": 36, - "recall": 0.3889, - "total": 1485, - "trueNegative": 1394, - "truePositive": 35, - "wilsonPrecisionLower95": 0.8583 - }, - "threshold": 0.905 - } - }, - "metrics": { - "f1": 0.7922, - "falseNegative": 58, - "falsePositive": 6, - "precision": 0.9531, - "predictedPositive": 128, - "recall": 0.6778, - "total": 2970, - "trueNegative": 2784, - "truePositive": 122, - "wilsonPrecisionLower95": 0.9015 - }, - "threshold": 0.88 - }, - "followUpReminder": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8437, - "falseNegative": 45, - "falsePositive": 5, - "precision": 0.9643, - "predictedPositive": 140, - "recall": 0.75, - "total": 1485, - "trueNegative": 1300, - "truePositive": 135, - "wilsonPrecisionLower95": 0.9191 - }, - "threshold": 0.96 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.7063, - "falseNegative": 158, - "falsePositive": 10, - "precision": 0.9528, - "predictedPositive": 212, - "recall": 0.5611, - "total": 2970, - "trueNegative": 2600, - "truePositive": 202, - "wilsonPrecisionLower95": 0.9154 - }, - "threshold": 0.96 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.7157, - "falseNegative": 135, - "falsePositive": 8, - "precision": 0.9574, - "predictedPositive": 188, - "recall": 0.5714, - "total": 1485, - "trueNegative": 1162, - "truePositive": 180, - "wilsonPrecisionLower95": 0.9183 - }, - "threshold": 0.975 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.822, - "falseNegative": 91, - "falsePositive": 6, - "precision": 0.9739, - "predictedPositive": 230, - "recall": 0.7111, - "total": 1485, - "trueNegative": 1164, - "truePositive": 224, - "wilsonPrecisionLower95": 0.9443 - }, - "threshold": 0.505 - } - }, - "metrics": { - "f1": 0.8095, - "falseNegative": 186, - "falsePositive": 23, - "precision": 0.9507, - "predictedPositive": 467, - "recall": 0.7048, - "total": 2970, - "trueNegative": 2317, - "truePositive": 444, - "wilsonPrecisionLower95": 0.9272 - }, - "threshold": 0.71 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8741, - "falseNegative": 181, - "falsePositive": 39, - "precision": 0.9514, - "predictedPositive": 803, - "recall": 0.8085, - "total": 1485, - "trueNegative": 501, - "truePositive": 764, - "wilsonPrecisionLower95": 0.9343 - }, - "threshold": 0.95 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8775, - "falseNegative": 175, - "falsePositive": 40, - "precision": 0.9506, - "predictedPositive": 810, - "recall": 0.8148, - "total": 1485, - "trueNegative": 500, - "truePositive": 770, - "wilsonPrecisionLower95": 0.9335 - }, - "threshold": 0.88 - } - }, - "metrics": { - "f1": 0.8762, - "falseNegative": 354, - "falsePositive": 80, - "precision": 0.9505, - "predictedPositive": 1616, - "recall": 0.8127, - "total": 2970, - "trueNegative": 1000, - "truePositive": 1536, - "wilsonPrecisionLower95": 0.9388 - }, - "threshold": 0.905 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 45, - "recall": 1.0, - "total": 1485, - "trueNegative": 1440, - "truePositive": 45, - "wilsonPrecisionLower95": 0.9213 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9556, - "falseNegative": 2, - "falsePositive": 2, - "precision": 0.9556, - "predictedPositive": 45, - "recall": 0.9556, - "total": 1485, - "trueNegative": 1438, - "truePositive": 43, - "wilsonPrecisionLower95": 0.8517 - }, - "threshold": 0.75 - } - }, - "metrics": { - "f1": 0.9724, - "falseNegative": 2, - "falsePositive": 3, - "precision": 0.967, - "predictedPositive": 91, - "recall": 0.9778, - "total": 2970, - "trueNegative": 2877, - "truePositive": 88, - "wilsonPrecisionLower95": 0.9075 - }, - "threshold": 0.505 - }, - "task": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8331, - "falseNegative": 93, - "falsePositive": 14, - "precision": 0.9502, - "predictedPositive": 281, - "recall": 0.7417, - "total": 1485, - "trueNegative": 1111, - "truePositive": 267, - "wilsonPrecisionLower95": 0.9181 - }, - "threshold": 0.95 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7445, - "falseNegative": 140, - "falsePositive": 11, - "precision": 0.9524, - "predictedPositive": 231, - "recall": 0.6111, - "total": 1485, - "trueNegative": 1114, - "truePositive": 220, - "wilsonPrecisionLower95": 0.9168 - }, - "threshold": 0.86 - } - }, - "metrics": { - "f1": 0.7672, - "falseNegative": 257, - "falsePositive": 24, - "precision": 0.9507, - "predictedPositive": 487, - "recall": 0.6431, - "total": 2970, - "trueNegative": 2226, - "truePositive": 463, - "wilsonPrecisionLower95": 0.9277 - }, - "threshold": 0.92 - } - }, - "trainingRecords": 17762, - "validation": { - "intentsMeeting95Precision": 8, - "macroF1": 0.698, - "macroPrecision": 0.9692, - "macroRecall": 0.6056, - "minimumPrecision": 0.9444, - "perIntent": { - "blessing": { - "f1": 0.2524, - "falseNegative": 77, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 13, - "recall": 0.1444, - "total": 2970, - "trueNegative": 2880, - "truePositive": 13, - "wilsonPrecisionLower95": 0.7719 - }, - "complaint": { - "f1": 0.965, - "falseNegative": 15, - "falsePositive": 10, - "precision": 0.9718, - "predictedPositive": 355, - "recall": 0.9583, - "total": 2970, - "trueNegative": 2600, - "truePositive": 345, - "wilsonPrecisionLower95": 0.9489 - }, - "confirmationDecision": { - "f1": 0.7948, - "falseNegative": 58, - "falsePositive": 5, - "precision": 0.9606, - "predictedPositive": 127, - "recall": 0.6778, - "total": 2970, - "trueNegative": 2785, - "truePositive": 122, - "wilsonPrecisionLower95": 0.9111 - }, - "followUpReminder": { - "f1": 0.5397, - "falseNegative": 224, - "falsePositive": 8, - "precision": 0.9444, - "predictedPositive": 144, - "recall": 0.3778, - "total": 2970, - "trueNegative": 2602, - "truePositive": 136, - "wilsonPrecisionLower95": 0.8942 - }, - "invitation": { - "f1": 0.3178, - "falseNegative": 146, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 34, - "recall": 0.1889, - "total": 2970, - "trueNegative": 2790, - "truePositive": 34, - "wilsonPrecisionLower95": 0.8985 - }, - "question": { - "f1": 0.771, - "falseNegative": 226, - "falsePositive": 14, - "precision": 0.9665, - "predictedPositive": 418, - "recall": 0.6413, - "total": 2970, - "trueNegative": 2326, - "truePositive": 404, - "wilsonPrecisionLower95": 0.9446 - }, - "replyableMessage": { - "f1": 0.8758, - "falseNegative": 356, - "falsePositive": 79, - "precision": 0.951, - "predictedPositive": 1613, - "recall": 0.8116, - "total": 2970, - "trueNegative": 1001, - "truePositive": 1534, - "wilsonPrecisionLower95": 0.9394 - }, - "scheduleNegotiation": { - "f1": 0.9778, - "falseNegative": 2, - "falsePositive": 2, - "precision": 0.9778, - "predictedPositive": 90, - "recall": 0.9778, - "total": 2970, - "trueNegative": 2878, - "truePositive": 88, - "wilsonPrecisionLower95": 0.9226 - }, - "task": { - "f1": 0.7876, - "falseNegative": 236, - "falsePositive": 25, - "precision": 0.9509, - "predictedPositive": 509, - "recall": 0.6722, - "total": 2970, - "trueNegative": 2225, - "truePositive": 484, - "wilsonPrecisionLower95": 0.9285 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 4e-06, - "augmentation": "punctuation", - "char_max": 7, - "char_min": 3, - "external_weight": 0.73, - "hard_example_weight": 1.7, - "l1_ratio": 0.15, - "max_char_features": 70000, - "max_word_features": 35000, - "min_df": 2, - "pseudo_weight": 0.28500000000000003, - "round": 9, - "word_max": 3 - }, - "featureCount": 92641, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 0 - }, - "complaint": { - "negative": 250, - "positive": 0 - }, - "confirmationDecision": { - "negative": 248, - "positive": 6 - }, - "followUpReminder": { - "negative": 250, - "positive": 1 - }, - "invitation": { - "negative": 250, - "positive": 0 - }, - "question": { - "negative": 146, - "positive": 8 - }, - "replyableMessage": { - "negative": 8, - "positive": 250 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 0 - }, - "task": { - "negative": 208, - "positive": 0 - } - }, - "round": 9, - "selectionObjective": 0.7717, - "syntheticRegression": { - "intentsMeeting95Precision": 4, - "macroF1": 0.6445, - "macroPrecision": 0.9357, - "macroRecall": 0.5386, - "minimumPrecision": 0.8583, - "perIntent": { - "blessing": { - "f1": 0.125, - "falseNegative": 84, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 6, - "recall": 0.0667, - "total": 2970, - "trueNegative": 2880, - "truePositive": 6, - "wilsonPrecisionLower95": 0.6097 - }, - "complaint": { - "f1": 0.949, - "falseNegative": 25, - "falsePositive": 11, - "precision": 0.9682, - "predictedPositive": 346, - "recall": 0.9306, - "total": 2970, - "trueNegative": 2599, - "truePositive": 335, - "wilsonPrecisionLower95": 0.944 - }, - "confirmationDecision": { - "f1": 0.7101, - "falseNegative": 71, - "falsePositive": 18, - "precision": 0.8583, - "predictedPositive": 127, - "recall": 0.6056, - "total": 2970, - "trueNegative": 2772, - "truePositive": 109, - "wilsonPrecisionLower95": 0.7871 - }, - "followUpReminder": { - "f1": 0.5244, - "falseNegative": 231, - "falsePositive": 3, - "precision": 0.9773, - "predictedPositive": 132, - "recall": 0.3583, - "total": 2970, - "trueNegative": 2607, - "truePositive": 129, - "wilsonPrecisionLower95": 0.9353 - }, - "invitation": { - "f1": 0.6286, - "falseNegative": 92, - "falsePositive": 12, - "precision": 0.88, - "predictedPositive": 100, - "recall": 0.4889, - "total": 2970, - "trueNegative": 2778, - "truePositive": 88, - "wilsonPrecisionLower95": 0.8019 - }, - "question": { - "f1": 0.6667, - "falseNegative": 310, - "falsePositive": 10, - "precision": 0.9697, - "predictedPositive": 330, - "recall": 0.5079, - "total": 2970, - "trueNegative": 2330, - "truePositive": 320, - "wilsonPrecisionLower95": 0.9451 - }, - "replyableMessage": { - "f1": 0.7919, - "falseNegative": 594, - "falsePositive": 87, - "precision": 0.9371, - "predictedPositive": 1383, - "recall": 0.6857, - "total": 2970, - "trueNegative": 993, - "truePositive": 1296, - "wilsonPrecisionLower95": 0.923 - }, - "scheduleNegotiation": { - "f1": 0.8941, - "falseNegative": 14, - "falsePositive": 4, - "precision": 0.95, - "predictedPositive": 80, - "recall": 0.8444, - "total": 2970, - "trueNegative": 2876, - "truePositive": 76, - "wilsonPrecisionLower95": 0.8784 - }, - "task": { - "f1": 0.5108, - "falseNegative": 461, - "falsePositive": 35, - "precision": 0.881, - "predictedPositive": 294, - "recall": 0.3597, - "total": 2970, - "trueNegative": 2215, - "truePositive": 259, - "wilsonPrecisionLower95": 0.8389 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9862, - "falseNegative": 1, - "falsePositive": 4, - "precision": 0.9781, - "predictedPositive": 183, - "recall": 0.9944, - "total": 1485, - "trueNegative": 1301, - "truePositive": 179, - "wilsonPrecisionLower95": 0.9452 - }, - "threshold": 0.74 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9222, - "falseNegative": 20, - "falsePositive": 7, - "precision": 0.9581, - "predictedPositive": 167, - "recall": 0.8889, - "total": 1485, - "trueNegative": 1298, - "truePositive": 160, - "wilsonPrecisionLower95": 0.916 - }, - "threshold": 0.62 - } - }, - "metrics": { - "f1": 0.9528, - "falseNegative": 17, - "falsePositive": 17, - "precision": 0.9528, - "predictedPositive": 360, - "recall": 0.9528, - "total": 2970, - "trueNegative": 2593, - "truePositive": 343, - "wilsonPrecisionLower95": 0.9257 - }, - "threshold": 0.5 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9492, - "falseNegative": 6, - "falsePositive": 3, - "precision": 0.9655, - "predictedPositive": 87, - "recall": 0.9333, - "total": 1485, - "trueNegative": 1392, - "truePositive": 84, - "wilsonPrecisionLower95": 0.9035 - }, - "threshold": 0.965 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7042, - "falseNegative": 40, - "falsePositive": 2, - "precision": 0.9615, - "predictedPositive": 52, - "recall": 0.5556, - "total": 1485, - "trueNegative": 1393, - "truePositive": 50, - "wilsonPrecisionLower95": 0.8702 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.76, - "falseNegative": 66, - "falsePositive": 6, - "precision": 0.95, - "predictedPositive": 120, - "recall": 0.6333, - "total": 2970, - "trueNegative": 2784, - "truePositive": 114, - "wilsonPrecisionLower95": 0.8952 - }, - "threshold": 0.945 - }, - "followUpReminder": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8243, - "falseNegative": 51, - "falsePositive": 4, - "precision": 0.9699, - "predictedPositive": 133, - "recall": 0.7167, - "total": 1485, - "trueNegative": 1301, - "truePositive": 129, - "wilsonPrecisionLower95": 0.9252 - }, - "threshold": 0.99 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.6316, - "falseNegative": 192, - "falsePositive": 4, - "precision": 0.9767, - "predictedPositive": 172, - "recall": 0.4667, - "total": 2970, - "trueNegative": 2606, - "truePositive": 168, - "wilsonPrecisionLower95": 0.9417 - }, - "threshold": 0.995 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.544, - "falseNegative": 56, - "falsePositive": 1, - "precision": 0.9714, - "predictedPositive": 35, - "recall": 0.3778, - "total": 1485, - "trueNegative": 1394, - "truePositive": 34, - "wilsonPrecisionLower95": 0.8547 - }, - "threshold": 0.82 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.6694, - "falseNegative": 153, - "falsePositive": 7, - "precision": 0.9586, - "predictedPositive": 169, - "recall": 0.5143, - "total": 1485, - "trueNegative": 1163, - "truePositive": 162, - "wilsonPrecisionLower95": 0.917 - }, - "threshold": 0.985 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6762, - "falseNegative": 150, - "falsePositive": 8, - "precision": 0.9538, - "predictedPositive": 173, - "recall": 0.5238, - "total": 1485, - "trueNegative": 1162, - "truePositive": 165, - "wilsonPrecisionLower95": 0.9114 - }, - "threshold": 0.64 - } - }, - "metrics": { - "f1": 0.5392, - "falseNegative": 393, - "falsePositive": 12, - "precision": 0.9518, - "predictedPositive": 249, - "recall": 0.3762, - "total": 2970, - "trueNegative": 2328, - "truePositive": 237, - "wilsonPrecisionLower95": 0.9177 - }, - "threshold": 0.965 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9407, - "falseNegative": 65, - "falsePositive": 46, - "precision": 0.9503, - "predictedPositive": 926, - "recall": 0.9312, - "total": 1485, - "trueNegative": 494, - "truePositive": 880, - "wilsonPrecisionLower95": 0.9344 - }, - "threshold": 0.745 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6118, - "falseNegative": 521, - "falsePositive": 17, - "precision": 0.9615, - "predictedPositive": 441, - "recall": 0.4487, - "total": 1485, - "trueNegative": 523, - "truePositive": 424, - "wilsonPrecisionLower95": 0.9391 - }, - "threshold": 0.965 - } - }, - "metrics": { - "f1": 0.8037, - "falseNegative": 576, - "falsePositive": 66, - "precision": 0.9522, - "predictedPositive": 1380, - "recall": 0.6952, - "total": 2970, - "trueNegative": 1014, - "truePositive": 1314, - "wilsonPrecisionLower95": 0.9396 - }, - "threshold": 0.95 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9286, - "falseNegative": 6, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 39, - "recall": 0.8667, - "total": 1485, - "trueNegative": 1440, - "truePositive": 39, - "wilsonPrecisionLower95": 0.9103 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9302, - "falseNegative": 5, - "falsePositive": 1, - "precision": 0.9756, - "predictedPositive": 41, - "recall": 0.8889, - "total": 1485, - "trueNegative": 1439, - "truePositive": 40, - "wilsonPrecisionLower95": 0.874 - }, - "threshold": 0.61 - } - }, - "metrics": { - "f1": 0.9294, - "falseNegative": 11, - "falsePositive": 1, - "precision": 0.9875, - "predictedPositive": 80, - "recall": 0.8778, - "total": 2970, - "trueNegative": 2879, - "truePositive": 79, - "wilsonPrecisionLower95": 0.9325 - }, - "threshold": 0.61 - }, - "task": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8403, - "falseNegative": 89, - "falsePositive": 14, - "precision": 0.9509, - "predictedPositive": 285, - "recall": 0.7528, - "total": 1485, - "trueNegative": 1111, - "truePositive": 271, - "wilsonPrecisionLower95": 0.9192 - }, - "threshold": 0.93 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.2384, - "falseNegative": 311, - "falsePositive": 2, - "precision": 0.9608, - "predictedPositive": 51, - "recall": 0.1361, - "total": 1485, - "trueNegative": 1123, - "truePositive": 49, - "wilsonPrecisionLower95": 0.8678 - }, - "threshold": 0.995 - } - }, - "metrics": { - "f1": 0.6415, - "falseNegative": 372, - "falsePositive": 17, - "precision": 0.9534, - "predictedPositive": 365, - "recall": 0.4833, - "total": 2970, - "trueNegative": 2233, - "truePositive": 348, - "wilsonPrecisionLower95": 0.9267 - }, - "threshold": 0.975 - } - }, - "trainingRecords": 17823, - "validation": { - "intentsMeeting95Precision": 8, - "macroF1": 0.6858, - "macroPrecision": 0.9638, - "macroRecall": 0.5714, - "minimumPrecision": 0.9211, - "perIntent": { - "blessing": { - "f1": 0.2857, - "falseNegative": 75, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 15, - "recall": 0.1667, - "total": 2970, - "trueNegative": 2880, - "truePositive": 15, - "wilsonPrecisionLower95": 0.7961 - }, - "complaint": { - "f1": 0.9549, - "falseNegative": 21, - "falsePositive": 11, - "precision": 0.9686, - "predictedPositive": 350, - "recall": 0.9417, - "total": 2970, - "trueNegative": 2599, - "truePositive": 339, - "wilsonPrecisionLower95": 0.9446 - }, - "confirmationDecision": { - "f1": 0.8401, - "falseNegative": 46, - "falsePositive": 5, - "precision": 0.964, - "predictedPositive": 139, - "recall": 0.7444, - "total": 2970, - "trueNegative": 2785, - "truePositive": 134, - "wilsonPrecisionLower95": 0.9186 - }, - "followUpReminder": { - "f1": 0.5352, - "falseNegative": 227, - "falsePositive": 4, - "precision": 0.9708, - "predictedPositive": 137, - "recall": 0.3694, - "total": 2970, - "trueNegative": 2606, - "truePositive": 133, - "wilsonPrecisionLower95": 0.9273 - }, - "invitation": { - "f1": 0.5469, - "falseNegative": 110, - "falsePositive": 6, - "precision": 0.9211, - "predictedPositive": 76, - "recall": 0.3889, - "total": 2970, - "trueNegative": 2784, - "truePositive": 70, - "wilsonPrecisionLower95": 0.8383 - }, - "question": { - "f1": 0.6728, - "falseNegative": 303, - "falsePositive": 15, - "precision": 0.9561, - "predictedPositive": 342, - "recall": 0.519, - "total": 2970, - "trueNegative": 2325, - "truePositive": 327, - "wilsonPrecisionLower95": 0.9289 - }, - "replyableMessage": { - "f1": 0.8007, - "falseNegative": 586, - "falsePositive": 63, - "precision": 0.9539, - "predictedPositive": 1367, - "recall": 0.6899, - "total": 2970, - "trueNegative": 1017, - "truePositive": 1304, - "wilsonPrecisionLower95": 0.9415 - }, - "scheduleNegotiation": { - "f1": 0.9294, - "falseNegative": 11, - "falsePositive": 1, - "precision": 0.9875, - "predictedPositive": 80, - "recall": 0.8778, - "total": 2970, - "trueNegative": 2879, - "truePositive": 79, - "wilsonPrecisionLower95": 0.9325 - }, - "task": { - "f1": 0.6061, - "falseNegative": 400, - "falsePositive": 16, - "precision": 0.9524, - "predictedPositive": 336, - "recall": 0.4444, - "total": 2970, - "trueNegative": 2234, - "truePositive": 320, - "wilsonPrecisionLower95": 0.9241 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 6e-06, - "augmentation": "punctuation", - "char_max": 6, - "char_min": 2, - "external_weight": 0.75, - "hard_example_weight": 1.8, - "l1_ratio": 0.2, - "max_char_features": 75000, - "max_word_features": 38000, - "min_df": 1, - "pseudo_weight": 0.3, - "round": 10, - "word_max": 3 - }, - "featureCount": 113000, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 0 - }, - "complaint": { - "negative": 250, - "positive": 0 - }, - "confirmationDecision": { - "negative": 250, - "positive": 11 - }, - "followUpReminder": { - "negative": 250, - "positive": 4 - }, - "invitation": { - "negative": 250, - "positive": 0 - }, - "question": { - "negative": 182, - "positive": 10 - }, - "replyableMessage": { - "negative": 11, - "positive": 250 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 0 - }, - "task": { - "negative": 250, - "positive": 1 - } - }, - "round": 10, - "selectionObjective": 0.7542, - "syntheticRegression": { - "intentsMeeting95Precision": 4, - "macroF1": 0.6448, - "macroPrecision": 0.9609, - "macroRecall": 0.5363, - "minimumPrecision": 0.8734, - "perIntent": { - "blessing": { - "f1": 0.125, - "falseNegative": 84, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 6, - "recall": 0.0667, - "total": 2970, - "trueNegative": 2880, - "truePositive": 6, - "wilsonPrecisionLower95": 0.6097 - }, - "complaint": { - "f1": 0.9366, - "falseNegative": 20, - "falsePositive": 26, - "precision": 0.929, - "predictedPositive": 366, - "recall": 0.9444, - "total": 2970, - "trueNegative": 2584, - "truePositive": 340, - "wilsonPrecisionLower95": 0.8979 - }, - "confirmationDecision": { - "f1": 0.8166, - "falseNegative": 42, - "falsePositive": 20, - "precision": 0.8734, - "predictedPositive": 158, - "recall": 0.7667, - "total": 2970, - "trueNegative": 2770, - "truePositive": 138, - "wilsonPrecisionLower95": 0.8126 - }, - "followUpReminder": { - "f1": 0.5397, - "falseNegative": 224, - "falsePositive": 8, - "precision": 0.9444, - "predictedPositive": 144, - "recall": 0.3778, - "total": 2970, - "trueNegative": 2602, - "truePositive": 136, - "wilsonPrecisionLower95": 0.8942 - }, - "invitation": { - "f1": 0.5124, - "falseNegative": 118, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 62, - "recall": 0.3444, - "total": 2970, - "trueNegative": 2790, - "truePositive": 62, - "wilsonPrecisionLower95": 0.9417 - }, - "question": { - "f1": 0.5488, - "falseNegative": 391, - "falsePositive": 2, - "precision": 0.9917, - "predictedPositive": 241, - "recall": 0.3794, - "total": 2970, - "trueNegative": 2338, - "truePositive": 239, - "wilsonPrecisionLower95": 0.9703 - }, - "replyableMessage": { - "f1": 0.7216, - "falseNegative": 808, - "falsePositive": 27, - "precision": 0.9757, - "predictedPositive": 1109, - "recall": 0.5725, - "total": 2970, - "trueNegative": 1053, - "truePositive": 1082, - "wilsonPrecisionLower95": 0.9648 - }, - "scheduleNegotiation": { - "f1": 0.882, - "falseNegative": 19, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 71, - "recall": 0.7889, - "total": 2970, - "trueNegative": 2880, - "truePositive": 71, - "wilsonPrecisionLower95": 0.9487 - }, - "task": { - "f1": 0.7201, - "falseNegative": 298, - "falsePositive": 30, - "precision": 0.9336, - "predictedPositive": 452, - "recall": 0.5861, - "total": 2970, - "trueNegative": 2220, - "truePositive": 422, - "wilsonPrecisionLower95": 0.9068 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9807, - "falseNegative": 2, - "falsePositive": 5, - "precision": 0.9727, - "predictedPositive": 183, - "recall": 0.9889, - "total": 1485, - "trueNegative": 1300, - "truePositive": 178, - "wilsonPrecisionLower95": 0.9376 - }, - "threshold": 0.765 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9288, - "falseNegative": 17, - "falsePositive": 8, - "precision": 0.9532, - "predictedPositive": 171, - "recall": 0.9056, - "total": 1485, - "trueNegative": 1297, - "truePositive": 163, - "wilsonPrecisionLower95": 0.9104 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.9455, - "falseNegative": 22, - "falsePositive": 17, - "precision": 0.9521, - "predictedPositive": 355, - "recall": 0.9389, - "total": 2970, - "trueNegative": 2593, - "truePositive": 338, - "wilsonPrecisionLower95": 0.9247 - }, - "threshold": 0.595 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.7568, - "falseNegative": 34, - "falsePositive": 2, - "precision": 0.9655, - "predictedPositive": 58, - "recall": 0.6222, - "total": 1485, - "trueNegative": 1393, - "truePositive": 56, - "wilsonPrecisionLower95": 0.8827 - }, - "threshold": 0.955 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7733, - "falseNegative": 32, - "falsePositive": 2, - "precision": 0.9667, - "predictedPositive": 60, - "recall": 0.6444, - "total": 1485, - "trueNegative": 1393, - "truePositive": 58, - "wilsonPrecisionLower95": 0.8864 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.7723, - "falseNegative": 63, - "falsePositive": 6, - "precision": 0.9512, - "predictedPositive": 123, - "recall": 0.65, - "total": 2970, - "trueNegative": 2784, - "truePositive": 117, - "wilsonPrecisionLower95": 0.8977 - }, - "threshold": 0.765 - }, - "followUpReminder": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8483, - "falseNegative": 43, - "falsePositive": 6, - "precision": 0.958, - "predictedPositive": 143, - "recall": 0.7611, - "total": 1485, - "trueNegative": 1299, - "truePositive": 137, - "wilsonPrecisionLower95": 0.9115 - }, - "threshold": 0.995 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.3529, - "falseNegative": 141, - "falsePositive": 2, - "precision": 0.9512, - "predictedPositive": 41, - "recall": 0.2167, - "total": 1485, - "trueNegative": 1303, - "truePositive": 39, - "wilsonPrecisionLower95": 0.8386 - }, - "threshold": 0.99 - } - }, - "metrics": { - "f1": 0.6292, - "falseNegative": 192, - "falsePositive": 6, - "precision": 0.9655, - "predictedPositive": 174, - "recall": 0.4667, - "total": 2970, - "trueNegative": 2604, - "truePositive": 168, - "wilsonPrecisionLower95": 0.9268 - }, - "threshold": 0.995 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.3894, - "falseNegative": 68, - "falsePositive": 1, - "precision": 0.9565, - "predictedPositive": 23, - "recall": 0.2444, - "total": 1485, - "trueNegative": 1394, - "truePositive": 22, - "wilsonPrecisionLower95": 0.7901 - }, - "threshold": 0.98 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8175, - "falseNegative": 91, - "falsePositive": 9, - "precision": 0.9614, - "predictedPositive": 233, - "recall": 0.7111, - "total": 1485, - "trueNegative": 1161, - "truePositive": 224, - "wilsonPrecisionLower95": 0.9282 - }, - "threshold": 0.6 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.7182, - "falseNegative": 401, - "falsePositive": 26, - "precision": 0.9544, - "predictedPositive": 570, - "recall": 0.5757, - "total": 1485, - "trueNegative": 514, - "truePositive": 544, - "wilsonPrecisionLower95": 0.934 - }, - "threshold": 0.985 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6667, - "falseNegative": 460, - "falsePositive": 25, - "precision": 0.951, - "predictedPositive": 510, - "recall": 0.5132, - "total": 1485, - "trueNegative": 515, - "truePositive": 485, - "wilsonPrecisionLower95": 0.9286 - }, - "threshold": 0.945 - } - }, - "metrics": { - "f1": 0.7151, - "falseNegative": 807, - "falsePositive": 56, - "precision": 0.9508, - "predictedPositive": 1139, - "recall": 0.573, - "total": 2970, - "trueNegative": 1024, - "truePositive": 1083, - "wilsonPrecisionLower95": 0.9367 - }, - "threshold": 0.96 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9412, - "falseNegative": 5, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 40, - "recall": 0.8889, - "total": 1485, - "trueNegative": 1440, - "truePositive": 40, - "wilsonPrecisionLower95": 0.9124 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8052, - "falseNegative": 14, - "falsePositive": 1, - "precision": 0.9688, - "predictedPositive": 32, - "recall": 0.6889, - "total": 1485, - "trueNegative": 1439, - "truePositive": 31, - "wilsonPrecisionLower95": 0.8426 - }, - "threshold": 0.98 - } - }, - "metrics": { - "f1": 0.8375, - "falseNegative": 23, - "falsePositive": 3, - "precision": 0.9571, - "predictedPositive": 70, - "recall": 0.7444, - "total": 2970, - "trueNegative": 2877, - "truePositive": 67, - "wilsonPrecisionLower95": 0.8814 - }, - "threshold": 0.92 - }, - "task": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.7783, - "falseNegative": 123, - "falsePositive": 12, - "precision": 0.9518, - "predictedPositive": 249, - "recall": 0.6583, - "total": 1485, - "trueNegative": 1113, - "truePositive": 237, - "wilsonPrecisionLower95": 0.9177 - }, - "threshold": 0.84 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7994, - "falseNegative": 113, - "falsePositive": 11, - "precision": 0.9574, - "predictedPositive": 258, - "recall": 0.6861, - "total": 1485, - "trueNegative": 1114, - "truePositive": 247, - "wilsonPrecisionLower95": 0.9253 - }, - "threshold": 0.755 - } - }, - "metrics": { - "f1": 0.7977, - "falseNegative": 225, - "falsePositive": 26, - "precision": 0.9501, - "predictedPositive": 521, - "recall": 0.6875, - "total": 2970, - "trueNegative": 2224, - "truePositive": 495, - "wilsonPrecisionLower95": 0.9279 - }, - "threshold": 0.77 - } - }, - "trainingRecords": 17763, - "validation": { - "intentsMeeting95Precision": 8, - "macroF1": 0.6563, - "macroPrecision": 0.9695, - "macroRecall": 0.5386, - "minimumPrecision": 0.9528, - "perIntent": { - "blessing": { - "f1": 0.1818, - "falseNegative": 81, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 9, - "recall": 0.1, - "total": 2970, - "trueNegative": 2880, - "truePositive": 9, - "wilsonPrecisionLower95": 0.7009 - }, - "complaint": { - "f1": 0.9552, - "falseNegative": 19, - "falsePositive": 13, - "precision": 0.9633, - "predictedPositive": 354, - "recall": 0.9472, - "total": 2970, - "trueNegative": 2597, - "truePositive": 341, - "wilsonPrecisionLower95": 0.9382 - }, - "confirmationDecision": { - "f1": 0.7651, - "falseNegative": 66, - "falsePositive": 4, - "precision": 0.9661, - "predictedPositive": 118, - "recall": 0.6333, - "total": 2970, - "trueNegative": 2786, - "truePositive": 114, - "wilsonPrecisionLower95": 0.9161 - }, - "followUpReminder": { - "f1": 0.6471, - "falseNegative": 184, - "falsePositive": 8, - "precision": 0.9565, - "predictedPositive": 184, - "recall": 0.4889, - "total": 2970, - "trueNegative": 2602, - "truePositive": 176, - "wilsonPrecisionLower95": 0.9166 - }, - "invitation": { - "f1": 0.479, - "falseNegative": 123, - "falsePositive": 1, - "precision": 0.9828, - "predictedPositive": 58, - "recall": 0.3167, - "total": 2970, - "trueNegative": 2789, - "truePositive": 57, - "wilsonPrecisionLower95": 0.9086 - }, - "question": { - "f1": 0.5191, - "falseNegative": 406, - "falsePositive": 9, - "precision": 0.9614, - "predictedPositive": 233, - "recall": 0.3556, - "total": 2970, - "trueNegative": 2331, - "truePositive": 224, - "wilsonPrecisionLower95": 0.9282 - }, - "replyableMessage": { - "f1": 0.6929, - "falseNegative": 861, - "falsePositive": 51, - "precision": 0.9528, - "predictedPositive": 1080, - "recall": 0.5444, - "total": 2970, - "trueNegative": 1029, - "truePositive": 1029, - "wilsonPrecisionLower95": 0.9384 - }, - "scheduleNegotiation": { - "f1": 0.8765, - "falseNegative": 19, - "falsePositive": 1, - "precision": 0.9861, - "predictedPositive": 72, - "recall": 0.7889, - "total": 2970, - "trueNegative": 2879, - "truePositive": 71, - "wilsonPrecisionLower95": 0.9254 - }, - "task": { - "f1": 0.7896, - "falseNegative": 236, - "falsePositive": 22, - "precision": 0.9565, - "predictedPositive": 506, - "recall": 0.6722, - "total": 2970, - "trueNegative": 2228, - "truePositive": 484, - "wilsonPrecisionLower95": 0.935 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 5e-06, - "augmentation": "prefix", - "char_max": 6, - "char_min": 2, - "external_weight": 0.77, - "hard_example_weight": 1.8, - "l1_ratio": 0.1, - "max_char_features": 70000, - "max_word_features": 30000, - "min_df": 2, - "pseudo_weight": 0.31499999999999995, - "round": 11, - "word_max": 2 - }, - "featureCount": 81451, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 0 - }, - "complaint": { - "negative": 250, - "positive": 0 - }, - "confirmationDecision": { - "negative": 250, - "positive": 14 - }, - "followUpReminder": { - "negative": 250, - "positive": 2 - }, - "invitation": { - "negative": 250, - "positive": 1 - }, - "question": { - "negative": 233, - "positive": 0 - }, - "replyableMessage": { - "negative": 8, - "positive": 250 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 0 - }, - "task": { - "negative": 250, - "positive": 0 - } - }, - "round": 11, - "selectionObjective": 0.8015, - "syntheticRegression": { - "intentsMeeting95Precision": 4, - "macroF1": 0.6807, - "macroPrecision": 0.963, - "macroRecall": 0.6053, - "minimumPrecision": 0.8782, - "perIntent": { - "blessing": { - "f1": 0.125, - "falseNegative": 84, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 6, - "recall": 0.0667, - "total": 2970, - "trueNegative": 2880, - "truePositive": 6, - "wilsonPrecisionLower95": 0.6097 - }, - "complaint": { - "f1": 0.9655, - "falseNegative": 24, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 336, - "recall": 0.9333, - "total": 2970, - "trueNegative": 2610, - "truePositive": 336, - "wilsonPrecisionLower95": 0.9887 - }, - "confirmationDecision": { - "f1": 0.8155, - "falseNegative": 43, - "falsePositive": 19, - "precision": 0.8782, - "predictedPositive": 156, - "recall": 0.7611, - "total": 2970, - "trueNegative": 2771, - "truePositive": 137, - "wilsonPrecisionLower95": 0.8176 - }, - "followUpReminder": { - "f1": 0.2439, - "falseNegative": 310, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 50, - "recall": 0.1389, - "total": 2970, - "trueNegative": 2610, - "truePositive": 50, - "wilsonPrecisionLower95": 0.9287 - }, - "invitation": { - "f1": 0.5425, - "falseNegative": 113, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 67, - "recall": 0.3722, - "total": 2970, - "trueNegative": 2790, - "truePositive": 67, - "wilsonPrecisionLower95": 0.9458 - }, - "question": { - "f1": 0.8494, - "falseNegative": 162, - "falsePositive": 4, - "precision": 0.9915, - "predictedPositive": 472, - "recall": 0.7429, - "total": 2970, - "trueNegative": 2336, - "truePositive": 468, - "wilsonPrecisionLower95": 0.9784 - }, - "replyableMessage": { - "f1": 0.8975, - "falseNegative": 253, - "falsePositive": 121, - "precision": 0.9312, - "predictedPositive": 1758, - "recall": 0.8661, - "total": 2970, - "trueNegative": 959, - "truePositive": 1637, - "wilsonPrecisionLower95": 0.9184 - }, - "scheduleNegotiation": { - "f1": 0.9444, - "falseNegative": 5, - "falsePositive": 5, - "precision": 0.9444, - "predictedPositive": 90, - "recall": 0.9444, - "total": 2970, - "trueNegative": 2875, - "truePositive": 85, - "wilsonPrecisionLower95": 0.8765 - }, - "task": { - "f1": 0.743, - "falseNegative": 272, - "falsePositive": 38, - "precision": 0.9218, - "predictedPositive": 486, - "recall": 0.6222, - "total": 2970, - "trueNegative": 2212, - "truePositive": 448, - "wilsonPrecisionLower95": 0.8945 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9917, - "falseNegative": 0, - "falsePositive": 3, - "precision": 0.9836, - "predictedPositive": 183, - "recall": 1.0, - "total": 1485, - "trueNegative": 1302, - "truePositive": 180, - "wilsonPrecisionLower95": 0.9529 - }, - "threshold": 0.99 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9318, - "falseNegative": 16, - "falsePositive": 8, - "precision": 0.9535, - "predictedPositive": 172, - "recall": 0.9111, - "total": 1485, - "trueNegative": 1297, - "truePositive": 164, - "wilsonPrecisionLower95": 0.9109 - }, - "threshold": 0.84 - } - }, - "metrics": { - "f1": 0.9529, - "falseNegative": 16, - "falsePositive": 18, - "precision": 0.9503, - "predictedPositive": 362, - "recall": 0.9556, - "total": 2970, - "trueNegative": 2592, - "truePositive": 344, - "wilsonPrecisionLower95": 0.9228 - }, - "threshold": 0.84 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.931, - "falseNegative": 9, - "falsePositive": 3, - "precision": 0.9643, - "predictedPositive": 84, - "recall": 0.9, - "total": 1485, - "trueNegative": 1392, - "truePositive": 81, - "wilsonPrecisionLower95": 0.9002 - }, - "threshold": 0.97 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7222, - "falseNegative": 38, - "falsePositive": 2, - "precision": 0.963, - "predictedPositive": 54, - "recall": 0.5778, - "total": 1485, - "trueNegative": 1393, - "truePositive": 52, - "wilsonPrecisionLower95": 0.8746 - }, - "threshold": 0.86 - } - }, - "metrics": { - "f1": 0.8077, - "falseNegative": 54, - "falsePositive": 6, - "precision": 0.9545, - "predictedPositive": 132, - "recall": 0.7, - "total": 2970, - "trueNegative": 2784, - "truePositive": 126, - "wilsonPrecisionLower95": 0.9044 - }, - "threshold": 0.95 - }, - "followUpReminder": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.4833, - "falseNegative": 61, - "falsePositive": 1, - "precision": 0.9667, - "predictedPositive": 30, - "recall": 0.3222, - "total": 1485, - "trueNegative": 1394, - "truePositive": 29, - "wilsonPrecisionLower95": 0.8333 - }, - "threshold": 0.965 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.7993, - "falseNegative": 98, - "falsePositive": 11, - "precision": 0.9518, - "predictedPositive": 228, - "recall": 0.6889, - "total": 1485, - "trueNegative": 1159, - "truePositive": 217, - "wilsonPrecisionLower95": 0.9157 - }, - "threshold": 0.92 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7572, - "falseNegative": 117, - "falsePositive": 10, - "precision": 0.9519, - "predictedPositive": 208, - "recall": 0.6286, - "total": 1485, - "trueNegative": 1160, - "truePositive": 198, - "wilsonPrecisionLower95": 0.9138 - }, - "threshold": 0.735 - } - }, - "metrics": { - "f1": 0.797, - "falseNegative": 198, - "falsePositive": 22, - "precision": 0.9515, - "predictedPositive": 454, - "recall": 0.6857, - "total": 2970, - "trueNegative": 2318, - "truePositive": 432, - "wilsonPrecisionLower95": 0.9277 - }, - "threshold": 0.8 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9373, - "falseNegative": 71, - "falsePositive": 46, - "precision": 0.95, - "predictedPositive": 920, - "recall": 0.9249, - "total": 1485, - "trueNegative": 494, - "truePositive": 874, - "wilsonPrecisionLower95": 0.934 - }, - "threshold": 0.79 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8754, - "falseNegative": 179, - "falsePositive": 39, - "precision": 0.9516, - "predictedPositive": 805, - "recall": 0.8106, - "total": 1485, - "trueNegative": 501, - "truePositive": 766, - "wilsonPrecisionLower95": 0.9345 - }, - "threshold": 0.91 - } - }, - "metrics": { - "f1": 0.9105, - "falseNegative": 242, - "falsePositive": 82, - "precision": 0.9526, - "predictedPositive": 1730, - "recall": 0.872, - "total": 2970, - "trueNegative": 998, - "truePositive": 1648, - "wilsonPrecisionLower95": 0.9415 - }, - "threshold": 0.895 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9773, - "falseNegative": 2, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 43, - "recall": 0.9556, - "total": 1485, - "trueNegative": 1440, - "truePositive": 43, - "wilsonPrecisionLower95": 0.918 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9318, - "falseNegative": 4, - "falsePositive": 2, - "precision": 0.9535, - "predictedPositive": 43, - "recall": 0.9111, - "total": 1485, - "trueNegative": 1438, - "truePositive": 41, - "wilsonPrecisionLower95": 0.8454 - }, - "threshold": 0.9 - } - }, - "metrics": { - "f1": 0.9551, - "falseNegative": 5, - "falsePositive": 3, - "precision": 0.9659, - "predictedPositive": 88, - "recall": 0.9444, - "total": 2970, - "trueNegative": 2877, - "truePositive": 85, - "wilsonPrecisionLower95": 0.9045 - }, - "threshold": 0.885 - }, - "task": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8315, - "falseNegative": 96, - "falsePositive": 11, - "precision": 0.96, - "predictedPositive": 275, - "recall": 0.7333, - "total": 1485, - "trueNegative": 1114, - "truePositive": 264, - "wilsonPrecisionLower95": 0.9298 - }, - "threshold": 0.985 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8349, - "falseNegative": 92, - "falsePositive": 14, - "precision": 0.9504, - "predictedPositive": 282, - "recall": 0.7444, - "total": 1485, - "trueNegative": 1111, - "truePositive": 268, - "wilsonPrecisionLower95": 0.9184 - }, - "threshold": 0.845 - } - }, - "metrics": { - "f1": 0.7688, - "falseNegative": 256, - "falsePositive": 23, - "precision": 0.9528, - "predictedPositive": 487, - "recall": 0.6444, - "total": 2970, - "trueNegative": 2227, - "truePositive": 464, - "wilsonPrecisionLower95": 0.9301 - }, - "threshold": 0.975 - } - }, - "trainingRecords": 15345, - "validation": { - "intentsMeeting95Precision": 9, - "macroF1": 0.7269, - "macroPrecision": 0.9659, - "macroRecall": 0.6366, - "minimumPrecision": 0.9507, - "perIntent": { - "blessing": { - "f1": 0.2692, - "falseNegative": 76, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 14, - "recall": 0.1556, - "total": 2970, - "trueNegative": 2880, - "truePositive": 14, - "wilsonPrecisionLower95": 0.7847 - }, - "complaint": { - "f1": 0.9622, - "falseNegative": 16, - "falsePositive": 11, - "precision": 0.969, - "predictedPositive": 355, - "recall": 0.9556, - "total": 2970, - "trueNegative": 2599, - "truePositive": 344, - "wilsonPrecisionLower95": 0.9454 - }, - "confirmationDecision": { - "f1": 0.8365, - "falseNegative": 47, - "falsePositive": 5, - "precision": 0.9638, - "predictedPositive": 138, - "recall": 0.7389, - "total": 2970, - "trueNegative": 2785, - "truePositive": 133, - "wilsonPrecisionLower95": 0.918 - }, - "followUpReminder": { - "f1": 0.4335, - "falseNegative": 259, - "falsePositive": 5, - "precision": 0.9528, - "predictedPositive": 106, - "recall": 0.2806, - "total": 2970, - "trueNegative": 2605, - "truePositive": 101, - "wilsonPrecisionLower95": 0.8943 - }, - "invitation": { - "f1": 0.5669, - "falseNegative": 108, - "falsePositive": 2, - "precision": 0.973, - "predictedPositive": 74, - "recall": 0.4, - "total": 2970, - "trueNegative": 2788, - "truePositive": 72, - "wilsonPrecisionLower95": 0.9067 - }, - "question": { - "f1": 0.7786, - "falseNegative": 215, - "falsePositive": 21, - "precision": 0.9518, - "predictedPositive": 436, - "recall": 0.6587, - "total": 2970, - "trueNegative": 2319, - "truePositive": 415, - "wilsonPrecisionLower95": 0.9275 - }, - "replyableMessage": { - "f1": 0.9073, - "falseNegative": 250, - "falsePositive": 85, - "precision": 0.9507, - "predictedPositive": 1725, - "recall": 0.8677, - "total": 2970, - "trueNegative": 995, - "truePositive": 1640, - "wilsonPrecisionLower95": 0.9395 - }, - "scheduleNegotiation": { - "f1": 0.9545, - "falseNegative": 6, - "falsePositive": 2, - "precision": 0.9767, - "predictedPositive": 86, - "recall": 0.9333, - "total": 2970, - "trueNegative": 2878, - "truePositive": 84, - "wilsonPrecisionLower95": 0.9191 - }, - "task": { - "f1": 0.8332, - "falseNegative": 188, - "falsePositive": 25, - "precision": 0.9551, - "predictedPositive": 557, - "recall": 0.7389, - "total": 2970, - "trueNegative": 2225, - "truePositive": 532, - "wilsonPrecisionLower95": 0.9346 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 4e-06, - "augmentation": "prefix", - "char_max": 6, - "char_min": 3, - "external_weight": 0.79, - "hard_example_weight": 2.0, - "l1_ratio": 0.15, - "max_char_features": 75000, - "max_word_features": 35000, - "min_df": 2, - "pseudo_weight": 0.32999999999999996, - "round": 12, - "word_max": 3 - }, - "featureCount": 94458, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 0 - }, - "complaint": { - "negative": 250, - "positive": 0 - }, - "confirmationDecision": { - "negative": 250, - "positive": 15 - }, - "followUpReminder": { - "negative": 250, - "positive": 0 - }, - "invitation": { - "negative": 250, - "positive": 0 - }, - "question": { - "negative": 232, - "positive": 0 - }, - "replyableMessage": { - "negative": 10, - "positive": 250 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 0 - }, - "task": { - "negative": 250, - "positive": 0 - } - }, - "round": 12, - "selectionObjective": 0.7563, - "syntheticRegression": { - "intentsMeeting95Precision": 5, - "macroF1": 0.6379, - "macroPrecision": 0.9476, - "macroRecall": 0.5322, - "minimumPrecision": 0.8452, - "perIntent": { - "blessing": { - "f1": 0.1053, - "falseNegative": 85, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 5, - "recall": 0.0556, - "total": 2970, - "trueNegative": 2880, - "truePositive": 5, - "wilsonPrecisionLower95": 0.5655 - }, - "complaint": { - "f1": 0.9625, - "falseNegative": 26, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 334, - "recall": 0.9278, - "total": 2970, - "trueNegative": 2610, - "truePositive": 334, - "wilsonPrecisionLower95": 0.9886 - }, - "confirmationDecision": { - "f1": 0.7821, - "falseNegative": 49, - "falsePositive": 24, - "precision": 0.8452, - "predictedPositive": 155, - "recall": 0.7278, - "total": 2970, - "trueNegative": 2766, - "truePositive": 131, - "wilsonPrecisionLower95": 0.7799 - }, - "followUpReminder": { - "f1": 0.6142, - "falseNegative": 200, - "falsePositive": 1, - "precision": 0.9938, - "predictedPositive": 161, - "recall": 0.4444, - "total": 2970, - "trueNegative": 2609, - "truePositive": 160, - "wilsonPrecisionLower95": 0.9657 - }, - "invitation": { - "f1": 0.5992, - "falseNegative": 103, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 77, - "recall": 0.4278, - "total": 2970, - "trueNegative": 2790, - "truePositive": 77, - "wilsonPrecisionLower95": 0.9525 - }, - "question": { - "f1": 0.4319, - "falseNegative": 454, - "falsePositive": 9, - "precision": 0.9514, - "predictedPositive": 185, - "recall": 0.2794, - "total": 2970, - "trueNegative": 2331, - "truePositive": 176, - "wilsonPrecisionLower95": 0.9101 - }, - "replyableMessage": { - "f1": 0.7888, - "falseNegative": 624, - "falsePositive": 54, - "precision": 0.9591, - "predictedPositive": 1320, - "recall": 0.6698, - "total": 2970, - "trueNegative": 1026, - "truePositive": 1266, - "wilsonPrecisionLower95": 0.947 - }, - "scheduleNegotiation": { - "f1": 0.8144, - "falseNegative": 22, - "falsePositive": 9, - "precision": 0.8831, - "predictedPositive": 77, - "recall": 0.7556, - "total": 2970, - "trueNegative": 2871, - "truePositive": 68, - "wilsonPrecisionLower95": 0.7925 - }, - "task": { - "f1": 0.6429, - "falseNegative": 359, - "falsePositive": 42, - "precision": 0.8958, - "predictedPositive": 403, - "recall": 0.5014, - "total": 2970, - "trueNegative": 2208, - "truePositive": 361, - "wilsonPrecisionLower95": 0.8621 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9917, - "falseNegative": 0, - "falsePositive": 3, - "precision": 0.9836, - "predictedPositive": 183, - "recall": 1.0, - "total": 1485, - "trueNegative": 1302, - "truePositive": 180, - "wilsonPrecisionLower95": 0.9529 - }, - "threshold": 0.885 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8974, - "falseNegative": 27, - "falsePositive": 8, - "precision": 0.9503, - "predictedPositive": 161, - "recall": 0.85, - "total": 1485, - "trueNegative": 1297, - "truePositive": 153, - "wilsonPrecisionLower95": 0.905 - }, - "threshold": 0.9 - } - }, - "metrics": { - "f1": 0.9515, - "falseNegative": 17, - "falsePositive": 18, - "precision": 0.9501, - "predictedPositive": 361, - "recall": 0.9528, - "total": 2970, - "trueNegative": 2592, - "truePositive": 343, - "wilsonPrecisionLower95": 0.9226 - }, - "threshold": 0.77 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9944, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 89, - "recall": 0.9889, - "total": 1485, - "trueNegative": 1395, - "truePositive": 89, - "wilsonPrecisionLower95": 0.9586 - }, - "threshold": 0.915 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.4959, - "falseNegative": 60, - "falsePositive": 1, - "precision": 0.9677, - "predictedPositive": 31, - "recall": 0.3333, - "total": 1485, - "trueNegative": 1394, - "truePositive": 30, - "wilsonPrecisionLower95": 0.8381 - }, - "threshold": 0.985 - } - }, - "metrics": { - "f1": 0.8217, - "falseNegative": 51, - "falsePositive": 5, - "precision": 0.9627, - "predictedPositive": 134, - "recall": 0.7167, - "total": 2970, - "trueNegative": 2785, - "truePositive": 129, - "wilsonPrecisionLower95": 0.9156 - }, - "threshold": 0.915 - }, - "followUpReminder": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.5647, - "falseNegative": 108, - "falsePositive": 3, - "precision": 0.96, - "predictedPositive": 75, - "recall": 0.4, - "total": 1485, - "trueNegative": 1302, - "truePositive": 72, - "wilsonPrecisionLower95": 0.8889 - }, - "threshold": 0.995 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6542, - "falseNegative": 158, - "falsePositive": 8, - "precision": 0.9515, - "predictedPositive": 165, - "recall": 0.4984, - "total": 1485, - "trueNegative": 1162, - "truePositive": 157, - "wilsonPrecisionLower95": 0.9073 - }, - "threshold": 0.805 - } - }, - "metrics": { - "f1": 0.5711, - "falseNegative": 373, - "falsePositive": 13, - "precision": 0.9519, - "predictedPositive": 270, - "recall": 0.4079, - "total": 2970, - "trueNegative": 2327, - "truePositive": 257, - "wilsonPrecisionLower95": 0.9194 - }, - "threshold": 0.985 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8956, - "falseNegative": 147, - "falsePositive": 39, - "precision": 0.9534, - "predictedPositive": 837, - "recall": 0.8444, - "total": 1485, - "trueNegative": 501, - "truePositive": 798, - "wilsonPrecisionLower95": 0.9369 - }, - "threshold": 0.975 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6735, - "falseNegative": 453, - "falsePositive": 24, - "precision": 0.9535, - "predictedPositive": 516, - "recall": 0.5206, - "total": 1485, - "trueNegative": 516, - "truePositive": 492, - "wilsonPrecisionLower95": 0.9317 - }, - "threshold": 0.985 - } - }, - "metrics": { - "f1": 0.7866, - "falseNegative": 626, - "falsePositive": 60, - "precision": 0.9547, - "predictedPositive": 1324, - "recall": 0.6688, - "total": 2970, - "trueNegative": 1020, - "truePositive": 1264, - "wilsonPrecisionLower95": 0.9421 - }, - "threshold": 0.985 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9888, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 44, - "recall": 0.9778, - "total": 1485, - "trueNegative": 1440, - "truePositive": 44, - "wilsonPrecisionLower95": 0.9197 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8052, - "falseNegative": 14, - "falsePositive": 1, - "precision": 0.9688, - "predictedPositive": 32, - "recall": 0.6889, - "total": 1485, - "trueNegative": 1439, - "truePositive": 31, - "wilsonPrecisionLower95": 0.8426 - }, - "threshold": 0.94 - } - }, - "metrics": { - "f1": 0.9364, - "falseNegative": 9, - "falsePositive": 2, - "precision": 0.9759, - "predictedPositive": 83, - "recall": 0.9, - "total": 2970, - "trueNegative": 2878, - "truePositive": 81, - "wilsonPrecisionLower95": 0.9163 - }, - "threshold": 0.715 - }, - "task": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8475, - "falseNegative": 85, - "falsePositive": 14, - "precision": 0.9516, - "predictedPositive": 289, - "recall": 0.7639, - "total": 1485, - "trueNegative": 1111, - "truePositive": 275, - "wilsonPrecisionLower95": 0.9203 - }, - "threshold": 0.985 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.501, - "falseNegative": 238, - "falsePositive": 5, - "precision": 0.9606, - "predictedPositive": 127, - "recall": 0.3389, - "total": 1485, - "trueNegative": 1120, - "truePositive": 122, - "wilsonPrecisionLower95": 0.9111 - }, - "threshold": 0.985 - } - }, - "metrics": { - "f1": 0.6989, - "falseNegative": 323, - "falsePositive": 19, - "precision": 0.9543, - "predictedPositive": 416, - "recall": 0.5514, - "total": 2970, - "trueNegative": 2231, - "truePositive": 397, - "wilsonPrecisionLower95": 0.9298 - }, - "threshold": 0.985 - } - }, - "trainingRecords": 15345, - "validation": { - "intentsMeeting95Precision": 8, - "macroF1": 0.6603, - "macroPrecision": 0.9686, - "macroRecall": 0.5413, - "minimumPrecision": 0.9298, - "perIntent": { - "blessing": { - "f1": 0.2857, - "falseNegative": 75, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 15, - "recall": 0.1667, - "total": 2970, - "trueNegative": 2880, - "truePositive": 15, - "wilsonPrecisionLower95": 0.7961 - }, - "complaint": { - "f1": 0.946, - "falseNegative": 27, - "falsePositive": 11, - "precision": 0.968, - "predictedPositive": 344, - "recall": 0.925, - "total": 2970, - "trueNegative": 2599, - "truePositive": 333, - "wilsonPrecisionLower95": 0.9437 - }, - "confirmationDecision": { - "f1": 0.7933, - "falseNegative": 61, - "falsePositive": 1, - "precision": 0.9917, - "predictedPositive": 120, - "recall": 0.6611, - "total": 2970, - "trueNegative": 2789, - "truePositive": 119, - "wilsonPrecisionLower95": 0.9543 - }, - "followUpReminder": { - "f1": 0.6494, - "falseNegative": 185, - "falsePositive": 4, - "precision": 0.9777, - "predictedPositive": 179, - "recall": 0.4861, - "total": 2970, - "trueNegative": 2606, - "truePositive": 175, - "wilsonPrecisionLower95": 0.944 - }, - "invitation": { - "f1": 0.4473, - "falseNegative": 127, - "falsePositive": 4, - "precision": 0.9298, - "predictedPositive": 57, - "recall": 0.2944, - "total": 2970, - "trueNegative": 2786, - "truePositive": 53, - "wilsonPrecisionLower95": 0.833 - }, - "question": { - "f1": 0.4227, - "falseNegative": 459, - "falsePositive": 8, - "precision": 0.9553, - "predictedPositive": 179, - "recall": 0.2714, - "total": 2970, - "trueNegative": 2332, - "truePositive": 171, - "wilsonPrecisionLower95": 0.9143 - }, - "replyableMessage": { - "f1": 0.7956, - "falseNegative": 600, - "falsePositive": 63, - "precision": 0.9534, - "predictedPositive": 1353, - "recall": 0.6825, - "total": 2970, - "trueNegative": 1017, - "truePositive": 1290, - "wilsonPrecisionLower95": 0.9409 - }, - "scheduleNegotiation": { - "f1": 0.9036, - "falseNegative": 15, - "falsePositive": 1, - "precision": 0.9868, - "predictedPositive": 76, - "recall": 0.8333, - "total": 2970, - "trueNegative": 2879, - "truePositive": 75, - "wilsonPrecisionLower95": 0.9292 - }, - "task": { - "f1": 0.6989, - "falseNegative": 323, - "falsePositive": 19, - "precision": 0.9543, - "predictedPositive": 416, - "recall": 0.5514, - "total": 2970, - "trueNegative": 2231, - "truePositive": 397, - "wilsonPrecisionLower95": 0.9298 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 3e-06, - "augmentation": "prefix", - "char_max": 7, - "char_min": 2, - "external_weight": 0.81, - "hard_example_weight": 2.0, - "l1_ratio": 0.15, - "max_char_features": 80000, - "max_word_features": 38000, - "min_df": 2, - "pseudo_weight": 0.345, - "round": 13, - "word_max": 3 - }, - "featureCount": 99458, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 0 - }, - "complaint": { - "negative": 250, - "positive": 1 - }, - "confirmationDecision": { - "negative": 250, - "positive": 18 - }, - "followUpReminder": { - "negative": 250, - "positive": 0 - }, - "invitation": { - "negative": 250, - "positive": 0 - }, - "question": { - "negative": 250, - "positive": 0 - }, - "replyableMessage": { - "negative": 12, - "positive": 250 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 0 - }, - "task": { - "negative": 250, - "positive": 0 - } - }, - "round": 13, - "selectionObjective": 0.768, - "syntheticRegression": { - "intentsMeeting95Precision": 4, - "macroF1": 0.6577, - "macroPrecision": 0.9568, - "macroRecall": 0.5722, - "minimumPrecision": 0.859, - "perIntent": { - "blessing": { - "f1": 0.1443, - "falseNegative": 83, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 7, - "recall": 0.0778, - "total": 2970, - "trueNegative": 2880, - "truePositive": 7, - "wilsonPrecisionLower95": 0.6457 - }, - "complaint": { - "f1": 0.9744, - "falseNegative": 17, - "falsePositive": 1, - "precision": 0.9971, - "predictedPositive": 344, - "recall": 0.9528, - "total": 2970, - "trueNegative": 2609, - "truePositive": 343, - "wilsonPrecisionLower95": 0.9837 - }, - "confirmationDecision": { - "f1": 0.7976, - "falseNegative": 46, - "falsePositive": 22, - "precision": 0.859, - "predictedPositive": 156, - "recall": 0.7444, - "total": 2970, - "trueNegative": 2768, - "truePositive": 134, - "wilsonPrecisionLower95": 0.7957 - }, - "followUpReminder": { - "f1": 0.2972, - "falseNegative": 297, - "falsePositive": 1, - "precision": 0.9844, - "predictedPositive": 64, - "recall": 0.175, - "total": 2970, - "trueNegative": 2609, - "truePositive": 63, - "wilsonPrecisionLower95": 0.9167 - }, - "invitation": { - "f1": 0.6077, - "falseNegative": 101, - "falsePositive": 1, - "precision": 0.9875, - "predictedPositive": 80, - "recall": 0.4389, - "total": 2970, - "trueNegative": 2789, - "truePositive": 79, - "wilsonPrecisionLower95": 0.9325 - }, - "question": { - "f1": 0.5229, - "falseNegative": 407, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 223, - "recall": 0.354, - "total": 2970, - "trueNegative": 2340, - "truePositive": 223, - "wilsonPrecisionLower95": 0.9831 - }, - "replyableMessage": { - "f1": 0.9179, - "falseNegative": 202, - "falsePositive": 100, - "precision": 0.9441, - "predictedPositive": 1788, - "recall": 0.8931, - "total": 2970, - "trueNegative": 980, - "truePositive": 1688, - "wilsonPrecisionLower95": 0.9324 - }, - "scheduleNegotiation": { - "f1": 0.8889, - "falseNegative": 14, - "falsePositive": 5, - "precision": 0.9383, - "predictedPositive": 81, - "recall": 0.8444, - "total": 2970, - "trueNegative": 2875, - "truePositive": 76, - "wilsonPrecisionLower95": 0.8635 - }, - "task": { - "f1": 0.7681, - "falseNegative": 238, - "falsePositive": 53, - "precision": 0.9009, - "predictedPositive": 535, - "recall": 0.6694, - "total": 2970, - "trueNegative": 2197, - "truePositive": 482, - "wilsonPrecisionLower95": 0.8727 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9727, - "falseNegative": 2, - "falsePositive": 8, - "precision": 0.957, - "predictedPositive": 186, - "recall": 0.9889, - "total": 1485, - "trueNegative": 1297, - "truePositive": 178, - "wilsonPrecisionLower95": 0.9174 - }, - "threshold": 0.98 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.95, - "falseNegative": 9, - "falsePositive": 9, - "precision": 0.95, - "predictedPositive": 180, - "recall": 0.95, - "total": 1485, - "trueNegative": 1296, - "truePositive": 171, - "wilsonPrecisionLower95": 0.9077 - }, - "threshold": 0.905 - } - }, - "metrics": { - "f1": 0.9529, - "falseNegative": 16, - "falsePositive": 18, - "precision": 0.9503, - "predictedPositive": 362, - "recall": 0.9556, - "total": 2970, - "trueNegative": 2592, - "truePositive": 344, - "wilsonPrecisionLower95": 0.9228 - }, - "threshold": 0.965 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.828, - "falseNegative": 25, - "falsePositive": 2, - "precision": 0.9701, - "predictedPositive": 67, - "recall": 0.7222, - "total": 1485, - "trueNegative": 1393, - "truePositive": 65, - "wilsonPrecisionLower95": 0.8975 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6763, - "falseNegative": 43, - "falsePositive": 2, - "precision": 0.9592, - "predictedPositive": 49, - "recall": 0.5222, - "total": 1485, - "trueNegative": 1393, - "truePositive": 47, - "wilsonPrecisionLower95": 0.8629 - }, - "threshold": 0.93 - } - }, - "metrics": { - "f1": 0.7855, - "falseNegative": 61, - "falsePositive": 4, - "precision": 0.9675, - "predictedPositive": 123, - "recall": 0.6611, - "total": 2970, - "trueNegative": 2786, - "truePositive": 119, - "wilsonPrecisionLower95": 0.9194 - }, - "threshold": 0.82 - }, - "followUpReminder": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7452, - "falseNegative": 122, - "falsePositive": 10, - "precision": 0.9507, - "predictedPositive": 203, - "recall": 0.6127, - "total": 1485, - "trueNegative": 1160, - "truePositive": 193, - "wilsonPrecisionLower95": 0.9117 - }, - "threshold": 0.915 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9441, - "falseNegative": 59, - "falsePositive": 46, - "precision": 0.9506, - "predictedPositive": 932, - "recall": 0.9376, - "total": 1485, - "trueNegative": 494, - "truePositive": 886, - "wilsonPrecisionLower95": 0.9348 - }, - "threshold": 0.935 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9067, - "falseNegative": 129, - "falsePositive": 39, - "precision": 0.9544, - "predictedPositive": 855, - "recall": 0.8635, - "total": 1485, - "trueNegative": 501, - "truePositive": 816, - "wilsonPrecisionLower95": 0.9383 - }, - "threshold": 0.905 - } - }, - "metrics": { - "f1": 0.9267, - "falseNegative": 182, - "falsePositive": 88, - "precision": 0.951, - "predictedPositive": 1796, - "recall": 0.9037, - "total": 2970, - "trueNegative": 992, - "truePositive": 1708, - "wilsonPrecisionLower95": 0.94 - }, - "threshold": 0.905 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9888, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 44, - "recall": 0.9778, - "total": 1485, - "trueNegative": 1440, - "truePositive": 44, - "wilsonPrecisionLower95": 0.9197 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9655, - "falseNegative": 3, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 42, - "recall": 0.9333, - "total": 1485, - "trueNegative": 1440, - "truePositive": 42, - "wilsonPrecisionLower95": 0.9162 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.9773, - "falseNegative": 4, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 86, - "recall": 0.9556, - "total": 2970, - "trueNegative": 2880, - "truePositive": 86, - "wilsonPrecisionLower95": 0.9572 - }, - "threshold": 0.5 - }, - "task": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.7876, - "falseNegative": 119, - "falsePositive": 11, - "precision": 0.9563, - "predictedPositive": 252, - "recall": 0.6694, - "total": 1485, - "trueNegative": 1114, - "truePositive": 241, - "wilsonPrecisionLower95": 0.9235 - }, - "threshold": 0.995 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8416, - "falseNegative": 89, - "falsePositive": 13, - "precision": 0.9542, - "predictedPositive": 284, - "recall": 0.7528, - "total": 1485, - "trueNegative": 1112, - "truePositive": 271, - "wilsonPrecisionLower95": 0.9233 - }, - "threshold": 0.895 - } - }, - "metrics": { - "f1": 0.6962, - "falseNegative": 327, - "falsePositive": 16, - "precision": 0.9609, - "predictedPositive": 409, - "recall": 0.5458, - "total": 2970, - "trueNegative": 2234, - "truePositive": 393, - "wilsonPrecisionLower95": 0.9374 - }, - "threshold": 0.995 - } - }, - "trainingRecords": 15346, - "validation": { - "intentsMeeting95Precision": 9, - "macroF1": 0.6724, - "macroPrecision": 0.9724, - "macroRecall": 0.5779, - "minimumPrecision": 0.9524, - "perIntent": { - "blessing": { - "f1": 0.2178, - "falseNegative": 79, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 11, - "recall": 0.1222, - "total": 2970, - "trueNegative": 2880, - "truePositive": 11, - "wilsonPrecisionLower95": 0.7412 - }, - "complaint": { - "f1": 0.9614, - "falseNegative": 11, - "falsePositive": 17, - "precision": 0.9536, - "predictedPositive": 366, - "recall": 0.9694, - "total": 2970, - "trueNegative": 2593, - "truePositive": 349, - "wilsonPrecisionLower95": 0.9269 - }, - "confirmationDecision": { - "f1": 0.7568, - "falseNegative": 68, - "falsePositive": 4, - "precision": 0.9655, - "predictedPositive": 116, - "recall": 0.6222, - "total": 2970, - "trueNegative": 2786, - "truePositive": 112, - "wilsonPrecisionLower95": 0.9147 - }, - "followUpReminder": { - "f1": 0.4895, - "falseNegative": 243, - "falsePositive": 1, - "precision": 0.9915, - "predictedPositive": 118, - "recall": 0.325, - "total": 2970, - "trueNegative": 2609, - "truePositive": 117, - "wilsonPrecisionLower95": 0.9536 - }, - "invitation": { - "f1": 0.4192, - "falseNegative": 132, - "falsePositive": 1, - "precision": 0.9796, - "predictedPositive": 49, - "recall": 0.2667, - "total": 2970, - "trueNegative": 2789, - "truePositive": 48, - "wilsonPrecisionLower95": 0.8931 - }, - "question": { - "f1": 0.4888, - "falseNegative": 423, - "falsePositive": 10, - "precision": 0.9539, - "predictedPositive": 217, - "recall": 0.3286, - "total": 2970, - "trueNegative": 2330, - "truePositive": 207, - "wilsonPrecisionLower95": 0.9173 - }, - "replyableMessage": { - "f1": 0.9258, - "falseNegative": 188, - "falsePositive": 85, - "precision": 0.9524, - "predictedPositive": 1787, - "recall": 0.9005, - "total": 2970, - "trueNegative": 995, - "truePositive": 1702, - "wilsonPrecisionLower95": 0.9416 - }, - "scheduleNegotiation": { - "f1": 0.9773, - "falseNegative": 4, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 86, - "recall": 0.9556, - "total": 2970, - "trueNegative": 2880, - "truePositive": 86, - "wilsonPrecisionLower95": 0.9572 - }, - "task": { - "f1": 0.8153, - "falseNegative": 208, - "falsePositive": 24, - "precision": 0.9552, - "predictedPositive": 536, - "recall": 0.7111, - "total": 2970, - "trueNegative": 2226, - "truePositive": 512, - "wilsonPrecisionLower95": 0.9342 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 2.5e-06, - "augmentation": "prefix", - "char_max": 7, - "char_min": 3, - "external_weight": 0.8300000000000001, - "hard_example_weight": 2.2, - "l1_ratio": 0.2, - "max_char_features": 80000, - "max_word_features": 40000, - "min_df": 1, - "pseudo_weight": 0.36, - "round": 14, - "word_max": 3 - }, - "featureCount": 120000, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 0 - }, - "complaint": { - "negative": 250, - "positive": 8 - }, - "confirmationDecision": { - "negative": 250, - "positive": 26 - }, - "followUpReminder": { - "negative": 250, - "positive": 0 - }, - "invitation": { - "negative": 250, - "positive": 0 - }, - "question": { - "negative": 250, - "positive": 0 - }, - "replyableMessage": { - "negative": 34, - "positive": 250 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 0 - }, - "task": { - "negative": 250, - "positive": 3 - } - }, - "round": 14, - "selectionObjective": 0.7391, - "syntheticRegression": { - "intentsMeeting95Precision": 5, - "macroF1": 0.6159, - "macroPrecision": 0.9488, - "macroRecall": 0.5171, - "minimumPrecision": 0.8315, - "perIntent": { - "blessing": { - "f1": 0.1443, - "falseNegative": 83, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 7, - "recall": 0.0778, - "total": 2970, - "trueNegative": 2880, - "truePositive": 7, - "wilsonPrecisionLower95": 0.6457 - }, - "complaint": { - "f1": 0.7987, - "falseNegative": 114, - "falsePositive": 10, - "precision": 0.9609, - "predictedPositive": 256, - "recall": 0.6833, - "total": 2970, - "trueNegative": 2600, - "truePositive": 246, - "wilsonPrecisionLower95": 0.9296 - }, - "confirmationDecision": { - "f1": 0.8407, - "falseNegative": 27, - "falsePositive": 31, - "precision": 0.8315, - "predictedPositive": 184, - "recall": 0.85, - "total": 2970, - "trueNegative": 2759, - "truePositive": 153, - "wilsonPrecisionLower95": 0.7708 - }, - "followUpReminder": { - "f1": 0.7361, - "falseNegative": 148, - "falsePositive": 4, - "precision": 0.9815, - "predictedPositive": 216, - "recall": 0.5889, - "total": 2970, - "trueNegative": 2606, - "truePositive": 212, - "wilsonPrecisionLower95": 0.9534 - }, - "invitation": { - "f1": 0.5591, - "falseNegative": 109, - "falsePositive": 3, - "precision": 0.9595, - "predictedPositive": 74, - "recall": 0.3944, - "total": 2970, - "trueNegative": 2787, - "truePositive": 71, - "wilsonPrecisionLower95": 0.8875 - }, - "question": { - "f1": 0.7917, - "falseNegative": 212, - "falsePositive": 8, - "precision": 0.9812, - "predictedPositive": 426, - "recall": 0.6635, - "total": 2970, - "trueNegative": 2332, - "truePositive": 418, - "wilsonPrecisionLower95": 0.9634 - }, - "replyableMessage": { - "f1": 0.6113, - "falseNegative": 1037, - "falsePositive": 48, - "precision": 0.9467, - "predictedPositive": 901, - "recall": 0.4513, - "total": 2970, - "trueNegative": 1032, - "truePositive": 853, - "wilsonPrecisionLower95": 0.9301 - }, - "scheduleNegotiation": { - "f1": 0.9176, - "falseNegative": 12, - "falsePositive": 2, - "precision": 0.975, - "predictedPositive": 80, - "recall": 0.8667, - "total": 2970, - "trueNegative": 2878, - "truePositive": 78, - "wilsonPrecisionLower95": 0.9134 - }, - "task": { - "f1": 0.1432, - "falseNegative": 664, - "falsePositive": 6, - "precision": 0.9032, - "predictedPositive": 62, - "recall": 0.0778, - "total": 2970, - "trueNegative": 2244, - "truePositive": 56, - "wilsonPrecisionLower95": 0.8045 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9809, - "falseNegative": 0, - "falsePositive": 7, - "precision": 0.9626, - "predictedPositive": 187, - "recall": 1.0, - "total": 1485, - "trueNegative": 1298, - "truePositive": 180, - "wilsonPrecisionLower95": 0.9248 - }, - "threshold": 0.99 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8625, - "falseNegative": 21, - "falsePositive": 1, - "precision": 0.9857, - "predictedPositive": 70, - "recall": 0.7667, - "total": 1485, - "trueNegative": 1394, - "truePositive": 69, - "wilsonPrecisionLower95": 0.9234 - }, - "threshold": 0.905 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.828, - "falseNegative": 25, - "falsePositive": 2, - "precision": 0.9701, - "predictedPositive": 67, - "recall": 0.7222, - "total": 1485, - "trueNegative": 1393, - "truePositive": 65, - "wilsonPrecisionLower95": 0.8975 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.8328, - "falseNegative": 48, - "falsePositive": 5, - "precision": 0.9635, - "predictedPositive": 137, - "recall": 0.7333, - "total": 2970, - "trueNegative": 2785, - "truePositive": 132, - "wilsonPrecisionLower95": 0.9174 - }, - "threshold": 0.685 - }, - "followUpReminder": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8788, - "falseNegative": 35, - "falsePositive": 5, - "precision": 0.9667, - "predictedPositive": 150, - "recall": 0.8056, - "total": 1485, - "trueNegative": 1300, - "truePositive": 145, - "wilsonPrecisionLower95": 0.9243 - }, - "threshold": 0.845 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6113, - "falseNegative": 99, - "falsePositive": 4, - "precision": 0.9529, - "predictedPositive": 85, - "recall": 0.45, - "total": 1485, - "trueNegative": 1301, - "truePositive": 81, - "wilsonPrecisionLower95": 0.8852 - }, - "threshold": 0.97 - } - }, - "metrics": { - "f1": 0.7682, - "falseNegative": 128, - "falsePositive": 12, - "precision": 0.9508, - "predictedPositive": 244, - "recall": 0.6444, - "total": 2970, - "trueNegative": 2598, - "truePositive": 232, - "wilsonPrecisionLower95": 0.916 - }, - "threshold": 0.91 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.847, - "falseNegative": 77, - "falsePositive": 9, - "precision": 0.9636, - "predictedPositive": 247, - "recall": 0.7556, - "total": 1485, - "trueNegative": 1161, - "truePositive": 238, - "wilsonPrecisionLower95": 0.9322 - }, - "threshold": 0.995 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7168, - "falseNegative": 134, - "falsePositive": 9, - "precision": 0.9526, - "predictedPositive": 190, - "recall": 0.5746, - "total": 1485, - "trueNegative": 1161, - "truePositive": 181, - "wilsonPrecisionLower95": 0.9124 - }, - "threshold": 0.865 - } - }, - "metrics": { - "f1": 0.7524, - "falseNegative": 238, - "falsePositive": 20, - "precision": 0.9515, - "predictedPositive": 412, - "recall": 0.6222, - "total": 2970, - "trueNegative": 2320, - "truePositive": 392, - "wilsonPrecisionLower95": 0.9262 - }, - "threshold": 0.975 - }, - "replyableMessage": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9349, - "falseNegative": 76, - "falsePositive": 45, - "precision": 0.9508, - "predictedPositive": 914, - "recall": 0.9196, - "total": 1485, - "trueNegative": 495, - "truePositive": 869, - "wilsonPrecisionLower95": 0.9348 - }, - "threshold": 0.955 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 45, - "recall": 1.0, - "total": 1485, - "trueNegative": 1440, - "truePositive": 45, - "wilsonPrecisionLower95": 0.9213 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.967, - "falseNegative": 1, - "falsePositive": 2, - "precision": 0.9565, - "predictedPositive": 46, - "recall": 0.9778, - "total": 1485, - "trueNegative": 1438, - "truePositive": 44, - "wilsonPrecisionLower95": 0.8547 - }, - "threshold": 0.995 - } - }, - "metrics": { - "f1": 0.9727, - "falseNegative": 1, - "falsePositive": 4, - "precision": 0.957, - "predictedPositive": 93, - "recall": 0.9889, - "total": 2970, - "trueNegative": 2876, - "truePositive": 89, - "wilsonPrecisionLower95": 0.8946 - }, - "threshold": 0.64 - }, - "task": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "trainingRecords": 15345, - "validation": { - "intentsMeeting95Precision": 8, - "macroF1": 0.6332, - "macroPrecision": 0.9668, - "macroRecall": 0.5246, - "minimumPrecision": 0.9, - "perIntent": { - "blessing": { - "f1": 0.3019, - "falseNegative": 74, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 16, - "recall": 0.1778, - "total": 2970, - "trueNegative": 2880, - "truePositive": 16, - "wilsonPrecisionLower95": 0.8064 - }, - "complaint": { - "f1": 0.8199, - "falseNegative": 105, - "falsePositive": 7, - "precision": 0.9733, - "predictedPositive": 262, - "recall": 0.7083, - "total": 2970, - "trueNegative": 2603, - "truePositive": 255, - "wilsonPrecisionLower95": 0.9459 - }, - "confirmationDecision": { - "f1": 0.8454, - "falseNegative": 46, - "falsePositive": 3, - "precision": 0.9781, - "predictedPositive": 137, - "recall": 0.7444, - "total": 2970, - "trueNegative": 2787, - "truePositive": 134, - "wilsonPrecisionLower95": 0.9376 - }, - "followUpReminder": { - "f1": 0.7597, - "falseNegative": 134, - "falsePositive": 9, - "precision": 0.9617, - "predictedPositive": 235, - "recall": 0.6278, - "total": 2970, - "trueNegative": 2601, - "truePositive": 226, - "wilsonPrecisionLower95": 0.9288 - }, - "invitation": { - "f1": 0.3273, - "falseNegative": 144, - "falsePositive": 4, - "precision": 0.9, - "predictedPositive": 40, - "recall": 0.2, - "total": 2970, - "trueNegative": 2786, - "truePositive": 36, - "wilsonPrecisionLower95": 0.7695 - }, - "question": { - "f1": 0.7854, - "falseNegative": 211, - "falsePositive": 18, - "precision": 0.9588, - "predictedPositive": 437, - "recall": 0.6651, - "total": 2970, - "trueNegative": 2322, - "truePositive": 419, - "wilsonPrecisionLower95": 0.9358 - }, - "replyableMessage": { - "f1": 0.6272, - "falseNegative": 1006, - "falsePositive": 45, - "precision": 0.9516, - "predictedPositive": 929, - "recall": 0.4677, - "total": 2970, - "trueNegative": 1035, - "truePositive": 884, - "wilsonPrecisionLower95": 0.9358 - }, - "scheduleNegotiation": { - "f1": 0.9834, - "falseNegative": 1, - "falsePositive": 2, - "precision": 0.978, - "predictedPositive": 91, - "recall": 0.9889, - "total": 2970, - "trueNegative": 2878, - "truePositive": 89, - "wilsonPrecisionLower95": 0.9234 - }, - "task": { - "f1": 0.2482, - "falseNegative": 618, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 102, - "recall": 0.1417, - "total": 2970, - "trueNegative": 2250, - "truePositive": 102, - "wilsonPrecisionLower95": 0.9637 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 3.5e-06, - "augmentation": "prefix", - "char_max": 6, - "char_min": 2, - "external_weight": 0.8500000000000001, - "hard_example_weight": 2.2, - "l1_ratio": 0.2, - "max_char_features": 85000, - "max_word_features": 42000, - "min_df": 1, - "pseudo_weight": 0.375, - "round": 15, - "word_max": 3 - }, - "featureCount": 127000, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 1 - }, - "complaint": { - "negative": 250, - "positive": 8 - }, - "confirmationDecision": { - "negative": 250, - "positive": 30 - }, - "followUpReminder": { - "negative": 250, - "positive": 0 - }, - "invitation": { - "negative": 250, - "positive": 0 - }, - "question": { - "negative": 250, - "positive": 0 - }, - "replyableMessage": { - "negative": 39, - "positive": 250 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 0 - }, - "task": { - "negative": 250, - "positive": 0 - } - }, - "round": 15, - "selectionObjective": 0.7896, - "syntheticRegression": { - "intentsMeeting95Precision": 7, - "macroF1": 0.7019, - "macroPrecision": 0.9601, - "macroRecall": 0.6059, - "minimumPrecision": 0.8634, - "perIntent": { - "blessing": { - "f1": 0.2, - "falseNegative": 80, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 10, - "recall": 0.1111, - "total": 2970, - "trueNegative": 2880, - "truePositive": 10, - "wilsonPrecisionLower95": 0.7225 - }, - "complaint": { - "f1": 0.9641, - "falseNegative": 24, - "falsePositive": 1, - "precision": 0.997, - "predictedPositive": 337, - "recall": 0.9333, - "total": 2970, - "trueNegative": 2609, - "truePositive": 336, - "wilsonPrecisionLower95": 0.9834 - }, - "confirmationDecision": { - "f1": 0.8152, - "falseNegative": 41, - "falsePositive": 22, - "precision": 0.8634, - "predictedPositive": 161, - "recall": 0.7722, - "total": 2970, - "trueNegative": 2768, - "truePositive": 139, - "wilsonPrecisionLower95": 0.8018 - }, - "followUpReminder": { - "f1": 0.6115, - "falseNegative": 201, - "falsePositive": 1, - "precision": 0.9938, - "predictedPositive": 160, - "recall": 0.4417, - "total": 2970, - "trueNegative": 2609, - "truePositive": 159, - "wilsonPrecisionLower95": 0.9655 - }, - "invitation": { - "f1": 0.6084, - "falseNegative": 100, - "falsePositive": 3, - "precision": 0.9639, - "predictedPositive": 83, - "recall": 0.4444, - "total": 2970, - "trueNegative": 2787, - "truePositive": 80, - "wilsonPrecisionLower95": 0.899 - }, - "question": { - "f1": 0.5421, - "falseNegative": 395, - "falsePositive": 2, - "precision": 0.9916, - "predictedPositive": 237, - "recall": 0.373, - "total": 2970, - "trueNegative": 2338, - "truePositive": 235, - "wilsonPrecisionLower95": 0.9698 - }, - "replyableMessage": { - "f1": 0.862, - "falseNegative": 410, - "falsePositive": 64, - "precision": 0.9585, - "predictedPositive": 1544, - "recall": 0.7831, - "total": 2970, - "trueNegative": 1016, - "truePositive": 1480, - "wilsonPrecisionLower95": 0.9474 - }, - "scheduleNegotiation": { - "f1": 0.9379, - "falseNegative": 7, - "falsePositive": 4, - "precision": 0.954, - "predictedPositive": 87, - "recall": 0.9222, - "total": 2970, - "trueNegative": 2876, - "truePositive": 83, - "wilsonPrecisionLower95": 0.8877 - }, - "task": { - "f1": 0.7763, - "falseNegative": 236, - "falsePositive": 43, - "precision": 0.9184, - "predictedPositive": 527, - "recall": 0.6722, - "total": 2970, - "trueNegative": 2207, - "truePositive": 484, - "wilsonPrecisionLower95": 0.8919 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9836, - "falseNegative": 0, - "falsePositive": 6, - "precision": 0.9677, - "predictedPositive": 186, - "recall": 1.0, - "total": 1485, - "trueNegative": 1299, - "truePositive": 180, - "wilsonPrecisionLower95": 0.9314 - }, - "threshold": 0.945 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.928, - "falseNegative": 19, - "falsePositive": 6, - "precision": 0.9641, - "predictedPositive": 167, - "recall": 0.8944, - "total": 1485, - "trueNegative": 1299, - "truePositive": 161, - "wilsonPrecisionLower95": 0.9238 - }, - "threshold": 0.77 - } - }, - "metrics": { - "f1": 0.9365, - "falseNegative": 28, - "falsePositive": 17, - "precision": 0.9513, - "predictedPositive": 349, - "recall": 0.9222, - "total": 2970, - "trueNegative": 2593, - "truePositive": 332, - "wilsonPrecisionLower95": 0.9234 - }, - "threshold": 0.88 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.954, - "falseNegative": 7, - "falsePositive": 1, - "precision": 0.9881, - "predictedPositive": 84, - "recall": 0.9222, - "total": 1485, - "trueNegative": 1394, - "truePositive": 83, - "wilsonPrecisionLower95": 0.9356 - }, - "threshold": 0.85 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6906, - "falseNegative": 42, - "falsePositive": 1, - "precision": 0.9796, - "predictedPositive": 49, - "recall": 0.5333, - "total": 1485, - "trueNegative": 1394, - "truePositive": 48, - "wilsonPrecisionLower95": 0.8931 - }, - "threshold": 0.985 - } - }, - "metrics": { - "f1": 0.8598, - "falseNegative": 39, - "falsePositive": 7, - "precision": 0.9527, - "predictedPositive": 148, - "recall": 0.7833, - "total": 2970, - "trueNegative": 2783, - "truePositive": 141, - "wilsonPrecisionLower95": 0.9056 - }, - "threshold": 0.935 - }, - "followUpReminder": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8328, - "falseNegative": 48, - "falsePositive": 5, - "precision": 0.9635, - "predictedPositive": 137, - "recall": 0.7333, - "total": 1485, - "trueNegative": 1300, - "truePositive": 132, - "wilsonPrecisionLower95": 0.9174 - }, - "threshold": 0.99 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9133, - "falseNegative": 11, - "falsePositive": 4, - "precision": 0.9518, - "predictedPositive": 83, - "recall": 0.8778, - "total": 1485, - "trueNegative": 1391, - "truePositive": 79, - "wilsonPrecisionLower95": 0.8825 - }, - "threshold": 0.97 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7548, - "falseNegative": 118, - "falsePositive": 10, - "precision": 0.9517, - "predictedPositive": 207, - "recall": 0.6254, - "total": 1485, - "trueNegative": 1160, - "truePositive": 197, - "wilsonPrecisionLower95": 0.9134 - }, - "threshold": 0.8 - } - }, - "metrics": { - "f1": 0.7678, - "falseNegative": 225, - "falsePositive": 20, - "precision": 0.9529, - "predictedPositive": 425, - "recall": 0.6429, - "total": 2970, - "trueNegative": 2320, - "truePositive": 405, - "wilsonPrecisionLower95": 0.9284 - }, - "threshold": 0.985 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.907, - "falseNegative": 126, - "falsePositive": 42, - "precision": 0.9512, - "predictedPositive": 861, - "recall": 0.8667, - "total": 1485, - "trueNegative": 498, - "truePositive": 819, - "wilsonPrecisionLower95": 0.9347 - }, - "threshold": 0.98 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8252, - "falseNegative": 258, - "falsePositive": 33, - "precision": 0.9542, - "predictedPositive": 720, - "recall": 0.727, - "total": 1485, - "trueNegative": 507, - "truePositive": 687, - "wilsonPrecisionLower95": 0.9363 - }, - "threshold": 0.96 - } - }, - "metrics": { - "f1": 0.8678, - "falseNegative": 384, - "falsePositive": 75, - "precision": 0.9526, - "predictedPositive": 1581, - "recall": 0.7968, - "total": 2970, - "trueNegative": 1005, - "truePositive": 1506, - "wilsonPrecisionLower95": 0.9409 - }, - "threshold": 0.965 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9157, - "falseNegative": 7, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 38, - "recall": 0.8444, - "total": 1485, - "trueNegative": 1440, - "truePositive": 38, - "wilsonPrecisionLower95": 0.9082 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9556, - "falseNegative": 2, - "falsePositive": 2, - "precision": 0.9556, - "predictedPositive": 45, - "recall": 0.9556, - "total": 1485, - "trueNegative": 1438, - "truePositive": 43, - "wilsonPrecisionLower95": 0.8517 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.9364, - "falseNegative": 9, - "falsePositive": 2, - "precision": 0.9759, - "predictedPositive": 83, - "recall": 0.9, - "total": 2970, - "trueNegative": 2878, - "truePositive": 81, - "wilsonPrecisionLower95": 0.9163 - }, - "threshold": 0.5 - }, - "task": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8593, - "falseNegative": 79, - "falsePositive": 13, - "precision": 0.9558, - "predictedPositive": 294, - "recall": 0.7806, - "total": 1485, - "trueNegative": 1112, - "truePositive": 281, - "wilsonPrecisionLower95": 0.9258 - }, - "threshold": 0.98 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8166, - "falseNegative": 104, - "falsePositive": 11, - "precision": 0.9588, - "predictedPositive": 267, - "recall": 0.7111, - "total": 1485, - "trueNegative": 1114, - "truePositive": 256, - "wilsonPrecisionLower95": 0.9277 - }, - "threshold": 0.96 - } - }, - "metrics": { - "f1": 0.8434, - "falseNegative": 176, - "falsePositive": 26, - "precision": 0.9544, - "predictedPositive": 570, - "recall": 0.7556, - "total": 2970, - "trueNegative": 2224, - "truePositive": 544, - "wilsonPrecisionLower95": 0.934 - }, - "threshold": 0.965 - } - }, - "trainingRecords": 15345, - "validation": { - "intentsMeeting95Precision": 8, - "macroF1": 0.711, - "macroPrecision": 0.9652, - "macroRecall": 0.6071, - "minimumPrecision": 0.9441, - "perIntent": { - "blessing": { - "f1": 0.3333, - "falseNegative": 72, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 18, - "recall": 0.2, - "total": 2970, - "trueNegative": 2880, - "truePositive": 18, - "wilsonPrecisionLower95": 0.8241 - }, - "complaint": { - "f1": 0.9565, - "falseNegative": 19, - "falsePositive": 12, - "precision": 0.966, - "predictedPositive": 353, - "recall": 0.9472, - "total": 2970, - "trueNegative": 2598, - "truePositive": 341, - "wilsonPrecisionLower95": 0.9415 - }, - "confirmationDecision": { - "f1": 0.8371, - "falseNegative": 49, - "falsePositive": 2, - "precision": 0.985, - "predictedPositive": 133, - "recall": 0.7278, - "total": 2970, - "trueNegative": 2788, - "truePositive": 131, - "wilsonPrecisionLower95": 0.9468 - }, - "followUpReminder": { - "f1": 0.5368, - "falseNegative": 225, - "falsePositive": 8, - "precision": 0.9441, - "predictedPositive": 143, - "recall": 0.375, - "total": 2970, - "trueNegative": 2602, - "truePositive": 135, - "wilsonPrecisionLower95": 0.8935 - }, - "invitation": { - "f1": 0.6061, - "falseNegative": 100, - "falsePositive": 4, - "precision": 0.9524, - "predictedPositive": 84, - "recall": 0.4444, - "total": 2970, - "trueNegative": 2786, - "truePositive": 80, - "wilsonPrecisionLower95": 0.8839 - }, - "question": { - "f1": 0.487, - "falseNegative": 424, - "falsePositive": 10, - "precision": 0.9537, - "predictedPositive": 216, - "recall": 0.327, - "total": 2970, - "trueNegative": 2330, - "truePositive": 206, - "wilsonPrecisionLower95": 0.9169 - }, - "replyableMessage": { - "f1": 0.8678, - "falseNegative": 384, - "falsePositive": 75, - "precision": 0.9526, - "predictedPositive": 1581, - "recall": 0.7968, - "total": 2970, - "trueNegative": 1005, - "truePositive": 1506, - "wilsonPrecisionLower95": 0.9409 - }, - "scheduleNegotiation": { - "f1": 0.9364, - "falseNegative": 9, - "falsePositive": 2, - "precision": 0.9759, - "predictedPositive": 83, - "recall": 0.9, - "total": 2970, - "trueNegative": 2878, - "truePositive": 81, - "wilsonPrecisionLower95": 0.9163 - }, - "task": { - "f1": 0.8384, - "falseNegative": 183, - "falsePositive": 24, - "precision": 0.9572, - "predictedPositive": 561, - "recall": 0.7458, - "total": 2970, - "trueNegative": 2226, - "truePositive": 537, - "wilsonPrecisionLower95": 0.9371 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 3e-06, - "augmentation": "numeric", - "char_max": 7, - "char_min": 2, - "external_weight": 0.8700000000000001, - "hard_example_weight": 2.3, - "l1_ratio": 0.15, - "max_char_features": 80000, - "max_word_features": 38000, - "min_df": 2, - "pseudo_weight": 0.39, - "round": 16, - "word_max": 3 - }, - "featureCount": 100252, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 3 - }, - "complaint": { - "negative": 250, - "positive": 11 - }, - "confirmationDecision": { - "negative": 250, - "positive": 25 - }, - "followUpReminder": { - "negative": 250, - "positive": 0 - }, - "invitation": { - "negative": 250, - "positive": 0 - }, - "question": { - "negative": 250, - "positive": 0 - }, - "replyableMessage": { - "negative": 76, - "positive": 250 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 0 - }, - "task": { - "negative": 250, - "positive": 0 - } - }, - "round": 16, - "selectionObjective": 0.8105, - "syntheticRegression": { - "intentsMeeting95Precision": 5, - "macroF1": 0.726, - "macroPrecision": 0.9485, - "macroRecall": 0.6388, - "minimumPrecision": 0.8195, - "perIntent": { - "blessing": { - "f1": 0.125, - "falseNegative": 84, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 6, - "recall": 0.0667, - "total": 2970, - "trueNegative": 2880, - "truePositive": 6, - "wilsonPrecisionLower95": 0.6097 - }, - "complaint": { - "f1": 0.9132, - "falseNegative": 55, - "falsePositive": 3, - "precision": 0.9903, - "predictedPositive": 308, - "recall": 0.8472, - "total": 2970, - "trueNegative": 2607, - "truePositive": 305, - "wilsonPrecisionLower95": 0.9718 - }, - "confirmationDecision": { - "f1": 0.6965, - "falseNegative": 71, - "falsePositive": 24, - "precision": 0.8195, - "predictedPositive": 133, - "recall": 0.6056, - "total": 2970, - "trueNegative": 2766, - "truePositive": 109, - "wilsonPrecisionLower95": 0.7455 - }, - "followUpReminder": { - "f1": 0.7409, - "falseNegative": 147, - "falsePositive": 2, - "precision": 0.9907, - "predictedPositive": 215, - "recall": 0.5917, - "total": 2970, - "trueNegative": 2608, - "truePositive": 213, - "wilsonPrecisionLower95": 0.9667 - }, - "invitation": { - "f1": 0.6264, - "falseNegative": 97, - "falsePositive": 2, - "precision": 0.9765, - "predictedPositive": 85, - "recall": 0.4611, - "total": 2970, - "trueNegative": 2788, - "truePositive": 83, - "wilsonPrecisionLower95": 0.9182 - }, - "question": { - "f1": 0.8457, - "falseNegative": 164, - "falsePositive": 6, - "precision": 0.9873, - "predictedPositive": 472, - "recall": 0.7397, - "total": 2970, - "trueNegative": 2334, - "truePositive": 466, - "wilsonPrecisionLower95": 0.9725 - }, - "replyableMessage": { - "f1": 0.8725, - "falseNegative": 371, - "falsePositive": 73, - "precision": 0.9541, - "predictedPositive": 1592, - "recall": 0.8037, - "total": 2970, - "trueNegative": 1007, - "truePositive": 1519, - "wilsonPrecisionLower95": 0.9427 - }, - "scheduleNegotiation": { - "f1": 0.9061, - "falseNegative": 8, - "falsePositive": 9, - "precision": 0.9011, - "predictedPositive": 91, - "recall": 0.9111, - "total": 2970, - "trueNegative": 2871, - "truePositive": 82, - "wilsonPrecisionLower95": 0.8226 - }, - "task": { - "f1": 0.8081, - "falseNegative": 200, - "falsePositive": 47, - "precision": 0.9171, - "predictedPositive": 567, - "recall": 0.7222, - "total": 2970, - "trueNegative": 2203, - "truePositive": 520, - "wilsonPrecisionLower95": 0.8915 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9371, - "falseNegative": 16, - "falsePositive": 6, - "precision": 0.9647, - "predictedPositive": 170, - "recall": 0.9111, - "total": 1485, - "trueNegative": 1299, - "truePositive": 164, - "wilsonPrecisionLower95": 0.9251 - }, - "threshold": 0.995 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8836, - "falseNegative": 32, - "falsePositive": 7, - "precision": 0.9548, - "predictedPositive": 155, - "recall": 0.8222, - "total": 1485, - "trueNegative": 1298, - "truePositive": 148, - "wilsonPrecisionLower95": 0.9097 - }, - "threshold": 0.99 - } - }, - "metrics": { - "f1": 0.8994, - "falseNegative": 56, - "falsePositive": 12, - "precision": 0.962, - "predictedPositive": 316, - "recall": 0.8444, - "total": 2970, - "trueNegative": 2598, - "truePositive": 304, - "wilsonPrecisionLower95": 0.9348 - }, - "threshold": 0.995 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8625, - "falseNegative": 21, - "falsePositive": 1, - "precision": 0.9857, - "predictedPositive": 70, - "recall": 0.7667, - "total": 1485, - "trueNegative": 1394, - "truePositive": 69, - "wilsonPrecisionLower95": 0.9234 - }, - "threshold": 0.915 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.7415, - "falseNegative": 71, - "falsePositive": 5, - "precision": 0.9561, - "predictedPositive": 114, - "recall": 0.6056, - "total": 2970, - "trueNegative": 2785, - "truePositive": 109, - "wilsonPrecisionLower95": 0.9014 - }, - "threshold": 0.96 - }, - "followUpReminder": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8667, - "falseNegative": 37, - "falsePositive": 7, - "precision": 0.9533, - "predictedPositive": 150, - "recall": 0.7944, - "total": 1485, - "trueNegative": 1298, - "truePositive": 143, - "wilsonPrecisionLower95": 0.9068 - }, - "threshold": 0.96 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6031, - "falseNegative": 101, - "falsePositive": 3, - "precision": 0.9634, - "predictedPositive": 82, - "recall": 0.4389, - "total": 1485, - "trueNegative": 1302, - "truePositive": 79, - "wilsonPrecisionLower95": 0.8979 - }, - "threshold": 0.97 - } - }, - "metrics": { - "f1": 0.7492, - "falseNegative": 139, - "falsePositive": 9, - "precision": 0.9609, - "predictedPositive": 230, - "recall": 0.6139, - "total": 2970, - "trueNegative": 2601, - "truePositive": 221, - "wilsonPrecisionLower95": 0.9273 - }, - "threshold": 0.97 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.3604, - "falseNegative": 70, - "falsePositive": 1, - "precision": 0.9524, - "predictedPositive": 21, - "recall": 0.2222, - "total": 1485, - "trueNegative": 1394, - "truePositive": 20, - "wilsonPrecisionLower95": 0.7733 - }, - "threshold": 0.965 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8273, - "falseNegative": 85, - "falsePositive": 11, - "precision": 0.9544, - "predictedPositive": 241, - "recall": 0.7302, - "total": 1485, - "trueNegative": 1159, - "truePositive": 230, - "wilsonPrecisionLower95": 0.9201 - }, - "threshold": 0.985 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.797, - "falseNegative": 99, - "falsePositive": 11, - "precision": 0.9515, - "predictedPositive": 227, - "recall": 0.6857, - "total": 1485, - "trueNegative": 1159, - "truePositive": 216, - "wilsonPrecisionLower95": 0.9153 - }, - "threshold": 0.835 - } - }, - "metrics": { - "f1": 0.8051, - "falseNegative": 190, - "falsePositive": 23, - "precision": 0.9503, - "predictedPositive": 463, - "recall": 0.6984, - "total": 2970, - "trueNegative": 2317, - "truePositive": 440, - "wilsonPrecisionLower95": 0.9266 - }, - "threshold": 0.94 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9343, - "falseNegative": 77, - "falsePositive": 45, - "precision": 0.9507, - "predictedPositive": 913, - "recall": 0.9185, - "total": 1485, - "trueNegative": 495, - "truePositive": 868, - "wilsonPrecisionLower95": 0.9347 - }, - "threshold": 0.91 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8207, - "falseNegative": 263, - "falsePositive": 35, - "precision": 0.9512, - "predictedPositive": 717, - "recall": 0.7217, - "total": 1485, - "trueNegative": 505, - "truePositive": 682, - "wilsonPrecisionLower95": 0.9329 - }, - "threshold": 0.98 - } - }, - "metrics": { - "f1": 0.8572, - "falseNegative": 419, - "falsePositive": 71, - "precision": 0.954, - "predictedPositive": 1542, - "recall": 0.7783, - "total": 2970, - "trueNegative": 1009, - "truePositive": 1471, - "wilsonPrecisionLower95": 0.9423 - }, - "threshold": 0.98 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9888, - "falseNegative": 1, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 44, - "recall": 0.9778, - "total": 1485, - "trueNegative": 1440, - "truePositive": 44, - "wilsonPrecisionLower95": 0.9197 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.989, - "falseNegative": 0, - "falsePositive": 1, - "precision": 0.9783, - "predictedPositive": 46, - "recall": 1.0, - "total": 1485, - "trueNegative": 1439, - "truePositive": 45, - "wilsonPrecisionLower95": 0.8866 - }, - "threshold": 0.955 - } - }, - "metrics": { - "f1": 0.9778, - "falseNegative": 2, - "falsePositive": 2, - "precision": 0.9778, - "predictedPositive": 90, - "recall": 0.9778, - "total": 2970, - "trueNegative": 2878, - "truePositive": 88, - "wilsonPrecisionLower95": 0.9226 - }, - "threshold": 0.935 - }, - "task": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8836, - "falseNegative": 64, - "falsePositive": 14, - "precision": 0.9548, - "predictedPositive": 310, - "recall": 0.8222, - "total": 1485, - "trueNegative": 1111, - "truePositive": 296, - "wilsonPrecisionLower95": 0.9256 - }, - "threshold": 0.975 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.7896, - "falseNegative": 118, - "falsePositive": 11, - "precision": 0.9565, - "predictedPositive": 253, - "recall": 0.6722, - "total": 1485, - "trueNegative": 1114, - "truePositive": 242, - "wilsonPrecisionLower95": 0.9238 - }, - "threshold": 0.96 - } - }, - "metrics": { - "f1": 0.8316, - "falseNegative": 189, - "falsePositive": 26, - "precision": 0.9533, - "predictedPositive": 557, - "recall": 0.7375, - "total": 2970, - "trueNegative": 2224, - "truePositive": 531, - "wilsonPrecisionLower95": 0.9325 - }, - "threshold": 0.97 - } - }, - "trainingRecords": 16726, - "validation": { - "intentsMeeting95Precision": 9, - "macroF1": 0.7397, - "macroPrecision": 0.9707, - "macroRecall": 0.6388, - "minimumPrecision": 0.9509, - "perIntent": { - "blessing": { - "f1": 0.3636, - "falseNegative": 70, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 20, - "recall": 0.2222, - "total": 2970, - "trueNegative": 2880, - "truePositive": 20, - "wilsonPrecisionLower95": 0.8389 - }, - "complaint": { - "f1": 0.9109, - "falseNegative": 48, - "falsePositive": 13, - "precision": 0.96, - "predictedPositive": 325, - "recall": 0.8667, - "total": 2970, - "trueNegative": 2597, - "truePositive": 312, - "wilsonPrecisionLower95": 0.9328 - }, - "confirmationDecision": { - "f1": 0.6236, - "falseNegative": 98, - "falsePositive": 1, - "precision": 0.988, - "predictedPositive": 83, - "recall": 0.4556, - "total": 2970, - "trueNegative": 2789, - "truePositive": 82, - "wilsonPrecisionLower95": 0.9349 - }, - "followUpReminder": { - "f1": 0.75, - "falseNegative": 138, - "falsePositive": 10, - "precision": 0.9569, - "predictedPositive": 232, - "recall": 0.6167, - "total": 2970, - "trueNegative": 2600, - "truePositive": 222, - "wilsonPrecisionLower95": 0.9225 - }, - "invitation": { - "f1": 0.4917, - "falseNegative": 121, - "falsePositive": 1, - "precision": 0.9833, - "predictedPositive": 60, - "recall": 0.3278, - "total": 2970, - "trueNegative": 2789, - "truePositive": 59, - "wilsonPrecisionLower95": 0.9114 - }, - "question": { - "f1": 0.8124, - "falseNegative": 184, - "falsePositive": 22, - "precision": 0.953, - "predictedPositive": 468, - "recall": 0.7079, - "total": 2970, - "trueNegative": 2318, - "truePositive": 446, - "wilsonPrecisionLower95": 0.9299 - }, - "replyableMessage": { - "f1": 0.8807, - "falseNegative": 340, - "falsePositive": 80, - "precision": 0.9509, - "predictedPositive": 1630, - "recall": 0.8201, - "total": 2970, - "trueNegative": 1000, - "truePositive": 1550, - "wilsonPrecisionLower95": 0.9393 - }, - "scheduleNegotiation": { - "f1": 0.9889, - "falseNegative": 1, - "falsePositive": 1, - "precision": 0.9889, - "predictedPositive": 90, - "recall": 0.9889, - "total": 2970, - "trueNegative": 2879, - "truePositive": 89, - "wilsonPrecisionLower95": 0.9397 - }, - "task": { - "f1": 0.8359, - "falseNegative": 185, - "falsePositive": 25, - "precision": 0.9554, - "predictedPositive": 560, - "recall": 0.7431, - "total": 2970, - "trueNegative": 2225, - "truePositive": 535, - "wilsonPrecisionLower95": 0.9349 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 2e-06, - "augmentation": "numeric", - "char_max": 7, - "char_min": 3, - "external_weight": 0.8900000000000001, - "hard_example_weight": 2.5, - "l1_ratio": 0.2, - "max_char_features": 85000, - "max_word_features": 42000, - "min_df": 1, - "pseudo_weight": 0.405, - "round": 17, - "word_max": 3 - }, - "featureCount": 127000, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 1 - }, - "complaint": { - "negative": 250, - "positive": 4 - }, - "confirmationDecision": { - "negative": 250, - "positive": 23 - }, - "followUpReminder": { - "negative": 250, - "positive": 3 - }, - "invitation": { - "negative": 250, - "positive": 0 - }, - "question": { - "negative": 250, - "positive": 0 - }, - "replyableMessage": { - "negative": 70, - "positive": 250 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 0 - }, - "task": { - "negative": 250, - "positive": 0 - } - }, - "round": 17, - "selectionObjective": 0.6932, - "syntheticRegression": { - "intentsMeeting95Precision": 4, - "macroF1": 0.5086, - "macroPrecision": 0.9202, - "macroRecall": 0.4091, - "minimumPrecision": 0.7329, - "perIntent": { - "blessing": { - "f1": 0.1633, - "falseNegative": 82, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 8, - "recall": 0.0889, - "total": 2970, - "trueNegative": 2880, - "truePositive": 8, - "wilsonPrecisionLower95": 0.6756 - }, - "complaint": { - "f1": 0.914, - "falseNegative": 36, - "falsePositive": 25, - "precision": 0.9284, - "predictedPositive": 349, - "recall": 0.9, - "total": 2970, - "trueNegative": 2585, - "truePositive": 324, - "wilsonPrecisionLower95": 0.8964 - }, - "confirmationDecision": { - "f1": 0.6564, - "falseNegative": 73, - "falsePositive": 39, - "precision": 0.7329, - "predictedPositive": 146, - "recall": 0.5944, - "total": 2970, - "trueNegative": 2751, - "truePositive": 107, - "wilsonPrecisionLower95": 0.6558 - }, - "followUpReminder": { - "f1": 0.4, - "falseNegative": 270, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 90, - "recall": 0.25, - "total": 2970, - "trueNegative": 2610, - "truePositive": 90, - "wilsonPrecisionLower95": 0.9591 - }, - "invitation": { - "f1": 0.5954, - "falseNegative": 102, - "falsePositive": 4, - "precision": 0.9512, - "predictedPositive": 82, - "recall": 0.4333, - "total": 2970, - "trueNegative": 2786, - "truePositive": 78, - "wilsonPrecisionLower95": 0.8812 - }, - "question": { - "f1": 0.4976, - "falseNegative": 419, - "falsePositive": 7, - "precision": 0.9679, - "predictedPositive": 218, - "recall": 0.3349, - "total": 2970, - "trueNegative": 2333, - "truePositive": 211, - "wilsonPrecisionLower95": 0.9352 - }, - "replyableMessage": { - "f1": 0.2549, - "falseNegative": 1612, - "falsePositive": 13, - "precision": 0.9553, - "predictedPositive": 291, - "recall": 0.1471, - "total": 2970, - "trueNegative": 1067, - "truePositive": 278, - "wilsonPrecisionLower95": 0.9251 - }, - "scheduleNegotiation": { - "f1": 0.8333, - "falseNegative": 20, - "falsePositive": 8, - "precision": 0.8974, - "predictedPositive": 78, - "recall": 0.7778, - "total": 2970, - "trueNegative": 2872, - "truePositive": 70, - "wilsonPrecisionLower95": 0.8105 - }, - "task": { - "f1": 0.2629, - "falseNegative": 608, - "falsePositive": 20, - "precision": 0.8485, - "predictedPositive": 132, - "recall": 0.1556, - "total": 2970, - "trueNegative": 2230, - "truePositive": 112, - "wilsonPrecisionLower95": 0.7775 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.989, - "falseNegative": 0, - "falsePositive": 4, - "precision": 0.9783, - "predictedPositive": 184, - "recall": 1.0, - "total": 1485, - "trueNegative": 1301, - "truePositive": 180, - "wilsonPrecisionLower95": 0.9454 - }, - "threshold": 0.995 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8869, - "falseNegative": 31, - "falsePositive": 7, - "precision": 0.9551, - "predictedPositive": 156, - "recall": 0.8278, - "total": 1485, - "trueNegative": 1298, - "truePositive": 149, - "wilsonPrecisionLower95": 0.9103 - }, - "threshold": 0.995 - } - }, - "metrics": { - "f1": 0.944, - "falseNegative": 23, - "falsePositive": 17, - "precision": 0.952, - "predictedPositive": 354, - "recall": 0.9361, - "total": 2970, - "trueNegative": 2593, - "truePositive": 337, - "wilsonPrecisionLower95": 0.9244 - }, - "threshold": 0.975 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8447, - "falseNegative": 22, - "falsePositive": 3, - "precision": 0.9577, - "predictedPositive": 71, - "recall": 0.7556, - "total": 1485, - "trueNegative": 1392, - "truePositive": 68, - "wilsonPrecisionLower95": 0.883 - }, - "threshold": 0.825 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.7723, - "falseNegative": 63, - "falsePositive": 6, - "precision": 0.9512, - "predictedPositive": 123, - "recall": 0.65, - "total": 2970, - "trueNegative": 2784, - "truePositive": 117, - "wilsonPrecisionLower95": 0.8977 - }, - "threshold": 0.965 - }, - "followUpReminder": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.6625, - "falseNegative": 155, - "falsePositive": 8, - "precision": 0.9524, - "predictedPositive": 168, - "recall": 0.5079, - "total": 1485, - "trueNegative": 1162, - "truePositive": 160, - "wilsonPrecisionLower95": 0.9089 - }, - "threshold": 0.97 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "replyableMessage": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9655, - "falseNegative": 3, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 42, - "recall": 0.9333, - "total": 1485, - "trueNegative": 1440, - "truePositive": 42, - "wilsonPrecisionLower95": 0.9162 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9425, - "falseNegative": 4, - "falsePositive": 1, - "precision": 0.9762, - "predictedPositive": 42, - "recall": 0.9111, - "total": 1485, - "trueNegative": 1439, - "truePositive": 41, - "wilsonPrecisionLower95": 0.8768 - }, - "threshold": 0.73 - } - }, - "metrics": { - "f1": 0.9486, - "falseNegative": 7, - "falsePositive": 2, - "precision": 0.9765, - "predictedPositive": 85, - "recall": 0.9222, - "total": 2970, - "trueNegative": 2878, - "truePositive": 83, - "wilsonPrecisionLower95": 0.9182 - }, - "threshold": 0.5 - }, - "task": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "trainingRecords": 16820, - "validation": { - "intentsMeeting95Precision": 8, - "macroF1": 0.5631, - "macroPrecision": 0.9693, - "macroRecall": 0.4421, - "minimumPrecision": 0.9474, - "perIntent": { - "blessing": { - "f1": 0.3019, - "falseNegative": 74, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 16, - "recall": 0.1778, - "total": 2970, - "trueNegative": 2880, - "truePositive": 16, - "wilsonPrecisionLower95": 0.8064 - }, - "complaint": { - "f1": 0.94, - "falseNegative": 31, - "falsePositive": 11, - "precision": 0.9676, - "predictedPositive": 340, - "recall": 0.9139, - "total": 2970, - "trueNegative": 2599, - "truePositive": 329, - "wilsonPrecisionLower95": 0.943 - }, - "confirmationDecision": { - "f1": 0.6545, - "falseNegative": 90, - "falsePositive": 5, - "precision": 0.9474, - "predictedPositive": 95, - "recall": 0.5, - "total": 2970, - "trueNegative": 2785, - "truePositive": 90, - "wilsonPrecisionLower95": 0.8827 - }, - "followUpReminder": { - "f1": 0.5282, - "falseNegative": 229, - "falsePositive": 5, - "precision": 0.9632, - "predictedPositive": 136, - "recall": 0.3639, - "total": 2970, - "trueNegative": 2605, - "truePositive": 131, - "wilsonPrecisionLower95": 0.9168 - }, - "invitation": { - "f1": 0.4378, - "falseNegative": 129, - "falsePositive": 2, - "precision": 0.9623, - "predictedPositive": 53, - "recall": 0.2833, - "total": 2970, - "trueNegative": 2788, - "truePositive": 51, - "wilsonPrecisionLower95": 0.8725 - }, - "question": { - "f1": 0.5428, - "falseNegative": 392, - "falsePositive": 9, - "precision": 0.9636, - "predictedPositive": 247, - "recall": 0.3778, - "total": 2970, - "trueNegative": 2331, - "truePositive": 238, - "wilsonPrecisionLower95": 0.9322 - }, - "replyableMessage": { - "f1": 0.2757, - "falseNegative": 1586, - "falsePositive": 11, - "precision": 0.9651, - "predictedPositive": 315, - "recall": 0.1608, - "total": 2970, - "trueNegative": 1069, - "truePositive": 304, - "wilsonPrecisionLower95": 0.9386 - }, - "scheduleNegotiation": { - "f1": 0.954, - "falseNegative": 7, - "falsePositive": 1, - "precision": 0.9881, - "predictedPositive": 84, - "recall": 0.9222, - "total": 2970, - "trueNegative": 2879, - "truePositive": 83, - "wilsonPrecisionLower95": 0.9356 - }, - "task": { - "f1": 0.4332, - "falseNegative": 519, - "falsePositive": 7, - "precision": 0.9663, - "predictedPositive": 208, - "recall": 0.2792, - "total": 2970, - "trueNegative": 2243, - "truePositive": 201, - "wilsonPrecisionLower95": 0.9322 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 1.5e-06, - "augmentation": "numeric", - "char_max": 8, - "char_min": 2, - "external_weight": 0.91, - "hard_example_weight": 2.5, - "l1_ratio": 0.2, - "max_char_features": 90000, - "max_word_features": 45000, - "min_df": 1, - "pseudo_weight": 0.42000000000000004, - "round": 18, - "word_max": 3 - }, - "featureCount": 135000, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 1 - }, - "complaint": { - "negative": 250, - "positive": 8 - }, - "confirmationDecision": { - "negative": 250, - "positive": 27 - }, - "followUpReminder": { - "negative": 250, - "positive": 0 - }, - "invitation": { - "negative": 250, - "positive": 0 - }, - "question": { - "negative": 250, - "positive": 0 - }, - "replyableMessage": { - "negative": 90, - "positive": 32 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 0 - }, - "task": { - "negative": 250, - "positive": 0 - } - }, - "round": 18, - "selectionObjective": 0.7725, - "syntheticRegression": { - "intentsMeeting95Precision": 4, - "macroF1": 0.6489, - "macroPrecision": 0.9452, - "macroRecall": 0.5429, - "minimumPrecision": 0.8205, - "perIntent": { - "blessing": { - "f1": 0.2178, - "falseNegative": 79, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 11, - "recall": 0.1222, - "total": 2970, - "trueNegative": 2880, - "truePositive": 11, - "wilsonPrecisionLower95": 0.7412 - }, - "complaint": { - "f1": 0.8661, - "falseNegative": 85, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 275, - "recall": 0.7639, - "total": 2970, - "trueNegative": 2610, - "truePositive": 275, - "wilsonPrecisionLower95": 0.9862 - }, - "confirmationDecision": { - "f1": 0.6465, - "falseNegative": 84, - "falsePositive": 21, - "precision": 0.8205, - "predictedPositive": 117, - "recall": 0.5333, - "total": 2970, - "trueNegative": 2769, - "truePositive": 96, - "wilsonPrecisionLower95": 0.7411 - }, - "followUpReminder": { - "f1": 0.5851, - "falseNegative": 207, - "falsePositive": 10, - "precision": 0.9387, - "predictedPositive": 163, - "recall": 0.425, - "total": 2970, - "trueNegative": 2600, - "truePositive": 153, - "wilsonPrecisionLower95": 0.8908 - }, - "invitation": { - "f1": 0.626, - "falseNegative": 98, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 82, - "recall": 0.4556, - "total": 2970, - "trueNegative": 2790, - "truePositive": 82, - "wilsonPrecisionLower95": 0.9552 - }, - "question": { - "f1": 0.7311, - "falseNegative": 267, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 363, - "recall": 0.5762, - "total": 2970, - "trueNegative": 2340, - "truePositive": 363, - "wilsonPrecisionLower95": 0.9895 - }, - "replyableMessage": { - "f1": 0.8939, - "falseNegative": 273, - "falsePositive": 111, - "precision": 0.9358, - "predictedPositive": 1728, - "recall": 0.8556, - "total": 2970, - "trueNegative": 969, - "truePositive": 1617, - "wilsonPrecisionLower95": 0.9232 - }, - "scheduleNegotiation": { - "f1": 0.9341, - "falseNegative": 5, - "falsePositive": 7, - "precision": 0.9239, - "predictedPositive": 92, - "recall": 0.9444, - "total": 2970, - "trueNegative": 2873, - "truePositive": 85, - "wilsonPrecisionLower95": 0.8512 - }, - "task": { - "f1": 0.3393, - "falseNegative": 569, - "falsePositive": 19, - "precision": 0.8882, - "predictedPositive": 170, - "recall": 0.2097, - "total": 2970, - "trueNegative": 2231, - "truePositive": 151, - "wilsonPrecisionLower95": 0.832 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "confirmationDecision": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9486, - "falseNegative": 7, - "falsePositive": 2, - "precision": 0.9765, - "predictedPositive": 85, - "recall": 0.9222, - "total": 1485, - "trueNegative": 1393, - "truePositive": 83, - "wilsonPrecisionLower95": 0.9182 - }, - "threshold": 0.9 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.8447, - "falseNegative": 44, - "falsePositive": 6, - "precision": 0.9577, - "predictedPositive": 142, - "recall": 0.7556, - "total": 2970, - "trueNegative": 2784, - "truePositive": 136, - "wilsonPrecisionLower95": 0.9109 - }, - "threshold": 0.955 - }, - "followUpReminder": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.8822, - "falseNegative": 34, - "falsePositive": 5, - "precision": 0.9669, - "predictedPositive": 151, - "recall": 0.8111, - "total": 1485, - "trueNegative": 1300, - "truePositive": 146, - "wilsonPrecisionLower95": 0.9248 - }, - "threshold": 0.98 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.7641, - "falseNegative": 130, - "falsePositive": 12, - "precision": 0.9504, - "predictedPositive": 242, - "recall": 0.6389, - "total": 2970, - "trueNegative": 2598, - "truePositive": 230, - "wilsonPrecisionLower95": 0.9153 - }, - "threshold": 0.99 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8103, - "falseNegative": 95, - "falsePositive": 8, - "precision": 0.9649, - "predictedPositive": 228, - "recall": 0.6984, - "total": 1485, - "trueNegative": 1162, - "truePositive": 220, - "wilsonPrecisionLower95": 0.9323 - }, - "threshold": 0.915 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9575, - "falseNegative": 33, - "falsePositive": 48, - "precision": 0.95, - "predictedPositive": 960, - "recall": 0.9651, - "total": 1485, - "trueNegative": 492, - "truePositive": 912, - "wilsonPrecisionLower95": 0.9343 - }, - "threshold": 0.845 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8361, - "falseNegative": 241, - "falsePositive": 35, - "precision": 0.9526, - "predictedPositive": 739, - "recall": 0.745, - "total": 1485, - "trueNegative": 505, - "truePositive": 704, - "wilsonPrecisionLower95": 0.9348 - }, - "threshold": 0.98 - } - }, - "metrics": { - "f1": 0.901, - "falseNegative": 274, - "falsePositive": 81, - "precision": 0.9523, - "predictedPositive": 1697, - "recall": 0.855, - "total": 2970, - "trueNegative": 999, - "truePositive": 1616, - "wilsonPrecisionLower95": 0.9411 - }, - "threshold": 0.975 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9655, - "falseNegative": 3, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 42, - "recall": 0.9333, - "total": 1485, - "trueNegative": 1440, - "truePositive": 42, - "wilsonPrecisionLower95": 0.9162 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 45, - "recall": 1.0, - "total": 1485, - "trueNegative": 1440, - "truePositive": 45, - "wilsonPrecisionLower95": 0.9213 - }, - "threshold": 0.5 - } - }, - "metrics": { - "f1": 0.9831, - "falseNegative": 3, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 87, - "recall": 0.9667, - "total": 2970, - "trueNegative": 2880, - "truePositive": 87, - "wilsonPrecisionLower95": 0.9577 - }, - "threshold": 0.5 - }, - "task": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "trainingRecords": 16841, - "validation": { - "intentsMeeting95Precision": 8, - "macroF1": 0.6817, - "macroPrecision": 0.9748, - "macroRecall": 0.5634, - "minimumPrecision": 0.931, - "perIntent": { - "blessing": { - "f1": 0.3486, - "falseNegative": 71, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 19, - "recall": 0.2111, - "total": 2970, - "trueNegative": 2880, - "truePositive": 19, - "wilsonPrecisionLower95": 0.8318 - }, - "complaint": { - "f1": 0.88, - "falseNegative": 74, - "falsePositive": 4, - "precision": 0.9862, - "predictedPositive": 290, - "recall": 0.7944, - "total": 2970, - "trueNegative": 2606, - "truePositive": 286, - "wilsonPrecisionLower95": 0.9651 - }, - "confirmationDecision": { - "f1": 0.7183, - "falseNegative": 78, - "falsePositive": 2, - "precision": 0.9808, - "predictedPositive": 104, - "recall": 0.5667, - "total": 2970, - "trueNegative": 2788, - "truePositive": 102, - "wilsonPrecisionLower95": 0.9326 - }, - "followUpReminder": { - "f1": 0.6345, - "falseNegative": 189, - "falsePositive": 8, - "precision": 0.9553, - "predictedPositive": 179, - "recall": 0.475, - "total": 2970, - "trueNegative": 2602, - "truePositive": 171, - "wilsonPrecisionLower95": 0.9143 - }, - "invitation": { - "f1": 0.4538, - "falseNegative": 126, - "falsePositive": 4, - "precision": 0.931, - "predictedPositive": 58, - "recall": 0.3, - "total": 2970, - "trueNegative": 2786, - "truePositive": 54, - "wilsonPrecisionLower95": 0.8357 - }, - "question": { - "f1": 0.7187, - "falseNegative": 271, - "falsePositive": 10, - "precision": 0.9729, - "predictedPositive": 369, - "recall": 0.5698, - "total": 2970, - "trueNegative": 2330, - "truePositive": 359, - "wilsonPrecisionLower95": 0.9508 - }, - "replyableMessage": { - "f1": 0.9005, - "falseNegative": 274, - "falsePositive": 83, - "precision": 0.9511, - "predictedPositive": 1699, - "recall": 0.855, - "total": 2970, - "trueNegative": 997, - "truePositive": 1616, - "wilsonPrecisionLower95": 0.9398 - }, - "scheduleNegotiation": { - "f1": 0.9831, - "falseNegative": 3, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 87, - "recall": 0.9667, - "total": 2970, - "trueNegative": 2880, - "truePositive": 87, - "wilsonPrecisionLower95": 0.9577 - }, - "task": { - "f1": 0.4979, - "falseNegative": 481, - "falsePositive": 1, - "precision": 0.9958, - "predictedPositive": 240, - "recall": 0.3319, - "total": 2970, - "trueNegative": 2249, - "truePositive": 239, - "wilsonPrecisionLower95": 0.9768 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 1.2e-06, - "augmentation": "numeric", - "char_max": 8, - "char_min": 3, - "external_weight": 0.93, - "hard_example_weight": 2.7, - "l1_ratio": 0.25, - "max_char_features": 95000, - "max_word_features": 45000, - "min_df": 1, - "pseudo_weight": 0.43499999999999994, - "round": 19, - "word_max": 3 - }, - "featureCount": 140000, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 1 - }, - "complaint": { - "negative": 250, - "positive": 2 - }, - "confirmationDecision": { - "negative": 250, - "positive": 36 - }, - "followUpReminder": { - "negative": 250, - "positive": 0 - }, - "invitation": { - "negative": 250, - "positive": 0 - }, - "question": { - "negative": 250, - "positive": 4 - }, - "replyableMessage": { - "negative": 99, - "positive": 32 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 0 - }, - "task": { - "negative": 250, - "positive": 0 - } - }, - "round": 19, - "selectionObjective": 0.7336, - "syntheticRegression": { - "intentsMeeting95Precision": 6, - "macroF1": 0.6005, - "macroPrecision": 0.9664, - "macroRecall": 0.4678, - "minimumPrecision": 0.887, - "perIntent": { - "blessing": { - "f1": 0.2, - "falseNegative": 80, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 10, - "recall": 0.1111, - "total": 2970, - "trueNegative": 2880, - "truePositive": 10, - "wilsonPrecisionLower95": 0.7225 - }, - "complaint": { - "f1": 0.9205, - "falseNegative": 53, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 307, - "recall": 0.8528, - "total": 2970, - "trueNegative": 2610, - "truePositive": 307, - "wilsonPrecisionLower95": 0.9876 - }, - "confirmationDecision": { - "f1": 0.6915, - "falseNegative": 78, - "falsePositive": 13, - "precision": 0.887, - "predictedPositive": 115, - "recall": 0.5667, - "total": 2970, - "trueNegative": 2777, - "truePositive": 102, - "wilsonPrecisionLower95": 0.8162 - }, - "followUpReminder": { - "f1": 0.4778, - "falseNegative": 247, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 113, - "recall": 0.3139, - "total": 2970, - "trueNegative": 2610, - "truePositive": 113, - "wilsonPrecisionLower95": 0.9671 - }, - "invitation": { - "f1": 0.7092, - "falseNegative": 80, - "falsePositive": 2, - "precision": 0.9804, - "predictedPositive": 102, - "recall": 0.5556, - "total": 2970, - "trueNegative": 2788, - "truePositive": 100, - "wilsonPrecisionLower95": 0.9313 - }, - "question": { - "f1": 0.4703, - "falseNegative": 436, - "falsePositive": 1, - "precision": 0.9949, - "predictedPositive": 195, - "recall": 0.3079, - "total": 2970, - "trueNegative": 2339, - "truePositive": 194, - "wilsonPrecisionLower95": 0.9715 - }, - "replyableMessage": { - "f1": 0.6399, - "falseNegative": 957, - "falsePositive": 93, - "precision": 0.9094, - "predictedPositive": 1026, - "recall": 0.4937, - "total": 2970, - "trueNegative": 987, - "truePositive": 933, - "wilsonPrecisionLower95": 0.8902 - }, - "scheduleNegotiation": { - "f1": 0.8079, - "falseNegative": 29, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 61, - "recall": 0.6778, - "total": 2970, - "trueNegative": 2880, - "truePositive": 61, - "wilsonPrecisionLower95": 0.9408 - }, - "task": { - "f1": 0.4872, - "falseNegative": 482, - "falsePositive": 19, - "precision": 0.9261, - "predictedPositive": 257, - "recall": 0.3306, - "total": 2970, - "trueNegative": 2231, - "truePositive": 238, - "wilsonPrecisionLower95": 0.8874 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "confirmationDecision": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "followUpReminder": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "replyableMessage": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 0.9309, - "falseNegative": 83, - "falsePositive": 45, - "precision": 0.9504, - "predictedPositive": 907, - "recall": 0.9122, - "total": 1485, - "trueNegative": 495, - "truePositive": 862, - "wilsonPrecisionLower95": 0.9343 - }, - "threshold": 0.98 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 45, - "recall": 1.0, - "total": 1485, - "trueNegative": 1440, - "truePositive": 45, - "wilsonPrecisionLower95": 0.9213 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.9195, - "falseNegative": 5, - "falsePositive": 2, - "precision": 0.9524, - "predictedPositive": 42, - "recall": 0.8889, - "total": 1485, - "trueNegative": 1438, - "truePositive": 40, - "wilsonPrecisionLower95": 0.8421 - }, - "threshold": 0.945 - } - }, - "metrics": { - "f1": 0.9667, - "falseNegative": 3, - "falsePositive": 3, - "precision": 0.9667, - "predictedPositive": 90, - "recall": 0.9667, - "total": 2970, - "trueNegative": 2877, - "truePositive": 87, - "wilsonPrecisionLower95": 0.9065 - }, - "threshold": 0.83 - }, - "task": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "trainingRecords": 16801, - "validation": { - "intentsMeeting95Precision": 6, - "macroF1": 0.6316, - "macroPrecision": 0.9616, - "macroRecall": 0.4969, - "minimumPrecision": 0.9242, - "perIntent": { - "blessing": { - "f1": 0.4483, - "falseNegative": 64, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 26, - "recall": 0.2889, - "total": 2970, - "trueNegative": 2880, - "truePositive": 26, - "wilsonPrecisionLower95": 0.8713 - }, - "complaint": { - "f1": 0.8953, - "falseNegative": 65, - "falsePositive": 4, - "precision": 0.9866, - "predictedPositive": 299, - "recall": 0.8194, - "total": 2970, - "trueNegative": 2606, - "truePositive": 295, - "wilsonPrecisionLower95": 0.9661 - }, - "confirmationDecision": { - "f1": 0.5891, - "falseNegative": 104, - "falsePositive": 2, - "precision": 0.9744, - "predictedPositive": 78, - "recall": 0.4222, - "total": 2970, - "trueNegative": 2788, - "truePositive": 76, - "wilsonPrecisionLower95": 0.9112 - }, - "followUpReminder": { - "f1": 0.528, - "falseNegative": 228, - "falsePositive": 8, - "precision": 0.9429, - "predictedPositive": 140, - "recall": 0.3667, - "total": 2970, - "trueNegative": 2602, - "truePositive": 132, - "wilsonPrecisionLower95": 0.8913 - }, - "invitation": { - "f1": 0.6015, - "falseNegative": 100, - "falsePositive": 6, - "precision": 0.9302, - "predictedPositive": 86, - "recall": 0.4444, - "total": 2970, - "trueNegative": 2784, - "truePositive": 80, - "wilsonPrecisionLower95": 0.856 - }, - "question": { - "f1": 0.442, - "falseNegative": 447, - "falsePositive": 15, - "precision": 0.9242, - "predictedPositive": 198, - "recall": 0.2905, - "total": 2970, - "trueNegative": 2325, - "truePositive": 183, - "wilsonPrecisionLower95": 0.8788 - }, - "replyableMessage": { - "f1": 0.6347, - "falseNegative": 990, - "falsePositive": 46, - "precision": 0.9514, - "predictedPositive": 946, - "recall": 0.4762, - "total": 2970, - "trueNegative": 1034, - "truePositive": 900, - "wilsonPrecisionLower95": 0.9357 - }, - "scheduleNegotiation": { - "f1": 0.9605, - "falseNegative": 5, - "falsePositive": 2, - "precision": 0.977, - "predictedPositive": 87, - "recall": 0.9444, - "total": 2970, - "trueNegative": 2878, - "truePositive": 85, - "wilsonPrecisionLower95": 0.92 - }, - "task": { - "f1": 0.5853, - "falseNegative": 418, - "falsePositive": 10, - "precision": 0.9679, - "predictedPositive": 312, - "recall": 0.4194, - "total": 2970, - "trueNegative": 2240, - "truePositive": 302, - "wilsonPrecisionLower95": 0.942 - } - }, - "recordCount": 2970 - } - }, - { - "configuration": { - "alpha": 1e-06, - "augmentation": "numeric", - "char_max": 8, - "char_min": 2, - "external_weight": 0.9500000000000001, - "hard_example_weight": 3.0, - "l1_ratio": 0.25, - "max_char_features": 100000, - "max_word_features": 50000, - "min_df": 1, - "pseudo_weight": 0.44999999999999996, - "round": 20, - "word_max": 3 - }, - "featureCount": 150000, - "pseudoLabelCounts": { - "blessing": { - "negative": 250, - "positive": 1 - }, - "complaint": { - "negative": 250, - "positive": 2 - }, - "confirmationDecision": { - "negative": 250, - "positive": 25 - }, - "followUpReminder": { - "negative": 250, - "positive": 0 - }, - "invitation": { - "negative": 250, - "positive": 1 - }, - "question": { - "negative": 250, - "positive": 2 - }, - "replyableMessage": { - "negative": 104, - "positive": 32 - }, - "scheduleNegotiation": { - "negative": 250, - "positive": 1 - }, - "task": { - "negative": 250, - "positive": 0 - } - }, - "round": 20, - "selectionObjective": 0.7724, - "syntheticRegression": { - "intentsMeeting95Precision": 7, - "macroF1": 0.6637, - "macroPrecision": 0.9685, - "macroRecall": 0.5421, - "minimumPrecision": 0.9094, - "perIntent": { - "blessing": { - "f1": 0.2353, - "falseNegative": 78, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 12, - "recall": 0.1333, - "total": 2970, - "trueNegative": 2880, - "truePositive": 12, - "wilsonPrecisionLower95": 0.7575 - }, - "complaint": { - "f1": 0.8274, - "falseNegative": 106, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 254, - "recall": 0.7056, - "total": 2970, - "trueNegative": 2610, - "truePositive": 254, - "wilsonPrecisionLower95": 0.9851 - }, - "confirmationDecision": { - "f1": 0.8261, - "falseNegative": 47, - "falsePositive": 9, - "precision": 0.9366, - "predictedPositive": 142, - "recall": 0.7389, - "total": 2970, - "trueNegative": 2781, - "truePositive": 133, - "wilsonPrecisionLower95": 0.8839 - }, - "followUpReminder": { - "f1": 0.5155, - "falseNegative": 235, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 125, - "recall": 0.3472, - "total": 2970, - "trueNegative": 2610, - "truePositive": 125, - "wilsonPrecisionLower95": 0.9702 - }, - "invitation": { - "f1": 0.7241, - "falseNegative": 75, - "falsePositive": 5, - "precision": 0.9545, - "predictedPositive": 110, - "recall": 0.5833, - "total": 2970, - "trueNegative": 2785, - "truePositive": 105, - "wilsonPrecisionLower95": 0.898 - }, - "question": { - "f1": 0.5534, - "falseNegative": 389, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 241, - "recall": 0.3825, - "total": 2970, - "trueNegative": 2340, - "truePositive": 241, - "wilsonPrecisionLower95": 0.9843 - }, - "replyableMessage": { - "f1": 0.7688, - "falseNegative": 671, - "falsePositive": 62, - "precision": 0.9516, - "predictedPositive": 1281, - "recall": 0.645, - "total": 2970, - "trueNegative": 1018, - "truePositive": 1219, - "wilsonPrecisionLower95": 0.9384 - }, - "scheduleNegotiation": { - "f1": 0.9371, - "falseNegative": 8, - "falsePositive": 3, - "precision": 0.9647, - "predictedPositive": 85, - "recall": 0.9111, - "total": 2970, - "trueNegative": 2877, - "truePositive": 82, - "wilsonPrecisionLower95": 0.9013 - }, - "task": { - "f1": 0.5857, - "falseNegative": 409, - "falsePositive": 31, - "precision": 0.9094, - "predictedPositive": 342, - "recall": 0.4319, - "total": 2970, - "trueNegative": 2219, - "truePositive": 311, - "wilsonPrecisionLower95": 0.8742 - } - }, - "recordCount": 2970 - }, - "thresholds": { - "blessing": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "complaint": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "confirmationDecision": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.5954, - "falseNegative": 51, - "falsePositive": 2, - "precision": 0.9512, - "predictedPositive": 41, - "recall": 0.4333, - "total": 1485, - "trueNegative": 1393, - "truePositive": 39, - "wilsonPrecisionLower95": 0.8386 - }, - "threshold": 0.995 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "followUpReminder": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "invitation": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "question": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "replyableMessage": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.8117, - "falseNegative": 279, - "falsePositive": 30, - "precision": 0.9569, - "predictedPositive": 696, - "recall": 0.7048, - "total": 1485, - "trueNegative": 510, - "truePositive": 666, - "wilsonPrecisionLower95": 0.9391 - }, - "threshold": 0.995 - } - }, - "metrics": { - "f1": 0.8736, - "falseNegative": 363, - "falsePositive": 79, - "precision": 0.9508, - "predictedPositive": 1606, - "recall": 0.8079, - "total": 2970, - "trueNegative": 1001, - "truePositive": 1527, - "wilsonPrecisionLower95": 0.9391 - }, - "threshold": 0.995 - }, - "scheduleNegotiation": { - "abstained": false, - "byLanguage": { - "en": { - "abstained": false, - "metrics": { - "f1": 1.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 45, - "recall": 1.0, - "total": 1485, - "trueNegative": 1440, - "truePositive": 45, - "wilsonPrecisionLower95": 0.9213 - }, - "threshold": 0.5 - }, - "zh-Hans": { - "abstained": false, - "metrics": { - "f1": 0.967, - "falseNegative": 1, - "falsePositive": 2, - "precision": 0.9565, - "predictedPositive": 46, - "recall": 0.9778, - "total": 1485, - "trueNegative": 1438, - "truePositive": 44, - "wilsonPrecisionLower95": 0.8547 - }, - "threshold": 0.83 - } - }, - "metrics": { - "f1": 0.9834, - "falseNegative": 1, - "falsePositive": 2, - "precision": 0.978, - "predictedPositive": 91, - "recall": 0.9889, - "total": 2970, - "trueNegative": 2878, - "truePositive": 89, - "wilsonPrecisionLower95": 0.9234 - }, - "threshold": 0.83 - }, - "task": { - "abstained": true, - "byLanguage": { - "en": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - }, - "zh-Hans": { - "abstained": true, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "metrics": { - "f1": 0.0, - "falseNegative": 0, - "falsePositive": 0, - "precision": 0.0, - "predictedPositive": 0, - "recall": 0.0, - "total": 0, - "trueNegative": 0, - "truePositive": 0, - "wilsonPrecisionLower95": 0 - }, - "threshold": 1.0 - } - }, - "trainingRecords": 16755, - "validation": { - "intentsMeeting95Precision": 8, - "macroF1": 0.6867, - "macroPrecision": 0.9687, - "macroRecall": 0.5569, - "minimumPrecision": 0.9038, - "perIntent": { - "blessing": { - "f1": 0.3929, - "falseNegative": 68, - "falsePositive": 0, - "precision": 1.0, - "predictedPositive": 22, - "recall": 0.2444, - "total": 2970, - "trueNegative": 2880, - "truePositive": 22, - "wilsonPrecisionLower95": 0.8513 - }, - "complaint": { - "f1": 0.8066, - "falseNegative": 114, - "falsePositive": 4, - "precision": 0.984, - "predictedPositive": 250, - "recall": 0.6833, - "total": 2970, - "trueNegative": 2606, - "truePositive": 246, - "wilsonPrecisionLower95": 0.9596 - }, - "confirmationDecision": { - "f1": 0.6618, - "falseNegative": 90, - "falsePositive": 2, - "precision": 0.9783, - "predictedPositive": 92, - "recall": 0.5, - "total": 2970, - "trueNegative": 2788, - "truePositive": 90, - "wilsonPrecisionLower95": 0.9242 - }, - "followUpReminder": { - "f1": 0.5977, - "falseNegative": 204, - "falsePositive": 6, - "precision": 0.963, - "predictedPositive": 162, - "recall": 0.4333, - "total": 2970, - "trueNegative": 2604, - "truePositive": 156, - "wilsonPrecisionLower95": 0.9216 - }, - "invitation": { - "f1": 0.662, - "falseNegative": 86, - "falsePositive": 10, - "precision": 0.9038, - "predictedPositive": 104, - "recall": 0.5222, - "total": 2970, - "trueNegative": 2780, - "truePositive": 94, - "wilsonPrecisionLower95": 0.832 - }, - "question": { - "f1": 0.5643, - "falseNegative": 380, - "falsePositive": 6, - "precision": 0.9766, - "predictedPositive": 256, - "recall": 0.3968, - "total": 2970, - "trueNegative": 2334, - "truePositive": 250, - "wilsonPrecisionLower95": 0.9498 - }, - "replyableMessage": { - "f1": 0.7934, - "falseNegative": 621, - "falsePositive": 40, - "precision": 0.9694, - "predictedPositive": 1309, - "recall": 0.6714, - "total": 2970, - "trueNegative": 1040, - "truePositive": 1269, - "wilsonPrecisionLower95": 0.9587 - }, - "scheduleNegotiation": { - "f1": 0.9834, - "falseNegative": 1, - "falsePositive": 2, - "precision": 0.978, - "predictedPositive": 91, - "recall": 0.9889, - "total": 2970, - "trueNegative": 2878, - "truePositive": 89, - "wilsonPrecisionLower95": 0.9234 - }, - "task": { - "f1": 0.7184, - "falseNegative": 308, - "falsePositive": 15, - "precision": 0.9649, - "predictedPositive": 427, - "recall": 0.5722, - "total": 2970, - "trueNegative": 2235, - "truePositive": 412, - "wilsonPrecisionLower95": 0.9429 - } - }, - "recordCount": 2970 - } - } -] diff --git a/ModelTraining/ClipboardSemantics/README.md b/ModelTraining/ClipboardSemantics/README.md index 9f2469b..2cd5a71 100644 --- a/ModelTraining/ClipboardSemantics/README.md +++ b/ModelTraining/ClipboardSemantics/README.md @@ -153,9 +153,10 @@ The 2026-08-27 study completed the requested 20 rounds and two additional - The current deployed reference remains stronger: random-holdout macro F1 `0.7669` and research-only comprehensive macro F1 `0.2783`. -All three phases failed the release gate, so no model was promoted. Detailed -rounds, final evaluations, source breakdowns, and macOS replay instructions are -stored under `IterativeResearch/`. +All three phases failed the release gate, so no model was promoted. The compact +study summary and macOS replay instructions are stored under +`IterativeResearch/`; detailed round and evaluation JSON is reproducible and +gitignored. This harness is deliberately a Linux surrogate. It cannot emit the `NLModel`-compatible Create ML artifacts used by the keyboard extension.