diff --git a/CHANGELOG.md b/CHANGELOG.md index 43f92f3..96abd58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - **Clipboard intent precision**: specialize the local task and complaint models, add conservative language thresholds, and suppress complaint-only text from task routing unless it contains an explicit assignment. / **剪贴板意图精度**：专项优化本地任务与投诉模型，加入保守的分语言阈值，并在没有明确委派时阻止纯投诉文本进入任务路由。 - **Precision-first semantic routing**: add traceable multi-model consensus labels, local joint intent verifiers, confidence-margin abstention, layered source-isolated evaluation, and privacy-safe shadow disagreement counters while retaining current models until every acceptance gate passes. / **精度优先语义路由**：新增可追溯多模型共识银标、本地联合意图验证器、置信度与差值拒识、按来源隔离的分层评估及隐私安全的影子分歧计数；所有验收门槛通过前继续保留当前模型。 +- **Iterative semantic research**: add reproducible twenty-round weakly supervised model selection, high-confidence self-training consensus, per-intent fine-tuning, explicit-evidence experiments, frozen-overlap gates, and macOS replay manifests; retain deployed models when every candidate underperforms release baselines. / **迭代语义研究**：新增可复现的二十轮弱监督模型选择、高置信自训练共识、分意图微调、显式证据实验、冻结集重叠门禁及 macOS 重放清单；所有候选低于发布基线时继续保留现有部署模型。 - **Card depth**: add a faint directional long shadow to card surfaces throughout the iOS app in both light and dark appearances. / **卡片层次**：为 iOS App 内的卡片界面统一加入淡雅的定向长投影，并适配浅色与深色外观。 - **Neutral dialog actions**: use adaptive black or white styling for system alerts, confirmation dialogs, and modal actions instead of inheriting the green brand accent. / **中性弹窗操作色**：系统警告、确认弹窗及模态操作改用自适应黑色或白色，不再继承绿色品牌强调色。 - **Monthly usage calendar**: replace the seven-day Home and Mac Dashboard chart with a full current-month calendar whose date-circle opacity reflects each day's dictation characters, placing the Home calendar between its four metrics and Personal Dictionary. / **月度用量日历**：以完整的当月日历替换首页与 Mac 总览的近七天图表，并通过日期圆形背景透明度表示每天的听写字数；首页日历位于四项统计指标与个性词库之间。 diff --git a/ModelTraining/ClipboardSemantics/IterativeResearch/EvidenceGateFineTune/final-report.json b/ModelTraining/ClipboardSemantics/IterativeResearch/EvidenceGateFineTune/final-report.json new file mode 100644 index 0000000..7dfd2a9 --- /dev/null +++ b/ModelTraining/ClipboardSemantics/IterativeResearch/EvidenceGateFineTune/final-report.json @@ -0,0 +1,8422 @@ +{ + "baseTrainingRecords": 11500, + "deploymentDecision": "retain-current-deployed-models", + "elapsedSeconds": 188.088, + "finalEvaluations": { + "golden": { + "byLanguage": { + "en": { + "intentsMeeting95Precision": 1, + "macroF1": 0.654, + "macroPrecision": 0.985, + "macroRecall": 0.5396, + "minimumPrecision": 0.8649, + "perIntent": { + "blessing": { + "f1": 0.8235, + "falseNegative": 3, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 7, + "recall": 0.7, + "total": 126, + "trueNegative": 116, + "truePositive": 7, + "wilsonPrecisionLower95": 0.6457 + }, + "complaint": { + "f1": 0.6667, + "falseNegative": 3, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 3, + "recall": 0.5, + "total": 126, + "trueNegative": 120, + "truePositive": 3, + "wilsonPrecisionLower95": 0.4385 + }, + "confirmationDecision": { + "f1": 0.8889, + "falseNegative": 2, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 8, + "recall": 0.8, + "total": 126, + "trueNegative": 116, + "truePositive": 8, + "wilsonPrecisionLower95": 0.6756 + }, + "followUpReminder": { + "f1": 0.4706, + "falseNegative": 9, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 4, + "recall": 0.3077, + "total": 126, + "trueNegative": 113, + "truePositive": 4, + "wilsonPrecisionLower95": 0.5101 + }, + "invitation": { + "f1": 0.5455, + "falseNegative": 5, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 3, + "recall": 0.375, + "total": 126, + "trueNegative": 118, + "truePositive": 3, + "wilsonPrecisionLower95": 0.4385 + }, + "question": { + "f1": 0.9048, + "falseNegative": 4, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 19, + "recall": 0.8261, + "total": 126, + "trueNegative": 103, + "truePositive": 19, + "wilsonPrecisionLower95": 0.8318 + }, + "replyableMessage": { + "f1": 0.8889, + "falseNegative": 6, + "falsePositive": 10, + "precision": 0.8649, + "predictedPositive": 74, + "recall": 0.9143, + "total": 126, + "trueNegative": 46, + "truePositive": 64, + "wilsonPrecisionLower95": 0.7688 + }, + "scheduleNegotiation": { + "f1": 0.4615, + "falseNegative": 7, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 3, + "recall": 0.3, + "total": 126, + "trueNegative": 116, + "truePositive": 3, + "wilsonPrecisionLower95": 0.4385 + }, + "task": { + "f1": 0.2353, + "falseNegative": 13, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 2, + "recall": 0.1333, + "total": 126, + "trueNegative": 111, + "truePositive": 2, + "wilsonPrecisionLower95": 0.3424 + } + }, + "recordCount": 126 + }, + "zh-Hans": { + "intentsMeeting95Precision": 1, + "macroF1": 0.4022, + "macroPrecision": 0.7683, + "macroRecall": 0.3312, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.3333, + "falseNegative": 8, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 2, + "recall": 0.2, + "total": 126, + "trueNegative": 116, + "truePositive": 2, + "wilsonPrecisionLower95": 0.3424 + }, + "complaint": { + "f1": 0.2857, + "falseNegative": 5, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 1, + "recall": 0.1667, + "total": 126, + "trueNegative": 120, + "truePositive": 1, + "wilsonPrecisionLower95": 0.2065 + }, + "confirmationDecision": { + "f1": 0.5714, + "falseNegative": 6, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 4, + "recall": 0.4, + "total": 126, + "trueNegative": 116, + "truePositive": 4, + "wilsonPrecisionLower95": 0.5101 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 13, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 126, + "trueNegative": 113, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 8, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 126, + "trueNegative": 118, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 23, + "recall": 1.0, + "total": 126, + "trueNegative": 103, + "truePositive": 23, + "wilsonPrecisionLower95": 0.8569 + }, + "replyableMessage": { + "f1": 0.9143, + "falseNegative": 6, + "falsePositive": 6, + "precision": 0.9143, + "predictedPositive": 70, + "recall": 0.9143, + "total": 126, + "trueNegative": 50, + "truePositive": 64, + "wilsonPrecisionLower95": 0.8253 + }, + "scheduleNegotiation": { + "f1": 0.1818, + "falseNegative": 9, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 1, + "recall": 0.1, + "total": 126, + "trueNegative": 116, + "truePositive": 1, + "wilsonPrecisionLower95": 0.2065 + }, + "task": { + "f1": 0.3333, + "falseNegative": 12, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 3, + "recall": 0.2, + "total": 126, + "trueNegative": 111, + "truePositive": 3, + "wilsonPrecisionLower95": 0.4385 + } + }, + "recordCount": 126 + } + }, + "bySource": { + "acknowledgment": { + "intentsMeeting95Precision": 0, + "macroF1": 0.0, + "macroPrecision": 0.0, + "macroRecall": 0.0, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 8 + }, + "blessing_boundary": { + "intentsMeeting95Precision": 0, + "macroF1": 0.1482, + "macroPrecision": 0.1667, + "macroRecall": 0.1667, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.6667, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 1, + "recall": 0.5, + "total": 20, + "trueNegative": 18, + "truePositive": 1, + "wilsonPrecisionLower95": 0.2065 + }, + "replyableMessage": { + "f1": 0.6667, + "falseNegative": 0, + "falsePositive": 4, + "precision": 0.5, + "predictedPositive": 8, + "recall": 1.0, + "total": 20, + "trueNegative": 12, + "truePositive": 4, + "wilsonPrecisionLower95": 0.2152 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 2, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 18, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 20 + }, + "blessing_golden": { + "intentsMeeting95Precision": 1, + "macroF1": 0.1772, + "macroPrecision": 0.2222, + "macroRecall": 0.1556, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.6207, + "falseNegative": 11, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 9, + "recall": 0.45, + "total": 20, + "trueNegative": 0, + "truePositive": 9, + "wilsonPrecisionLower95": 0.7009 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.9744, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 19, + "recall": 0.95, + "total": 20, + "trueNegative": 0, + "truePositive": 19, + "wilsonPrecisionLower95": 0.8318 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 20 + }, + "complaint_request": { + "intentsMeeting95Precision": 0, + "macroF1": 0.1962, + "macroPrecision": 0.2222, + "macroRecall": 0.1759, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 6, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.8571, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 3, + "recall": 0.75, + "total": 6, + "trueNegative": 2, + "truePositive": 3, + "wilsonPrecisionLower95": 0.4385 + }, + "replyableMessage": { + "f1": 0.9091, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 5, + "recall": 0.8333, + "total": 6, + "trueNegative": 0, + "truePositive": 5, + "wilsonPrecisionLower95": 0.5655 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 6 + }, + "complaint_statement": { + "intentsMeeting95Precision": 0, + "macroF1": 0.2, + "macroPrecision": 0.2222, + "macroRecall": 0.1852, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.8, + "falseNegative": 2, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 4, + "recall": 0.6667, + "total": 6, + "trueNegative": 0, + "truePositive": 4, + "wilsonPrecisionLower95": 0.5101 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 6, + "recall": 1.0, + "total": 6, + "trueNegative": 0, + "truePositive": 6, + "wilsonPrecisionLower95": 0.6097 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 6 + }, + "confirmation_decision_boundary": { + "intentsMeeting95Precision": 0, + "macroF1": 0.0, + "macroPrecision": 0.0, + "macroRecall": 0.0, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 3, + "precision": 0.0, + "predictedPositive": 3, + "recall": 0.0, + "total": 20, + "trueNegative": 17, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 20 + }, + "confirmation_decision_golden": { + "intentsMeeting95Precision": 2, + "macroF1": 0.1944, + "macroPrecision": 0.2222, + "macroRecall": 0.1778, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.75, + "falseNegative": 8, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 12, + "recall": 0.6, + "total": 20, + "trueNegative": 0, + "truePositive": 12, + "wilsonPrecisionLower95": 0.7575 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 20, + "recall": 1.0, + "total": 20, + "trueNegative": 0, + "truePositive": 20, + "wilsonPrecisionLower95": 0.8389 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 20 + }, + "conversational_message": { + "intentsMeeting95Precision": 1, + "macroF1": 0.1111, + "macroPrecision": 0.1111, + "macroRecall": 0.1111, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 10, + "trueNegative": 10, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 10, + "trueNegative": 10, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 10, + "trueNegative": 10, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 10, + "trueNegative": 10, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 10, + "trueNegative": 10, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 10, + "trueNegative": 10, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 10, + "recall": 1.0, + "total": 10, + "trueNegative": 0, + "truePositive": 10, + "wilsonPrecisionLower95": 0.7225 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 10, + "trueNegative": 10, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 10, + "trueNegative": 10, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 10 + }, + "event_statement": { + "intentsMeeting95Precision": 0, + "macroF1": 0.0, + "macroPrecision": 0.0, + "macroRecall": 0.0, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 4 + }, + "follow_up_reminder_boundary": { + "intentsMeeting95Precision": 0, + "macroF1": 0.0, + "macroPrecision": 0.0, + "macroRecall": 0.0, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 3, + "precision": 0.0, + "predictedPositive": 3, + "recall": 0.0, + "total": 20, + "trueNegative": 17, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 20 + }, + "follow_up_reminder_golden": { + "intentsMeeting95Precision": 0, + "macroF1": 0.1179, + "macroPrecision": 0.3333, + "macroRecall": 0.0802, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.2609, + "falseNegative": 17, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 3, + "recall": 0.15, + "total": 20, + "trueNegative": 0, + "truePositive": 3, + "wilsonPrecisionLower95": 0.4385 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.6667, + "falseNegative": 7, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 7, + "recall": 0.5, + "total": 20, + "trueNegative": 6, + "truePositive": 7, + "wilsonPrecisionLower95": 0.6457 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.1333, + "falseNegative": 13, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 1, + "recall": 0.0714, + "total": 20, + "trueNegative": 6, + "truePositive": 1, + "wilsonPrecisionLower95": 0.2065 + } + }, + "recordCount": 20 + }, + "information_question": { + "intentsMeeting95Precision": 0, + "macroF1": 0.1693, + "macroPrecision": 0.2222, + "macroRecall": 0.1389, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.6667, + "falseNegative": 2, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 2, + "recall": 0.5, + "total": 4, + "trueNegative": 0, + "truePositive": 2, + "wilsonPrecisionLower95": 0.3424 + }, + "replyableMessage": { + "f1": 0.8571, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 3, + "recall": 0.75, + "total": 4, + "trueNegative": 0, + "truePositive": 3, + "wilsonPrecisionLower95": 0.4385 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 4 + }, + "invitation_question": { + "intentsMeeting95Precision": 0, + "macroF1": 0.2469, + "macroPrecision": 0.3333, + "macroRecall": 0.2361, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.2222, + "falseNegative": 7, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 1, + "recall": 0.125, + "total": 8, + "trueNegative": 0, + "truePositive": 1, + "wilsonPrecisionLower95": 0.2065 + }, + "question": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 8, + "recall": 1.0, + "total": 8, + "trueNegative": 0, + "truePositive": 8, + "wilsonPrecisionLower95": 0.6756 + }, + "replyableMessage": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 8, + "recall": 1.0, + "total": 8, + "trueNegative": 0, + "truePositive": 8, + "wilsonPrecisionLower95": 0.6756 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 8 + }, + "negative_news": { + "intentsMeeting95Precision": 0, + "macroF1": 0.0, + "macroPrecision": 0.0, + "macroRecall": 0.0, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 1, + "precision": 0.0, + "predictedPositive": 1, + "recall": 0.0, + "total": 6, + "trueNegative": 5, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 6 + }, + "neutral_fact": { + "intentsMeeting95Precision": 0, + "macroF1": 0.0, + "macroPrecision": 0.0, + "macroRecall": 0.0, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 10, + "trueNegative": 10, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 10, + "trueNegative": 10, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 10, + "trueNegative": 10, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 10, + "trueNegative": 10, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 10, + "trueNegative": 10, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 10, + "trueNegative": 10, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 2, + "precision": 0.0, + "predictedPositive": 2, + "recall": 0.0, + "total": 10, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 10, + "trueNegative": 10, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 10, + "trueNegative": 10, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 10 + }, + "positive_feedback": { + "intentsMeeting95Precision": 0, + "macroF1": 0.1111, + "macroPrecision": 0.1111, + "macroRecall": 0.1111, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 6, + "recall": 1.0, + "total": 6, + "trueNegative": 0, + "truePositive": 6, + "wilsonPrecisionLower95": 0.6097 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 6 + }, + "quoted_question": { + "intentsMeeting95Precision": 0, + "macroF1": 0.0, + "macroPrecision": 0.0, + "macroRecall": 0.0, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 4 + }, + "schedule_negotiation_boundary": { + "intentsMeeting95Precision": 0, + "macroF1": 0.2601, + "macroPrecision": 0.321, + "macroRecall": 0.25, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.4, + "falseNegative": 6, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 2, + "recall": 0.25, + "total": 20, + "trueNegative": 12, + "truePositive": 2, + "wilsonPrecisionLower95": 0.3424 + }, + "question": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 4, + "recall": 1.0, + "total": 20, + "trueNegative": 16, + "truePositive": 4, + "wilsonPrecisionLower95": 0.5101 + }, + "replyableMessage": { + "f1": 0.9412, + "falseNegative": 0, + "falsePositive": 1, + "precision": 0.8889, + "predictedPositive": 9, + "recall": 1.0, + "total": 20, + "trueNegative": 11, + "truePositive": 8, + "wilsonPrecisionLower95": 0.565 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 20 + }, + "schedule_negotiation_golden": { + "intentsMeeting95Precision": 2, + "macroF1": 0.2564, + "macroPrecision": 0.3333, + "macroRecall": 0.2389, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 18, + "recall": 1.0, + "total": 20, + "trueNegative": 2, + "truePositive": 18, + "wilsonPrecisionLower95": 0.8241 + }, + "replyableMessage": { + "f1": 0.9744, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 19, + "recall": 0.95, + "total": 20, + "trueNegative": 0, + "truePositive": 19, + "wilsonPrecisionLower95": 0.8318 + }, + "scheduleNegotiation": { + "f1": 0.3333, + "falseNegative": 16, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 4, + "recall": 0.2, + "total": 20, + "trueNegative": 0, + "truePositive": 4, + "wilsonPrecisionLower95": 0.5101 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 20 + }, + "self_plan": { + "intentsMeeting95Precision": 0, + "macroF1": 0.0741, + "macroPrecision": 0.1111, + "macroRecall": 0.0556, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.6667, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 1, + "recall": 0.5, + "total": 6, + "trueNegative": 4, + "truePositive": 1, + "wilsonPrecisionLower95": 0.2065 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 2, + "precision": 0.0, + "predictedPositive": 2, + "recall": 0.0, + "total": 6, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 6 + }, + "task_question": { + "intentsMeeting95Precision": 0, + "macroF1": 0.254, + "macroPrecision": 0.3333, + "macroRecall": 0.2407, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 6, + "recall": 1.0, + "total": 6, + "trueNegative": 0, + "truePositive": 6, + "wilsonPrecisionLower95": 0.6097 + }, + "replyableMessage": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 6, + "recall": 1.0, + "total": 6, + "trueNegative": 0, + "truePositive": 6, + "wilsonPrecisionLower95": 0.6097 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.2857, + "falseNegative": 5, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 1, + "recall": 0.1667, + "total": 6, + "trueNegative": 0, + "truePositive": 1, + "wilsonPrecisionLower95": 0.2065 + } + }, + "recordCount": 6 + }, + "task_statement": { + "intentsMeeting95Precision": 0, + "macroF1": 0.1643, + "macroPrecision": 0.2222, + "macroRecall": 0.1389, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 4, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.9333, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 7, + "recall": 0.875, + "total": 8, + "trueNegative": 0, + "truePositive": 7, + "wilsonPrecisionLower95": 0.6457 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.5455, + "falseNegative": 5, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 3, + "recall": 0.375, + "total": 8, + "trueNegative": 0, + "truePositive": 3, + "wilsonPrecisionLower95": 0.4385 + } + }, + "recordCount": 8 + } + }, + "intentsMeeting95Precision": 2, + "macroF1": 0.5476, + "macroPrecision": 0.9877, + "macroRecall": 0.4354, + "minimumPrecision": 0.8889, + "perIntent": { + "blessing": { + "f1": 0.6207, + "falseNegative": 11, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 9, + "recall": 0.45, + "total": 252, + "trueNegative": 232, + "truePositive": 9, + "wilsonPrecisionLower95": 0.7009 + }, + "complaint": { + "f1": 0.5, + "falseNegative": 8, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 4, + "recall": 0.3333, + "total": 252, + "trueNegative": 240, + "truePositive": 4, + "wilsonPrecisionLower95": 0.5101 + }, + "confirmationDecision": { + "f1": 0.75, + "falseNegative": 8, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 12, + "recall": 0.6, + "total": 252, + "trueNegative": 232, + "truePositive": 12, + "wilsonPrecisionLower95": 0.7575 + }, + "followUpReminder": { + "f1": 0.2667, + "falseNegative": 22, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 4, + "recall": 0.1538, + "total": 252, + "trueNegative": 226, + "truePositive": 4, + "wilsonPrecisionLower95": 0.5101 + }, + "invitation": { + "f1": 0.3158, + "falseNegative": 13, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 3, + "recall": 0.1875, + "total": 252, + "trueNegative": 236, + "truePositive": 3, + "wilsonPrecisionLower95": 0.4385 + }, + "question": { + "f1": 0.9545, + "falseNegative": 4, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 42, + "recall": 0.913, + "total": 252, + "trueNegative": 206, + "truePositive": 42, + "wilsonPrecisionLower95": 0.9162 + }, + "replyableMessage": { + "f1": 0.9014, + "falseNegative": 12, + "falsePositive": 16, + "precision": 0.8889, + "predictedPositive": 144, + "recall": 0.9143, + "total": 252, + "trueNegative": 96, + "truePositive": 128, + "wilsonPrecisionLower95": 0.8271 + }, + "scheduleNegotiation": { + "f1": 0.3333, + "falseNegative": 16, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 4, + "recall": 0.2, + "total": 252, + "trueNegative": 232, + "truePositive": 4, + "wilsonPrecisionLower95": 0.5101 + }, + "task": { + "f1": 0.2857, + "falseNegative": 25, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 5, + "recall": 0.1667, + "total": 252, + "trueNegative": 222, + "truePositive": 5, + "wilsonPrecisionLower95": 0.5655 + } + }, + "recordCount": 252 + }, + "randomHoldout": { + "byLanguage": { + "en": { + "intentsMeeting95Precision": 5, + "macroF1": 0.4512, + "macroPrecision": 0.7745, + "macroRecall": 0.3651, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 340, + "trueNegative": 340, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0952, + "falseNegative": 19, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 1, + "recall": 0.05, + "total": 340, + "trueNegative": 320, + "truePositive": 1, + "wilsonPrecisionLower95": 0.2065 + }, + "confirmationDecision": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 20, + "recall": 1.0, + "total": 340, + "trueNegative": 320, + "truePositive": 20, + "wilsonPrecisionLower95": 0.8389 + }, + "followUpReminder": { + "f1": 0.5, + "falseNegative": 40, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 20, + "recall": 0.3333, + "total": 340, + "trueNegative": 280, + "truePositive": 20, + "wilsonPrecisionLower95": 0.8389 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 20, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 340, + "trueNegative": 320, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.6087, + "falseNegative": 45, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 35, + "recall": 0.4375, + "total": 340, + "trueNegative": 260, + "truePositive": 35, + "wilsonPrecisionLower95": 0.9011 + }, + "replyableMessage": { + "f1": 0.8859, + "falseNegative": 37, + "falsePositive": 5, + "precision": 0.9702, + "predictedPositive": 168, + "recall": 0.815, + "total": 340, + "trueNegative": 135, + "truePositive": 163, + "wilsonPrecisionLower95": 0.9322 + }, + "scheduleNegotiation": { + "f1": 0.5714, + "falseNegative": 12, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 8, + "recall": 0.4, + "total": 340, + "trueNegative": 320, + "truePositive": 8, + "wilsonPrecisionLower95": 0.6756 + }, + "task": { + "f1": 0.4, + "falseNegative": 45, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 15, + "recall": 0.25, + "total": 340, + "trueNegative": 280, + "truePositive": 15, + "wilsonPrecisionLower95": 0.7961 + } + }, + "recordCount": 340 + }, + "zh-Hans": { + "intentsMeeting95Precision": 2, + "macroF1": 0.3012, + "macroPrecision": 0.6608, + "macroRecall": 0.2406, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 340, + "trueNegative": 340, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0952, + "falseNegative": 19, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 1, + "recall": 0.05, + "total": 340, + "trueNegative": 320, + "truePositive": 1, + "wilsonPrecisionLower95": 0.2065 + }, + "confirmationDecision": { + "f1": 0.5185, + "falseNegative": 13, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 7, + "recall": 0.35, + "total": 340, + "trueNegative": 320, + "truePositive": 7, + "wilsonPrecisionLower95": 0.6457 + }, + "followUpReminder": { + "f1": 0.0645, + "falseNegative": 58, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 2, + "recall": 0.0333, + "total": 340, + "trueNegative": 280, + "truePositive": 2, + "wilsonPrecisionLower95": 0.3424 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 20, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 340, + "trueNegative": 320, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.7786, + "falseNegative": 29, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 51, + "recall": 0.6375, + "total": 340, + "trueNegative": 260, + "truePositive": 51, + "wilsonPrecisionLower95": 0.93 + }, + "replyableMessage": { + "f1": 0.9203, + "falseNegative": 21, + "falsePositive": 10, + "precision": 0.9471, + "predictedPositive": 189, + "recall": 0.895, + "total": 340, + "trueNegative": 130, + "truePositive": 179, + "wilsonPrecisionLower95": 0.9054 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 20, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 340, + "trueNegative": 320, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.3333, + "falseNegative": 48, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 12, + "recall": 0.2, + "total": 340, + "trueNegative": 280, + "truePositive": 12, + "wilsonPrecisionLower95": 0.7575 + } + }, + "recordCount": 340 + } + }, + "bySource": { + "acknowledgment_boundary": { + "intentsMeeting95Precision": 0, + "macroF1": 0.0, + "macroPrecision": 0.0, + "macroRecall": 0.0, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 40 + }, + "complaint_support": { + "intentsMeeting95Precision": 1, + "macroF1": 0.1518, + "macroPrecision": 0.3333, + "macroRecall": 0.1222, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0952, + "falseNegative": 38, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 2, + "recall": 0.05, + "total": 40, + "trueNegative": 0, + "truePositive": 2, + "wilsonPrecisionLower95": 0.3424 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.3673, + "falseNegative": 31, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 9, + "recall": 0.225, + "total": 40, + "trueNegative": 0, + "truePositive": 9, + "wilsonPrecisionLower95": 0.7009 + }, + "replyableMessage": { + "f1": 0.9041, + "falseNegative": 7, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 33, + "recall": 0.825, + "total": 40, + "trueNegative": 0, + "truePositive": 33, + "wilsonPrecisionLower95": 0.8957 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 40 + }, + "confirmation_decision": { + "intentsMeeting95Precision": 2, + "macroF1": 0.19, + "macroPrecision": 0.2222, + "macroRecall": 0.1667, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.806, + "falseNegative": 13, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 27, + "recall": 0.675, + "total": 40, + "trueNegative": 0, + "truePositive": 27, + "wilsonPrecisionLower95": 0.8754 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.9041, + "falseNegative": 7, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 33, + "recall": 0.825, + "total": 40, + "trueNegative": 0, + "truePositive": 33, + "wilsonPrecisionLower95": 0.8957 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 40 + }, + "conversation": { + "intentsMeeting95Precision": 1, + "macroF1": 0.1053, + "macroPrecision": 0.1111, + "macroRecall": 0.1, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.9474, + "falseNegative": 4, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 36, + "recall": 0.9, + "total": 40, + "trueNegative": 0, + "truePositive": 36, + "wilsonPrecisionLower95": 0.9036 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 40 + }, + "event_boundary": { + "intentsMeeting95Precision": 0, + "macroF1": 0.0, + "macroPrecision": 0.0, + "macroRecall": 0.0, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 2, + "precision": 0.0, + "predictedPositive": 2, + "recall": 0.0, + "total": 40, + "trueNegative": 38, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 40 + }, + "fixed_invitation": { + "intentsMeeting95Precision": 2, + "macroF1": 0.2134, + "macroPrecision": 0.2222, + "macroRecall": 0.2056, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 40, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.9333, + "falseNegative": 5, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 35, + "recall": 0.875, + "total": 40, + "trueNegative": 0, + "truePositive": 35, + "wilsonPrecisionLower95": 0.9011 + }, + "replyableMessage": { + "f1": 0.9873, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 39, + "recall": 0.975, + "total": 40, + "trueNegative": 0, + "truePositive": 39, + "wilsonPrecisionLower95": 0.9103 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 40 + }, + "follow_up_instruction": { + "intentsMeeting95Precision": 1, + "macroF1": 0.0934, + "macroPrecision": 0.1111, + "macroRecall": 0.0806, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 40, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.8406, + "falseNegative": 11, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 29, + "recall": 0.725, + "total": 40, + "trueNegative": 0, + "truePositive": 29, + "wilsonPrecisionLower95": 0.883 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 40, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 40 + }, + "follow_up_personal_reminder": { + "intentsMeeting95Precision": 1, + "macroF1": 0.0789, + "macroPrecision": 0.1111, + "macroRecall": 0.0611, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.7097, + "falseNegative": 18, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 22, + "recall": 0.55, + "total": 40, + "trueNegative": 0, + "truePositive": 22, + "wilsonPrecisionLower95": 0.8513 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 40 + }, + "information_question": { + "intentsMeeting95Precision": 2, + "macroF1": 0.1421, + "macroPrecision": 0.2222, + "macroRecall": 0.1083, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.4906, + "falseNegative": 27, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 13, + "recall": 0.325, + "total": 40, + "trueNegative": 0, + "truePositive": 13, + "wilsonPrecisionLower95": 0.7719 + }, + "replyableMessage": { + "f1": 0.7879, + "falseNegative": 14, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 26, + "recall": 0.65, + "total": 40, + "trueNegative": 0, + "truePositive": 26, + "wilsonPrecisionLower95": 0.8713 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 40 + }, + "negative_news": { + "intentsMeeting95Precision": 0, + "macroF1": 0.0, + "macroPrecision": 0.0, + "macroRecall": 0.0, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 3, + "precision": 0.0, + "predictedPositive": 3, + "recall": 0.0, + "total": 40, + "trueNegative": 37, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 40 + }, + "neutral_fact": { + "intentsMeeting95Precision": 0, + "macroF1": 0.0, + "macroPrecision": 0.0, + "macroRecall": 0.0, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 5, + "precision": 0.0, + "predictedPositive": 5, + "recall": 0.0, + "total": 40, + "trueNegative": 35, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 40 + }, + "positive_feedback": { + "intentsMeeting95Precision": 1, + "macroF1": 0.1068, + "macroPrecision": 0.1111, + "macroRecall": 0.1028, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.961, + "falseNegative": 3, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 37, + "recall": 0.925, + "total": 40, + "trueNegative": 0, + "truePositive": 37, + "wilsonPrecisionLower95": 0.9059 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 40 + }, + "quoted_question_boundary": { + "intentsMeeting95Precision": 0, + "macroF1": 0.0, + "macroPrecision": 0.0, + "macroRecall": 0.0, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 1, + "precision": 0.0, + "predictedPositive": 1, + "recall": 0.0, + "total": 40, + "trueNegative": 39, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 40 + }, + "schedule_negotiation": { + "intentsMeeting95Precision": 2, + "macroF1": 0.2415, + "macroPrecision": 0.3333, + "macroRecall": 0.2139, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.8406, + "falseNegative": 11, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 29, + "recall": 0.725, + "total": 40, + "trueNegative": 0, + "truePositive": 29, + "wilsonPrecisionLower95": 0.883 + }, + "replyableMessage": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 40, + "recall": 1.0, + "total": 40, + "trueNegative": 0, + "truePositive": 40, + "wilsonPrecisionLower95": 0.9124 + }, + "scheduleNegotiation": { + "f1": 0.3333, + "falseNegative": 32, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 8, + "recall": 0.2, + "total": 40, + "trueNegative": 0, + "truePositive": 8, + "wilsonPrecisionLower95": 0.6756 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 40 + }, + "task_next_action": { + "intentsMeeting95Precision": 2, + "macroF1": 0.1517, + "macroPrecision": 0.2222, + "macroRecall": 0.125, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 40, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.9041, + "falseNegative": 7, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 33, + "recall": 0.825, + "total": 40, + "trueNegative": 0, + "truePositive": 33, + "wilsonPrecisionLower95": 0.8957 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.4615, + "falseNegative": 28, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 12, + "recall": 0.3, + "total": 40, + "trueNegative": 0, + "truePositive": 12, + "wilsonPrecisionLower95": 0.7575 + } + }, + "recordCount": 40 + }, + "task_request": { + "intentsMeeting95Precision": 2, + "macroF1": 0.1659, + "macroPrecision": 0.2222, + "macroRecall": 0.1417, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.9474, + "falseNegative": 4, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 36, + "recall": 0.9, + "total": 40, + "trueNegative": 0, + "truePositive": 36, + "wilsonPrecisionLower95": 0.9036 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.5455, + "falseNegative": 25, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 15, + "recall": 0.375, + "total": 40, + "trueNegative": 0, + "truePositive": 15, + "wilsonPrecisionLower95": 0.7961 + } + }, + "recordCount": 40 + }, + "vague_future_boundary": { + "intentsMeeting95Precision": 0, + "macroF1": 0.0, + "macroPrecision": 0.0, + "macroRecall": 0.0, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 4, + "precision": 0.0, + "predictedPositive": 4, + "recall": 0.0, + "total": 40, + "trueNegative": 36, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 40 + } + }, + "intentsMeeting95Precision": 5, + "macroF1": 0.3905, + "macroPrecision": 0.7731, + "macroRecall": 0.3029, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 680, + "trueNegative": 680, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0952, + "falseNegative": 38, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 2, + "recall": 0.05, + "total": 680, + "trueNegative": 640, + "truePositive": 2, + "wilsonPrecisionLower95": 0.3424 + }, + "confirmationDecision": { + "f1": 0.806, + "falseNegative": 13, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 27, + "recall": 0.675, + "total": 680, + "trueNegative": 640, + "truePositive": 27, + "wilsonPrecisionLower95": 0.8754 + }, + "followUpReminder": { + "f1": 0.3099, + "falseNegative": 98, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 22, + "recall": 0.1833, + "total": 680, + "trueNegative": 560, + "truePositive": 22, + "wilsonPrecisionLower95": 0.8513 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 40, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 680, + "trueNegative": 640, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.6992, + "falseNegative": 74, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 86, + "recall": 0.5375, + "total": 680, + "trueNegative": 520, + "truePositive": 86, + "wilsonPrecisionLower95": 0.9572 + }, + "replyableMessage": { + "f1": 0.9036, + "falseNegative": 58, + "falsePositive": 15, + "precision": 0.958, + "predictedPositive": 357, + "recall": 0.855, + "total": 680, + "trueNegative": 265, + "truePositive": 342, + "wilsonPrecisionLower95": 0.9318 + }, + "scheduleNegotiation": { + "f1": 0.3333, + "falseNegative": 32, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 8, + "recall": 0.2, + "total": 680, + "trueNegative": 640, + "truePositive": 8, + "wilsonPrecisionLower95": 0.6756 + }, + "task": { + "f1": 0.3673, + "falseNegative": 93, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 27, + "recall": 0.225, + "total": 680, + "trueNegative": 560, + "truePositive": 27, + "wilsonPrecisionLower95": 0.8754 + } + }, + "recordCount": 680 + }, + "researchOnlyComprehensive": { + "byLanguage": { + "en": { + "intentsMeeting95Precision": 2, + "macroF1": 0.299, + "macroPrecision": 0.5506, + "macroRecall": 0.2272, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.4667, + "falseNegative": 48, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 21, + "recall": 0.3043, + "total": 13176, + "trueNegative": 13107, + "truePositive": 21, + "wilsonPrecisionLower95": 0.8454 + }, + "complaint": { + "f1": 0.009, + "falseNegative": 432, + "falsePositive": 6, + "precision": 0.25, + "predictedPositive": 8, + "recall": 0.0046, + "total": 13176, + "trueNegative": 12736, + "truePositive": 2, + "wilsonPrecisionLower95": 0.0715 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 12, + "falsePositive": 11, + "precision": 0.0, + "predictedPositive": 11, + "recall": 0.0, + "total": 13176, + "trueNegative": 13153, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "followUpReminder": { + "f1": 0.297, + "falseNegative": 95, + "falsePositive": 47, + "precision": 0.3896, + "predictedPositive": 77, + "recall": 0.24, + "total": 13176, + "trueNegative": 13004, + "truePositive": 30, + "wilsonPrecisionLower95": 0.2884 + }, + "invitation": { + "f1": 0.3014, + "falseNegative": 50, + "falsePositive": 1, + "precision": 0.9167, + "predictedPositive": 12, + "recall": 0.1803, + "total": 13176, + "trueNegative": 13114, + "truePositive": 11, + "wilsonPrecisionLower95": 0.6461 + }, + "question": { + "f1": 0.4565, + "falseNegative": 1862, + "falsePositive": 29, + "precision": 0.9648, + "predictedPositive": 823, + "recall": 0.2989, + "total": 13176, + "trueNegative": 10491, + "truePositive": 794, + "wilsonPrecisionLower95": 0.9499 + }, + "replyableMessage": { + "f1": 0.7642, + "falseNegative": 2967, + "falsePositive": 1917, + "precision": 0.805, + "predictedPositive": 9831, + "recall": 0.7273, + "total": 13176, + "trueNegative": 378, + "truePositive": 7914, + "wilsonPrecisionLower95": 0.7971 + }, + "scheduleNegotiation": { + "f1": 0.14, + "falseNegative": 60, + "falsePositive": 26, + "precision": 0.2121, + "predictedPositive": 33, + "recall": 0.1045, + "total": 13176, + "trueNegative": 13083, + "truePositive": 7, + "wilsonPrecisionLower95": 0.1068 + }, + "task": { + "f1": 0.2563, + "falseNegative": 912, + "falsePositive": 289, + "precision": 0.4173, + "predictedPositive": 496, + "recall": 0.185, + "total": 13176, + "trueNegative": 11768, + "truePositive": 207, + "wilsonPrecisionLower95": 0.3747 + } + }, + "recordCount": 13176 + }, + "zh-Hans": { + "intentsMeeting95Precision": 0, + "macroF1": 0.1436, + "macroPrecision": 0.5377, + "macroRecall": 0.1039, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.2105, + "falseNegative": 15, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 2, + "recall": 0.1176, + "total": 28019, + "trueNegative": 28002, + "truePositive": 2, + "wilsonPrecisionLower95": 0.3424 + }, + "complaint": { + "f1": 0.1895, + "falseNegative": 241, + "falsePositive": 7, + "precision": 0.8056, + "predictedPositive": 36, + "recall": 0.1074, + "total": 28019, + "trueNegative": 27742, + "truePositive": 29, + "wilsonPrecisionLower95": 0.6497 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 57, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 28019, + "trueNegative": 27962, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0075, + "falseNegative": 264, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 1, + "recall": 0.0038, + "total": 28019, + "trueNegative": 27754, + "truePositive": 1, + "wilsonPrecisionLower95": 0.2065 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 73, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 28019, + "trueNegative": 27946, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.1195, + "falseNegative": 6725, + "falsePositive": 36, + "precision": 0.9273, + "predictedPositive": 495, + "recall": 0.0639, + "total": 28019, + "trueNegative": 20799, + "truePositive": 459, + "wilsonPrecisionLower95": 0.901 + }, + "replyableMessage": { + "f1": 0.7622, + "falseNegative": 9480, + "falsePositive": 1096, + "precision": 0.9393, + "predictedPositive": 18042, + "recall": 0.6413, + "total": 28019, + "trueNegative": 497, + "truePositive": 16946, + "wilsonPrecisionLower95": 0.9357 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 4, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 28019, + "trueNegative": 28015, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.003, + "falseNegative": 658, + "falsePositive": 5, + "precision": 0.1667, + "predictedPositive": 6, + "recall": 0.0015, + "total": 28019, + "trueNegative": 27355, + "truePositive": 1, + "wilsonPrecisionLower95": 0.0301 + } + }, + "recordCount": 28019 + } + }, + "bySource": { + "AmazonScience/MASSIVE en-US test": { + "intentsMeeting95Precision": 2, + "macroF1": 0.1742, + "macroPrecision": 0.2929, + "macroRecall": 0.1343, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 2967, + "trueNegative": 2967, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 1, + "precision": 0.0, + "predictedPositive": 1, + "recall": 0.0, + "total": 2967, + "trueNegative": 2966, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 1, + "precision": 0.0, + "predictedPositive": 1, + "recall": 0.0, + "total": 2967, + "trueNegative": 2966, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "followUpReminder": { + "f1": 0.321, + "falseNegative": 71, + "falsePositive": 39, + "precision": 0.4, + "predictedPositive": 65, + "recall": 0.268, + "total": 2967, + "trueNegative": 2831, + "truePositive": 26, + "wilsonPrecisionLower95": 0.2897 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 2967, + "trueNegative": 2967, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.3271, + "falseNegative": 1300, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 316, + "recall": 0.1955, + "total": 2967, + "trueNegative": 1351, + "truePositive": 316, + "wilsonPrecisionLower95": 0.988 + }, + "replyableMessage": { + "f1": 0.7951, + "falseNegative": 1009, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 1958, + "recall": 0.6599, + "total": 2967, + "trueNegative": 0, + "truePositive": 1958, + "wilsonPrecisionLower95": 0.998 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 5, + "precision": 0.0, + "predictedPositive": 5, + "recall": 0.0, + "total": 2967, + "trueNegative": 2962, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "task": { + "f1": 0.125, + "falseNegative": 495, + "falsePositive": 149, + "precision": 0.2359, + "predictedPositive": 195, + "recall": 0.085, + "total": 2967, + "trueNegative": 2277, + "truePositive": 46, + "wilsonPrecisionLower95": 0.1818 + } + }, + "recordCount": 2967 + }, + "AmazonScience/MASSIVE zh-CN test": { + "intentsMeeting95Precision": 2, + "macroF1": 0.1042, + "macroPrecision": 0.3687, + "macroRecall": 0.0795, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 2850, + "trueNegative": 2850, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 2850, + "trueNegative": 2850, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 2850, + "trueNegative": 2850, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.012, + "falseNegative": 165, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 1, + "recall": 0.006, + "total": 2850, + "trueNegative": 2684, + "truePositive": 1, + "wilsonPrecisionLower95": 0.2065 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 2850, + "trueNegative": 2850, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.1535, + "falseNegative": 1432, + "falsePositive": 2, + "precision": 0.9848, + "predictedPositive": 132, + "recall": 0.0832, + "total": 2850, + "trueNegative": 1286, + "truePositive": 130, + "wilsonPrecisionLower95": 0.9464 + }, + "replyableMessage": { + "f1": 0.7684, + "falseNegative": 1072, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 1778, + "recall": 0.6239, + "total": 2850, + "trueNegative": 0, + "truePositive": 1778, + "wilsonPrecisionLower95": 0.9978 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 2850, + "trueNegative": 2850, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0042, + "falseNegative": 477, + "falsePositive": 2, + "precision": 0.3333, + "predictedPositive": 3, + "recall": 0.0021, + "total": 2850, + "trueNegative": 2370, + "truePositive": 1, + "wilsonPrecisionLower95": 0.0615 + } + }, + "recordCount": 2850 + }, + "CFPB via hpe-ai/customer-complaints": { + "intentsMeeting95Precision": 1, + "macroF1": 0.082, + "macroPrecision": 0.2222, + "macroRecall": 0.0621, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 250, + "trueNegative": 250, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 250, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 250, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 250, + "trueNegative": 250, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 250, + "trueNegative": 250, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 250, + "trueNegative": 250, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 17, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 250, + "trueNegative": 233, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.7013, + "falseNegative": 115, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 135, + "recall": 0.54, + "total": 250, + "trueNegative": 0, + "truePositive": 135, + "wilsonPrecisionLower95": 0.9723 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 250, + "trueNegative": 250, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.037, + "falseNegative": 52, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 1, + "recall": 0.0189, + "total": 250, + "trueNegative": 197, + "truePositive": 1, + "wilsonPrecisionLower95": 0.2065 + } + }, + "recordCount": 250 + }, + "CPED test split": { + "intentsMeeting95Precision": 0, + "macroF1": 0.1202, + "macroPrecision": 0.3154, + "macroRecall": 0.091, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.2105, + "falseNegative": 15, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 2, + "recall": 0.1176, + "total": 25049, + "trueNegative": 25032, + "truePositive": 2, + "wilsonPrecisionLower95": 0.3424 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 150, + "falsePositive": 7, + "precision": 0.0, + "predictedPositive": 7, + "recall": 0.0, + "total": 25049, + "trueNegative": 24892, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 57, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 25049, + "trueNegative": 24992, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 99, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 25049, + "trueNegative": 24950, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 73, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 25049, + "trueNegative": 24976, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.1106, + "falseNegative": 5257, + "falsePositive": 34, + "precision": 0.9063, + "predictedPositive": 363, + "recall": 0.0589, + "total": 25049, + "trueNegative": 19429, + "truePositive": 329, + "wilsonPrecisionLower95": 0.872 + }, + "replyableMessage": { + "f1": 0.7609, + "falseNegative": 8378, + "falsePositive": 1096, + "precision": 0.9322, + "predictedPositive": 16174, + "recall": 0.6428, + "total": 25049, + "trueNegative": 497, + "truePositive": 15078, + "wilsonPrecisionLower95": 0.9283 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 4, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 25049, + "trueNegative": 25045, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 168, + "falsePositive": 3, + "precision": 0.0, + "predictedPositive": 3, + "recall": 0.0, + "total": 25049, + "trueNegative": 24878, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + } + }, + "recordCount": 25049 + }, + "Google Research GoEmotions test": { + "intentsMeeting95Precision": 2, + "macroF1": 0.2362, + "macroPrecision": 0.4125, + "macroRecall": 0.175, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.4651, + "falseNegative": 46, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 20, + "recall": 0.303, + "total": 5418, + "trueNegative": 5352, + "truePositive": 20, + "wilsonPrecisionLower95": 0.8389 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 20, + "falsePositive": 4, + "precision": 0.0, + "predictedPositive": 4, + "recall": 0.0, + "total": 5418, + "trueNegative": 5394, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 5, + "precision": 0.0, + "predictedPositive": 5, + "recall": 0.0, + "total": 5418, + "trueNegative": 5413, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 3, + "precision": 0.0, + "predictedPositive": 3, + "recall": 0.0, + "total": 5418, + "trueNegative": 5415, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 5418, + "trueNegative": 5418, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.4649, + "falseNegative": 363, + "falsePositive": 26, + "precision": 0.8667, + "predictedPositive": 195, + "recall": 0.3177, + "total": 5418, + "trueNegative": 4860, + "truePositive": 169, + "wilsonPrecisionLower95": 0.8118 + }, + "replyableMessage": { + "f1": 0.8467, + "falseNegative": 1440, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 3978, + "recall": 0.7342, + "total": 5418, + "trueNegative": 0, + "truePositive": 3978, + "wilsonPrecisionLower95": 0.999 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 1, + "precision": 0.0, + "predictedPositive": 1, + "recall": 0.0, + "total": 5418, + "trueNegative": 5417, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "task": { + "f1": 0.3492, + "falseNegative": 39, + "falsePositive": 2, + "precision": 0.8462, + "predictedPositive": 13, + "recall": 0.22, + "total": 5418, + "trueNegative": 5366, + "truePositive": 11, + "wilsonPrecisionLower95": 0.5777 + } + }, + "recordCount": 5418 + }, + "Meituan-Dianping/ASAP test": { + "intentsMeeting95Precision": 2, + "macroF1": 0.1385, + "macroPrecision": 0.2222, + "macroRecall": 0.1102, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 120, + "trueNegative": 120, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.3893, + "falseNegative": 91, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 29, + "recall": 0.2417, + "total": 120, + "trueNegative": 0, + "truePositive": 29, + "wilsonPrecisionLower95": 0.883 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 120, + "trueNegative": 120, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 120, + "trueNegative": 120, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 120, + "trueNegative": 120, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 36, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 120, + "trueNegative": 84, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.8571, + "falseNegative": 30, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 90, + "recall": 0.75, + "total": 120, + "trueNegative": 0, + "truePositive": 90, + "wilsonPrecisionLower95": 0.9591 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 120, + "trueNegative": 120, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 13, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 120, + "trueNegative": 107, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 120 + }, + "Sulak2020 GitHub issues multi-repo dataset": { + "intentsMeeting95Precision": 2, + "macroF1": 0.1469, + "macroPrecision": 0.2963, + "macroRecall": 0.1143, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 2, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 1004, + "trueNegative": 1002, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.024, + "falseNegative": 162, + "falsePositive": 1, + "precision": 0.6667, + "predictedPositive": 3, + "recall": 0.0122, + "total": 1004, + "trueNegative": 839, + "truePositive": 2, + "wilsonPrecisionLower95": 0.2077 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 2, + "falsePositive": 1, + "precision": 0.0, + "predictedPositive": 1, + "recall": 0.0, + "total": 1004, + "trueNegative": 1001, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 2, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 1004, + "trueNegative": 1002, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 1004, + "trueNegative": 1004, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.4658, + "falseNegative": 39, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 17, + "recall": 0.3036, + "total": 1004, + "trueNegative": 948, + "truePositive": 17, + "wilsonPrecisionLower95": 0.8157 + }, + "replyableMessage": { + "f1": 0.8326, + "falseNegative": 288, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 716, + "recall": 0.7131, + "total": 1004, + "trueNegative": 0, + "truePositive": 716, + "wilsonPrecisionLower95": 0.9947 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 25, + "falsePositive": 2, + "precision": 0.0, + "predictedPositive": 2, + "recall": 0.0, + "total": 1004, + "trueNegative": 977, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "task": { + "f1": 0.0, + "falseNegative": 90, + "falsePositive": 1, + "precision": 0.0, + "predictedPositive": 1, + "recall": 0.0, + "total": 1004, + "trueNegative": 913, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + } + }, + "recordCount": 1004 + }, + "Verified Enron Intent Dataset": { + "intentsMeeting95Precision": 1, + "macroF1": 0.3899, + "macroPrecision": 0.5025, + "macroRecall": 0.3852, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 1, + "recall": 1.0, + "total": 3537, + "trueNegative": 3536, + "truePositive": 1, + "wilsonPrecisionLower95": 0.2065 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 3537, + "trueNegative": 3537, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 10, + "falsePositive": 4, + "precision": 0.0, + "predictedPositive": 4, + "recall": 0.0, + "total": 3537, + "trueNegative": 3523, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "followUpReminder": { + "f1": 0.2286, + "falseNegative": 22, + "falsePositive": 5, + "precision": 0.4444, + "predictedPositive": 9, + "recall": 0.1538, + "total": 3537, + "trueNegative": 3506, + "truePositive": 4, + "wilsonPrecisionLower95": 0.1888 + }, + "invitation": { + "f1": 0.3014, + "falseNegative": 50, + "falsePositive": 1, + "precision": 0.9167, + "predictedPositive": 12, + "recall": 0.1803, + "total": 3537, + "trueNegative": 3475, + "truePositive": 11, + "wilsonPrecisionLower95": 0.6461 + }, + "question": { + "f1": 0.8, + "falseNegative": 143, + "falsePositive": 3, + "precision": 0.9898, + "predictedPositive": 295, + "recall": 0.6713, + "total": 3537, + "trueNegative": 3099, + "truePositive": 292, + "wilsonPrecisionLower95": 0.9705 + }, + "replyableMessage": { + "f1": 0.5259, + "falseNegative": 115, + "falsePositive": 1917, + "precision": 0.3702, + "predictedPositive": 3044, + "recall": 0.9074, + "total": 3537, + "trueNegative": 378, + "truePositive": 1127, + "wilsonPrecisionLower95": 0.3533 + }, + "scheduleNegotiation": { + "f1": 0.209, + "falseNegative": 35, + "falsePositive": 18, + "precision": 0.28, + "predictedPositive": 25, + "recall": 0.1667, + "total": 3537, + "trueNegative": 3477, + "truePositive": 7, + "wilsonPrecisionLower95": 0.1428 + }, + "task": { + "f1": 0.4441, + "falseNegative": 236, + "falsePositive": 137, + "precision": 0.521, + "predictedPositive": 286, + "recall": 0.387, + "total": 3537, + "trueNegative": 3015, + "truePositive": 149, + "wilsonPrecisionLower95": 0.4632 + } + }, + "recordCount": 3537 + } + }, + "intentsMeeting95Precision": 2, + "macroF1": 0.2325, + "macroPrecision": 0.6097, + "macroRecall": 0.1647, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.422, + "falseNegative": 63, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 23, + "recall": 0.2674, + "total": 41195, + "trueNegative": 41109, + "truePositive": 23, + "wilsonPrecisionLower95": 0.8569 + }, + "complaint": { + "f1": 0.0829, + "falseNegative": 673, + "falsePositive": 13, + "precision": 0.7045, + "predictedPositive": 44, + "recall": 0.044, + "total": 41195, + "trueNegative": 40478, + "truePositive": 31, + "wilsonPrecisionLower95": 0.5578 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 69, + "falsePositive": 11, + "precision": 0.0, + "predictedPositive": 11, + "recall": 0.0, + "total": 41195, + "trueNegative": 41115, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "followUpReminder": { + "f1": 0.1325, + "falseNegative": 359, + "falsePositive": 47, + "precision": 0.3974, + "predictedPositive": 78, + "recall": 0.0795, + "total": 41195, + "trueNegative": 40758, + "truePositive": 31, + "wilsonPrecisionLower95": 0.2961 + }, + "invitation": { + "f1": 0.1507, + "falseNegative": 123, + "falsePositive": 1, + "precision": 0.9167, + "predictedPositive": 12, + "recall": 0.0821, + "total": 41195, + "trueNegative": 41060, + "truePositive": 11, + "wilsonPrecisionLower95": 0.6461 + }, + "question": { + "f1": 0.2246, + "falseNegative": 8587, + "falsePositive": 65, + "precision": 0.9507, + "predictedPositive": 1318, + "recall": 0.1273, + "total": 41195, + "trueNegative": 31290, + "truePositive": 1253, + "wilsonPrecisionLower95": 0.9376 + }, + "replyableMessage": { + "f1": 0.7628, + "falseNegative": 12447, + "falsePositive": 3013, + "precision": 0.8919, + "predictedPositive": 27873, + "recall": 0.6664, + "total": 41195, + "trueNegative": 875, + "truePositive": 24860, + "wilsonPrecisionLower95": 0.8882 + }, + "scheduleNegotiation": { + "f1": 0.1346, + "falseNegative": 64, + "falsePositive": 26, + "precision": 0.2121, + "predictedPositive": 33, + "recall": 0.0986, + "total": 41195, + "trueNegative": 41098, + "truePositive": 7, + "wilsonPrecisionLower95": 0.1068 + }, + "task": { + "f1": 0.1825, + "falseNegative": 1570, + "falsePositive": 294, + "precision": 0.4143, + "predictedPositive": 502, + "recall": 0.117, + "total": 41195, + "trueNegative": 39123, + "truePositive": 208, + "wilsonPrecisionLower95": 0.3721 + } + }, + "recordCount": 41195 + }, + "targetedRelease": { + "byLanguage": { + "en": { + "intentsMeeting95Precision": 4, + "macroF1": 0.3481, + "macroPrecision": 0.5476, + "macroRecall": 0.2799, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 320, + "trueNegative": 320, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.3562, + "falseNegative": 94, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 26, + "recall": 0.2167, + "total": 320, + "trueNegative": 200, + "truePositive": 26, + "wilsonPrecisionLower95": 0.8713 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 320, + "trueNegative": 320, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.8732, + "falseNegative": 9, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 31, + "recall": 0.775, + "total": 320, + "trueNegative": 280, + "truePositive": 31, + "wilsonPrecisionLower95": 0.8897 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 320, + "trueNegative": 320, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.4762, + "falseNegative": 55, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 25, + "recall": 0.3125, + "total": 320, + "trueNegative": 240, + "truePositive": 25, + "wilsonPrecisionLower95": 0.8668 + }, + "replyableMessage": { + "f1": 0.8819, + "falseNegative": 32, + "falsePositive": 13, + "precision": 0.9282, + "predictedPositive": 181, + "recall": 0.84, + "total": 320, + "trueNegative": 107, + "truePositive": 168, + "wilsonPrecisionLower95": 0.881 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 320, + "trueNegative": 320, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.5455, + "falseNegative": 50, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 30, + "recall": 0.375, + "total": 320, + "trueNegative": 240, + "truePositive": 30, + "wilsonPrecisionLower95": 0.8865 + } + }, + "recordCount": 320 + }, + "zh-Hans": { + "intentsMeeting95Precision": 3, + "macroF1": 0.2553, + "macroPrecision": 0.3278, + "macroRecall": 0.2275, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 320, + "trueNegative": 320, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 120, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 320, + "trueNegative": 200, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 320, + "trueNegative": 320, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 40, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 320, + "trueNegative": 280, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 320, + "trueNegative": 320, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.8811, + "falseNegative": 17, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 63, + "recall": 0.7875, + "total": 320, + "trueNegative": 240, + "truePositive": 63, + "wilsonPrecisionLower95": 0.9425 + }, + "replyableMessage": { + "f1": 0.9552, + "falseNegative": 8, + "falsePositive": 10, + "precision": 0.9505, + "predictedPositive": 202, + "recall": 0.96, + "total": 320, + "trueNegative": 110, + "truePositive": 192, + "wilsonPrecisionLower95": 0.9113 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 320, + "trueNegative": 320, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.4615, + "falseNegative": 56, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 24, + "recall": 0.3, + "total": 320, + "trueNegative": 240, + "truePositive": 24, + "wilsonPrecisionLower95": 0.862 + } + }, + "recordCount": 320 + } + }, + "bySource": { + "release_complaint_brief": { + "intentsMeeting95Precision": 1, + "macroF1": 0.1329, + "macroPrecision": 0.2222, + "macroRecall": 0.1222, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.2022, + "falseNegative": 71, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 9, + "recall": 0.1125, + "total": 80, + "trueNegative": 0, + "truePositive": 9, + "wilsonPrecisionLower95": 0.7009 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.9937, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 79, + "recall": 0.9875, + "total": 80, + "trueNegative": 0, + "truePositive": 79, + "wilsonPrecisionLower95": 0.9536 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 80 + }, + "release_complaint_question": { + "intentsMeeting95Precision": 3, + "macroF1": 0.1778, + "macroPrecision": 0.3333, + "macroRecall": 0.1375, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.2222, + "falseNegative": 70, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 10, + "recall": 0.125, + "total": 80, + "trueNegative": 0, + "truePositive": 10, + "wilsonPrecisionLower95": 0.7225 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.5047, + "falseNegative": 53, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 27, + "recall": 0.3375, + "total": 80, + "trueNegative": 0, + "truePositive": 27, + "wilsonPrecisionLower95": 0.8754 + }, + "replyableMessage": { + "f1": 0.8732, + "falseNegative": 18, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 62, + "recall": 0.775, + "total": 80, + "trueNegative": 0, + "truePositive": 62, + "wilsonPrecisionLower95": 0.9417 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 80 + }, + "release_complaint_unspoken": { + "intentsMeeting95Precision": 1, + "macroF1": 0.1231, + "macroPrecision": 0.2222, + "macroRecall": 0.1097, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.1609, + "falseNegative": 73, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 7, + "recall": 0.0875, + "total": 80, + "trueNegative": 0, + "truePositive": 7, + "wilsonPrecisionLower95": 0.6457 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.9474, + "falseNegative": 8, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 72, + "recall": 0.9, + "total": 80, + "trueNegative": 0, + "truePositive": 72, + "wilsonPrecisionLower95": 0.9493 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 80 + }, + "release_negative_boundary": { + "intentsMeeting95Precision": 0, + "macroF1": 0.0, + "macroPrecision": 0.0, + "macroRecall": 0.0, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 80 + }, + "release_personal_boundary": { + "intentsMeeting95Precision": 1, + "macroF1": 0.0621, + "macroPrecision": 0.1111, + "macroRecall": 0.0431, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.5586, + "falseNegative": 49, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 31, + "recall": 0.3875, + "total": 80, + "trueNegative": 0, + "truePositive": 31, + "wilsonPrecisionLower95": 0.8897 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 4, + "precision": 0.0, + "predictedPositive": 4, + "recall": 0.0, + "total": 80, + "trueNegative": 76, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 80 + }, + "release_resolved_boundary": { + "intentsMeeting95Precision": 0, + "macroF1": 0.0, + "macroPrecision": 0.0, + "macroRecall": 0.0, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 19, + "precision": 0.0, + "predictedPositive": 19, + "recall": 0.0, + "total": 80, + "trueNegative": 61, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 80 + }, + "release_task_owner": { + "intentsMeeting95Precision": 2, + "macroF1": 0.1474, + "macroPrecision": 0.2222, + "macroRecall": 0.1319, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.9937, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 79, + "recall": 0.9875, + "total": 80, + "trueNegative": 0, + "truePositive": 79, + "wilsonPrecisionLower95": 0.9536 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.3333, + "falseNegative": 64, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 16, + "recall": 0.2, + "total": 80, + "trueNegative": 0, + "truePositive": 16, + "wilsonPrecisionLower95": 0.8064 + } + }, + "recordCount": 80 + }, + "release_task_polite_request": { + "intentsMeeting95Precision": 3, + "macroF1": 0.2698, + "macroPrecision": 0.3333, + "macroRecall": 0.2319, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.8652, + "falseNegative": 19, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 61, + "recall": 0.7625, + "total": 80, + "trueNegative": 0, + "truePositive": 61, + "wilsonPrecisionLower95": 0.9408 + }, + "replyableMessage": { + "f1": 0.9189, + "falseNegative": 12, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 68, + "recall": 0.85, + "total": 80, + "trueNegative": 0, + "truePositive": 68, + "wilsonPrecisionLower95": 0.9465 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.6441, + "falseNegative": 42, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 38, + "recall": 0.475, + "total": 80, + "trueNegative": 0, + "truePositive": 38, + "wilsonPrecisionLower95": 0.9082 + } + }, + "recordCount": 80 + } + }, + "intentsMeeting95Precision": 4, + "macroF1": 0.3209, + "macroPrecision": 0.5489, + "macroRecall": 0.2537, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 640, + "trueNegative": 640, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.1955, + "falseNegative": 214, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 26, + "recall": 0.1083, + "total": 640, + "trueNegative": 400, + "truePositive": 26, + "wilsonPrecisionLower95": 0.8713 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 640, + "trueNegative": 640, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.5586, + "falseNegative": 49, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 31, + "recall": 0.3875, + "total": 640, + "trueNegative": 560, + "truePositive": 31, + "wilsonPrecisionLower95": 0.8897 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 640, + "trueNegative": 640, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.7097, + "falseNegative": 72, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 88, + "recall": 0.55, + "total": 640, + "trueNegative": 480, + "truePositive": 88, + "wilsonPrecisionLower95": 0.9582 + }, + "replyableMessage": { + "f1": 0.9195, + "falseNegative": 40, + "falsePositive": 23, + "precision": 0.9399, + "predictedPositive": 383, + "recall": 0.9, + "total": 640, + "trueNegative": 217, + "truePositive": 360, + "wilsonPrecisionLower95": 0.9115 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 640, + "trueNegative": 640, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.5047, + "falseNegative": 106, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 54, + "recall": 0.3375, + "total": 640, + "trueNegative": 480, + "truePositive": 54, + "wilsonPrecisionLower95": 0.9336 + } + }, + "recordCount": 640 + } + }, + "generatedAtUnix": 1787826081, + "openTrainingRecords": 3845, + "overlapChecks": { + "ModelTraining/ClipboardSemantics/comprehensive-online-holdout-corpus.jsonl": 0, + "ModelTraining/ClipboardSemantics/random-holdout-corpus.jsonl": 0, + "ModelTraining/ClipboardSemantics/targeted-release-holdout-corpus.jsonl": 0 + }, + "platformLimitation": "Linux surrogate research only. Apple Create ML/Core ML artifacts must be retrained and tested on macOS before deployment.", + "releaseGatePassed": false, + "roundCount": 20, + "schemaVersion": 1, + "seed": 20260827, + "selectedConfiguration": { + "alpha": 2e-05, + "augmentation": "none", + "char_max": 5, + "char_min": 2, + "external_weight": 0.5700000000000001, + "hard_example_weight": 1.0, + "l1_ratio": 0.0, + "max_char_features": 45000, + "max_word_features": 22000, + "min_df": 2, + "pseudo_weight": 0.16499999999999998, + "round": 1, + "word_max": 2 + }, + "selectedRound": 1, + "selectedRoundsByIntent": { + "blessing": 19, + "complaint": 1, + "confirmationDecision": 11, + "followUpReminder": 14, + "invitation": 1, + "question": 1, + "replyableMessage": 1, + "scheduleNegotiation": 7, + "task": 1 + }, + "selectedThresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 180, + "recall": 1.0, + "total": 1485, + "trueNegative": 1305, + "truePositive": 180, + "wilsonPrecisionLower95": 0.9791 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9526, + "falseNegative": 9, + "falsePositive": 8, + "precision": 0.9553, + "predictedPositive": 179, + "recall": 0.95, + "total": 1485, + "trueNegative": 1297, + "truePositive": 171, + "wilsonPrecisionLower95": 0.9143 + }, + "threshold": 0.54 + } + }, + "metrics": { + "f1": 0.9764, + "falseNegative": 9, + "falsePositive": 8, + "precision": 0.9777, + "predictedPositive": 359, + "recall": 0.975, + "total": 2970, + "trueNegative": 2602, + "truePositive": 351, + "wilsonPrecisionLower95": 0.9567 + }, + "threshold": 0.54 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.931, + "falseNegative": 9, + "falsePositive": 3, + "precision": 0.9643, + "predictedPositive": 84, + "recall": 0.9, + "total": 1485, + "trueNegative": 1392, + "truePositive": 81, + "wilsonPrecisionLower95": 0.9002 + }, + "threshold": 0.97 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7222, + "falseNegative": 38, + "falsePositive": 2, + "precision": 0.963, + "predictedPositive": 54, + "recall": 0.5778, + "total": 1485, + "trueNegative": 1393, + "truePositive": 52, + "wilsonPrecisionLower95": 0.8746 + }, + "threshold": 0.86 + } + }, + "metrics": { + "f1": 0.8077, + "falseNegative": 54, + "falsePositive": 6, + "precision": 0.9545, + "predictedPositive": 132, + "recall": 0.7, + "total": 2970, + "trueNegative": 2784, + "truePositive": 126, + "wilsonPrecisionLower95": 0.9044 + }, + "threshold": 0.95 + }, + "followUpReminder": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8788, + "falseNegative": 35, + "falsePositive": 5, + "precision": 0.9667, + "predictedPositive": 150, + "recall": 0.8056, + "total": 1485, + "trueNegative": 1300, + "truePositive": 145, + "wilsonPrecisionLower95": 0.9243 + }, + "threshold": 0.845 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6113, + "falseNegative": 99, + "falsePositive": 4, + "precision": 0.9529, + "predictedPositive": 85, + "recall": 0.45, + "total": 1485, + "trueNegative": 1301, + "truePositive": 81, + "wilsonPrecisionLower95": 0.8852 + }, + "threshold": 0.97 + } + }, + "metrics": { + "f1": 0.7682, + "falseNegative": 128, + "falsePositive": 12, + "precision": 0.9508, + "predictedPositive": 244, + "recall": 0.6444, + "total": 2970, + "trueNegative": 2598, + "truePositive": 232, + "wilsonPrecisionLower95": 0.916 + }, + "threshold": 0.91 + }, + "invitation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8302, + "falseNegative": 24, + "falsePositive": 3, + "precision": 0.9565, + "predictedPositive": 69, + "recall": 0.7333, + "total": 1485, + "trueNegative": 1392, + "truePositive": 66, + "wilsonPrecisionLower95": 0.8798 + }, + "threshold": 0.935 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6765, + "falseNegative": 44, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 46, + "recall": 0.5111, + "total": 1485, + "trueNegative": 1395, + "truePositive": 46, + "wilsonPrecisionLower95": 0.9229 + }, + "threshold": 0.505 + } + }, + "metrics": { + "f1": 0.5476, + "falseNegative": 111, + "falsePositive": 3, + "precision": 0.9583, + "predictedPositive": 72, + "recall": 0.3833, + "total": 2970, + "trueNegative": 2787, + "truePositive": 69, + "wilsonPrecisionLower95": 0.8845 + }, + "threshold": 0.935 + }, + "question": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8445, + "falseNegative": 76, + "falsePositive": 12, + "precision": 0.9522, + "predictedPositive": 251, + "recall": 0.7587, + "total": 1485, + "trueNegative": 1158, + "truePositive": 239, + "wilsonPrecisionLower95": 0.9183 + }, + "threshold": 0.725 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8536, + "falseNegative": 76, + "falsePositive": 6, + "precision": 0.9755, + "predictedPositive": 245, + "recall": 0.7587, + "total": 1485, + "trueNegative": 1164, + "truePositive": 239, + "wilsonPrecisionLower95": 0.9476 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.8729, + "falseNegative": 122, + "falsePositive": 26, + "precision": 0.9513, + "predictedPositive": 534, + "recall": 0.8063, + "total": 2970, + "trueNegative": 2314, + "truePositive": 508, + "wilsonPrecisionLower95": 0.9296 + }, + "threshold": 0.515 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.948, + "falseNegative": 52, + "falsePositive": 46, + "precision": 0.951, + "predictedPositive": 939, + "recall": 0.945, + "total": 1485, + "trueNegative": 494, + "truePositive": 893, + "wilsonPrecisionLower95": 0.9353 + }, + "threshold": 0.7 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9424, + "falseNegative": 62, + "falsePositive": 46, + "precision": 0.9505, + "predictedPositive": 929, + "recall": 0.9344, + "total": 1485, + "trueNegative": 494, + "truePositive": 883, + "wilsonPrecisionLower95": 0.9346 + }, + "threshold": 0.695 + } + }, + "metrics": { + "f1": 0.9454, + "falseNegative": 115, + "falsePositive": 90, + "precision": 0.9517, + "predictedPositive": 1865, + "recall": 0.9392, + "total": 2970, + "trueNegative": 990, + "truePositive": 1775, + "wilsonPrecisionLower95": 0.9411 + }, + "threshold": 0.7 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 45, + "recall": 1.0, + "total": 1485, + "trueNegative": 1440, + "truePositive": 45, + "wilsonPrecisionLower95": 0.9213 + }, + "threshold": 0.66 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 45, + "recall": 1.0, + "total": 1485, + "trueNegative": 1440, + "truePositive": 45, + "wilsonPrecisionLower95": 0.9213 + }, + "threshold": 0.55 + } + }, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 90, + "recall": 1.0, + "total": 2970, + "trueNegative": 2880, + "truePositive": 90, + "wilsonPrecisionLower95": 0.9591 + }, + "threshold": 0.66 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8403, + "falseNegative": 89, + "falsePositive": 14, + "precision": 0.9509, + "predictedPositive": 285, + "recall": 0.7528, + "total": 1485, + "trueNegative": 1111, + "truePositive": 271, + "wilsonPrecisionLower95": 0.9192 + }, + "threshold": 0.835 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8278, + "falseNegative": 98, + "falsePositive": 11, + "precision": 0.9597, + "predictedPositive": 273, + "recall": 0.7278, + "total": 1485, + "trueNegative": 1114, + "truePositive": 262, + "wilsonPrecisionLower95": 0.9293 + }, + "threshold": 0.745 + } + }, + "metrics": { + "f1": 0.8637, + "falseNegative": 153, + "falsePositive": 26, + "precision": 0.9562, + "predictedPositive": 593, + "recall": 0.7875, + "total": 2970, + "trueNegative": 2224, + "truePositive": 567, + "wilsonPrecisionLower95": 0.9365 + }, + "threshold": 0.755 + } + }, + "selectionObjective": 0.6928, + "trainingPolicy": "No user clipboard data. Official partial labels plus deterministic label-preserving augmentation and three-round high-confidence self-training consensus. Frozen holdouts never enter training or threshold calibration." +} diff --git a/ModelTraining/ClipboardSemantics/IterativeResearch/EvidenceGateFineTune/macos-replay.json b/ModelTraining/ClipboardSemantics/IterativeResearch/EvidenceGateFineTune/macos-replay.json new file mode 100644 index 0000000..eaef67a --- /dev/null +++ b/ModelTraining/ClipboardSemantics/IterativeResearch/EvidenceGateFineTune/macos-replay.json @@ -0,0 +1,629 @@ +{ + "automaticPromotionAllowed": false, + "baseCorpus": "ModelTraining/ClipboardSemantics/clipboard_semantic_corpus.jsonl", + "configuration": { + "alpha": 2e-05, + "augmentation": "none", + "char_max": 5, + "char_min": 2, + "external_weight": 0.5700000000000001, + "hard_example_weight": 1.0, + "l1_ratio": 0.0, + "max_char_features": 45000, + "max_word_features": 22000, + "min_df": 2, + "pseudo_weight": 0.16499999999999998, + "round": 1, + "word_max": 2 + }, + "configurationsByIntent": { + "blessing": { + "alpha": 1.2e-06, + "augmentation": "numeric", + "char_max": 8, + "char_min": 3, + "external_weight": 0.93, + "hard_example_weight": 2.7, + "l1_ratio": 0.25, + "max_char_features": 95000, + "max_word_features": 45000, + "min_df": 1, + "pseudo_weight": 0.43499999999999994, + "round": 19, + "word_max": 3 + }, + "complaint": { + "alpha": 2e-05, + "augmentation": "none", + "char_max": 5, + "char_min": 2, + "external_weight": 0.5700000000000001, + "hard_example_weight": 1.0, + "l1_ratio": 0.0, + "max_char_features": 45000, + "max_word_features": 22000, + "min_df": 2, + "pseudo_weight": 0.16499999999999998, + "round": 1, + "word_max": 2 + }, + "confirmationDecision": { + "alpha": 5e-06, + "augmentation": "prefix", + "char_max": 6, + "char_min": 2, + "external_weight": 0.77, + "hard_example_weight": 1.8, + "l1_ratio": 0.1, + "max_char_features": 70000, + "max_word_features": 30000, + "min_df": 2, + "pseudo_weight": 0.31499999999999995, + "round": 11, + "word_max": 2 + }, + "followUpReminder": { + "alpha": 2.5e-06, + "augmentation": "prefix", + "char_max": 7, + "char_min": 3, + "external_weight": 0.8300000000000001, + "hard_example_weight": 2.2, + "l1_ratio": 0.2, + "max_char_features": 80000, + "max_word_features": 40000, + "min_df": 1, + "pseudo_weight": 0.36, + "round": 14, + "word_max": 3 + }, + "invitation": { + "alpha": 2e-05, + "augmentation": "none", + "char_max": 5, + "char_min": 2, + "external_weight": 0.5700000000000001, + "hard_example_weight": 1.0, + "l1_ratio": 0.0, + "max_char_features": 45000, + "max_word_features": 22000, + "min_df": 2, + "pseudo_weight": 0.16499999999999998, + "round": 1, + "word_max": 2 + }, + "question": { + "alpha": 2e-05, + "augmentation": "none", + "char_max": 5, + "char_min": 2, + "external_weight": 0.5700000000000001, + "hard_example_weight": 1.0, + "l1_ratio": 0.0, + "max_char_features": 45000, + "max_word_features": 22000, + "min_df": 2, + "pseudo_weight": 0.16499999999999998, + "round": 1, + "word_max": 2 + }, + "replyableMessage": { + "alpha": 2e-05, + "augmentation": "none", + "char_max": 5, + "char_min": 2, + "external_weight": 0.5700000000000001, + "hard_example_weight": 1.0, + "l1_ratio": 0.0, + "max_char_features": 45000, + "max_word_features": 22000, + "min_df": 2, + "pseudo_weight": 0.16499999999999998, + "round": 1, + "word_max": 2 + }, + "scheduleNegotiation": { + "alpha": 6e-06, + "augmentation": "punctuation", + "char_max": 6, + "char_min": 3, + "external_weight": 0.6900000000000001, + "hard_example_weight": 1.5, + "l1_ratio": 0.1, + "max_char_features": 65000, + "max_word_features": 32000, + "min_df": 2, + "pseudo_weight": 0.255, + "round": 7, + "word_max": 3 + }, + "task": { + "alpha": 2e-05, + "augmentation": "none", + "char_max": 5, + "char_min": 2, + "external_weight": 0.5700000000000001, + "hard_example_weight": 1.0, + "l1_ratio": 0.0, + "max_char_features": 45000, + "max_word_features": 22000, + "min_df": 2, + "pseudo_weight": 0.16499999999999998, + "round": 1, + "word_max": 2 + } + }, + "openCorpus": "ModelTraining/ClipboardSemantics/open-training-corpus.jsonl", + "reason": "Surrogate feature weights are not deployable NLModel assets and cannot bypass the existing macOS acceptance policy.", + "requiredCommands": [ + "python3 Scripts/clipboard_semantics/generate_open_training_corpus.py", + "xcrun swift Scripts/clipboard_semantics/train_models.swift --algorithms maxEnt --corpus ModelTraining/ClipboardSemantics/combined-training-corpus.jsonl", + "xcrun swift Scripts/clipboard_semantics/evaluate_random_holdout.swift --corpus ModelTraining/ClipboardSemantics/random-holdout-corpus.jsonl" + ], + "schemaVersion": 1, + "seed": 20260827, + "selectedRound": 1, + "selectedRoundsByIntent": { + "blessing": 19, + "complaint": 1, + "confirmationDecision": 11, + "followUpReminder": 14, + "invitation": 1, + "question": 1, + "replyableMessage": 1, + "scheduleNegotiation": 7, + "task": 1 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 180, + "recall": 1.0, + "total": 1485, + "trueNegative": 1305, + "truePositive": 180, + "wilsonPrecisionLower95": 0.9791 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9526, + "falseNegative": 9, + "falsePositive": 8, + "precision": 0.9553, + "predictedPositive": 179, + "recall": 0.95, + "total": 1485, + "trueNegative": 1297, + "truePositive": 171, + "wilsonPrecisionLower95": 0.9143 + }, + "threshold": 0.54 + } + }, + "metrics": { + "f1": 0.9764, + "falseNegative": 9, + "falsePositive": 8, + "precision": 0.9777, + "predictedPositive": 359, + "recall": 0.975, + "total": 2970, + "trueNegative": 2602, + "truePositive": 351, + "wilsonPrecisionLower95": 0.9567 + }, + "threshold": 0.54 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.931, + "falseNegative": 9, + "falsePositive": 3, + "precision": 0.9643, + "predictedPositive": 84, + "recall": 0.9, + "total": 1485, + "trueNegative": 1392, + "truePositive": 81, + "wilsonPrecisionLower95": 0.9002 + }, + "threshold": 0.97 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7222, + "falseNegative": 38, + "falsePositive": 2, + "precision": 0.963, + "predictedPositive": 54, + "recall": 0.5778, + "total": 1485, + "trueNegative": 1393, + "truePositive": 52, + "wilsonPrecisionLower95": 0.8746 + }, + "threshold": 0.86 + } + }, + "metrics": { + "f1": 0.8077, + "falseNegative": 54, + "falsePositive": 6, + "precision": 0.9545, + "predictedPositive": 132, + "recall": 0.7, + "total": 2970, + "trueNegative": 2784, + "truePositive": 126, + "wilsonPrecisionLower95": 0.9044 + }, + "threshold": 0.95 + }, + "followUpReminder": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8788, + "falseNegative": 35, + "falsePositive": 5, + "precision": 0.9667, + "predictedPositive": 150, + "recall": 0.8056, + "total": 1485, + "trueNegative": 1300, + "truePositive": 145, + "wilsonPrecisionLower95": 0.9243 + }, + "threshold": 0.845 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6113, + "falseNegative": 99, + "falsePositive": 4, + "precision": 0.9529, + "predictedPositive": 85, + "recall": 0.45, + "total": 1485, + "trueNegative": 1301, + "truePositive": 81, + "wilsonPrecisionLower95": 0.8852 + }, + "threshold": 0.97 + } + }, + "metrics": { + "f1": 0.7682, + "falseNegative": 128, + "falsePositive": 12, + "precision": 0.9508, + "predictedPositive": 244, + "recall": 0.6444, + "total": 2970, + "trueNegative": 2598, + "truePositive": 232, + "wilsonPrecisionLower95": 0.916 + }, + "threshold": 0.91 + }, + "invitation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8302, + "falseNegative": 24, + "falsePositive": 3, + "precision": 0.9565, + "predictedPositive": 69, + "recall": 0.7333, + "total": 1485, + "trueNegative": 1392, + "truePositive": 66, + "wilsonPrecisionLower95": 0.8798 + }, + "threshold": 0.935 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6765, + "falseNegative": 44, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 46, + "recall": 0.5111, + "total": 1485, + "trueNegative": 1395, + "truePositive": 46, + "wilsonPrecisionLower95": 0.9229 + }, + "threshold": 0.505 + } + }, + "metrics": { + "f1": 0.5476, + "falseNegative": 111, + "falsePositive": 3, + "precision": 0.9583, + "predictedPositive": 72, + "recall": 0.3833, + "total": 2970, + "trueNegative": 2787, + "truePositive": 69, + "wilsonPrecisionLower95": 0.8845 + }, + "threshold": 0.935 + }, + "question": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8445, + "falseNegative": 76, + "falsePositive": 12, + "precision": 0.9522, + "predictedPositive": 251, + "recall": 0.7587, + "total": 1485, + "trueNegative": 1158, + "truePositive": 239, + "wilsonPrecisionLower95": 0.9183 + }, + "threshold": 0.725 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8536, + "falseNegative": 76, + "falsePositive": 6, + "precision": 0.9755, + "predictedPositive": 245, + "recall": 0.7587, + "total": 1485, + "trueNegative": 1164, + "truePositive": 239, + "wilsonPrecisionLower95": 0.9476 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.8729, + "falseNegative": 122, + "falsePositive": 26, + "precision": 0.9513, + "predictedPositive": 534, + "recall": 0.8063, + "total": 2970, + "trueNegative": 2314, + "truePositive": 508, + "wilsonPrecisionLower95": 0.9296 + }, + "threshold": 0.515 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.948, + "falseNegative": 52, + "falsePositive": 46, + "precision": 0.951, + "predictedPositive": 939, + "recall": 0.945, + "total": 1485, + "trueNegative": 494, + "truePositive": 893, + "wilsonPrecisionLower95": 0.9353 + }, + "threshold": 0.7 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9424, + "falseNegative": 62, + "falsePositive": 46, + "precision": 0.9505, + "predictedPositive": 929, + "recall": 0.9344, + "total": 1485, + "trueNegative": 494, + "truePositive": 883, + "wilsonPrecisionLower95": 0.9346 + }, + "threshold": 0.695 + } + }, + "metrics": { + "f1": 0.9454, + "falseNegative": 115, + "falsePositive": 90, + "precision": 0.9517, + "predictedPositive": 1865, + "recall": 0.9392, + "total": 2970, + "trueNegative": 990, + "truePositive": 1775, + "wilsonPrecisionLower95": 0.9411 + }, + "threshold": 0.7 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 45, + "recall": 1.0, + "total": 1485, + "trueNegative": 1440, + "truePositive": 45, + "wilsonPrecisionLower95": 0.9213 + }, + "threshold": 0.66 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 45, + "recall": 1.0, + "total": 1485, + "trueNegative": 1440, + "truePositive": 45, + "wilsonPrecisionLower95": 0.9213 + }, + "threshold": 0.55 + } + }, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 90, + "recall": 1.0, + "total": 2970, + "trueNegative": 2880, + "truePositive": 90, + "wilsonPrecisionLower95": 0.9591 + }, + "threshold": 0.66 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8403, + "falseNegative": 89, + "falsePositive": 14, + "precision": 0.9509, + "predictedPositive": 285, + "recall": 0.7528, + "total": 1485, + "trueNegative": 1111, + "truePositive": 271, + "wilsonPrecisionLower95": 0.9192 + }, + "threshold": 0.835 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8278, + "falseNegative": 98, + "falsePositive": 11, + "precision": 0.9597, + "predictedPositive": 273, + "recall": 0.7278, + "total": 1485, + "trueNegative": 1114, + "truePositive": 262, + "wilsonPrecisionLower95": 0.9293 + }, + "threshold": 0.745 + } + }, + "metrics": { + "f1": 0.8637, + "falseNegative": 153, + "falsePositive": 26, + "precision": 0.9562, + "predictedPositive": 593, + "recall": 0.7875, + "total": 2970, + "trueNegative": 2224, + "truePositive": 567, + "wilsonPrecisionLower95": 0.9365 + }, + "threshold": 0.755 + } + } +} diff --git a/ModelTraining/ClipboardSemantics/IterativeResearch/EvidenceGateFineTune/rounds.json b/ModelTraining/ClipboardSemantics/IterativeResearch/EvidenceGateFineTune/rounds.json new file mode 100644 index 0000000..f0b25a6 --- /dev/null +++ b/ModelTraining/ClipboardSemantics/IterativeResearch/EvidenceGateFineTune/rounds.json @@ -0,0 +1,14831 @@ +[ + { + "configuration": { + "alpha": 2e-05, + "augmentation": "none", + "char_max": 5, + "char_min": 2, + "external_weight": 0.5700000000000001, + "hard_example_weight": 1.0, + "l1_ratio": 0.0, + "max_char_features": 45000, + "max_word_features": 22000, + "min_df": 2, + "pseudo_weight": 0.16499999999999998, + "round": 1, + "word_max": 2 + }, + "featureCount": 56451, + "pseudoLabelCounts": {}, + "round": 1, + "selectionObjective": 0.6928, + "syntheticRegression": { + "intentsMeeting95Precision": 7, + "macroF1": 0.5923, + "macroPrecision": 0.9767, + "macroRecall": 0.4748, + "minimumPrecision": 0.813, + "perIntent": { + "blessing": { + "f1": 0.125, + "falseNegative": 84, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 6, + "recall": 0.0667, + "total": 2970, + "trueNegative": 2880, + "truePositive": 6, + "wilsonPrecisionLower95": 0.6097 + }, + "complaint": { + "f1": 0.4279, + "falseNegative": 262, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 98, + "recall": 0.2722, + "total": 2970, + "trueNegative": 2610, + "truePositive": 98, + "wilsonPrecisionLower95": 0.9623 + }, + "confirmationDecision": { + "f1": 0.6601, + "falseNegative": 80, + "falsePositive": 23, + "precision": 0.813, + "predictedPositive": 123, + "recall": 0.5556, + "total": 2970, + "trueNegative": 2767, + "truePositive": 100, + "wilsonPrecisionLower95": 0.735 + }, + "followUpReminder": { + "f1": 0.4382, + "falseNegative": 259, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 101, + "recall": 0.2806, + "total": 2970, + "trueNegative": 2610, + "truePositive": 101, + "wilsonPrecisionLower95": 0.9634 + }, + "invitation": { + "f1": 0.481, + "falseNegative": 123, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 57, + "recall": 0.3167, + "total": 2970, + "trueNegative": 2790, + "truePositive": 57, + "wilsonPrecisionLower95": 0.9369 + }, + "question": { + "f1": 0.8545, + "falseNegative": 160, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 470, + "recall": 0.746, + "total": 2970, + "trueNegative": 2340, + "truePositive": 470, + "wilsonPrecisionLower95": 0.9919 + }, + "replyableMessage": { + "f1": 0.9465, + "falseNegative": 156, + "falsePositive": 40, + "precision": 0.9775, + "predictedPositive": 1774, + "recall": 0.9175, + "total": 2970, + "trueNegative": 1040, + "truePositive": 1734, + "wilsonPrecisionLower95": 0.9694 + }, + "scheduleNegotiation": { + "f1": 0.8535, + "falseNegative": 23, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 67, + "recall": 0.7444, + "total": 2970, + "trueNegative": 2880, + "truePositive": 67, + "wilsonPrecisionLower95": 0.9458 + }, + "task": { + "f1": 0.544, + "falseNegative": 451, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 269, + "recall": 0.3736, + "total": 2970, + "trueNegative": 2250, + "truePositive": 269, + "wilsonPrecisionLower95": 0.9859 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9318, + "falseNegative": 4, + "falsePositive": 2, + "precision": 0.9535, + "predictedPositive": 43, + "recall": 0.9111, + "total": 1485, + "trueNegative": 1438, + "truePositive": 41, + "wilsonPrecisionLower95": 0.8454 + }, + "threshold": 0.915 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7568, + "falseNegative": 17, + "falsePositive": 1, + "precision": 0.9655, + "predictedPositive": 29, + "recall": 0.6222, + "total": 1485, + "trueNegative": 1439, + "truePositive": 28, + "wilsonPrecisionLower95": 0.8282 + }, + "threshold": 0.9 + } + }, + "metrics": { + "f1": 0.8375, + "falseNegative": 23, + "falsePositive": 3, + "precision": 0.9571, + "predictedPositive": 70, + "recall": 0.7444, + "total": 2970, + "trueNegative": 2877, + "truePositive": 67, + "wilsonPrecisionLower95": 0.8814 + }, + "threshold": 0.915 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 180, + "recall": 1.0, + "total": 1485, + "trueNegative": 1305, + "truePositive": 180, + "wilsonPrecisionLower95": 0.9791 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9526, + "falseNegative": 9, + "falsePositive": 8, + "precision": 0.9553, + "predictedPositive": 179, + "recall": 0.95, + "total": 1485, + "trueNegative": 1297, + "truePositive": 171, + "wilsonPrecisionLower95": 0.9143 + }, + "threshold": 0.54 + } + }, + "metrics": { + "f1": 0.9764, + "falseNegative": 9, + "falsePositive": 8, + "precision": 0.9777, + "predictedPositive": 359, + "recall": 0.975, + "total": 2970, + "trueNegative": 2602, + "truePositive": 351, + "wilsonPrecisionLower95": 0.9567 + }, + "threshold": 0.54 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8848, + "falseNegative": 17, + "falsePositive": 2, + "precision": 0.9733, + "predictedPositive": 75, + "recall": 0.8111, + "total": 1485, + "trueNegative": 1393, + "truePositive": 73, + "wilsonPrecisionLower95": 0.9079 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7947, + "falseNegative": 30, + "falsePositive": 1, + "precision": 0.9836, + "predictedPositive": 61, + "recall": 0.6667, + "total": 1485, + "trueNegative": 1394, + "truePositive": 60, + "wilsonPrecisionLower95": 0.9128 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.8418, + "falseNegative": 47, + "falsePositive": 3, + "precision": 0.9779, + "predictedPositive": 136, + "recall": 0.7389, + "total": 2970, + "trueNegative": 2787, + "truePositive": 133, + "wilsonPrecisionLower95": 0.9372 + }, + "threshold": 0.5 + }, + "followUpReminder": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8554, + "falseNegative": 41, + "falsePositive": 6, + "precision": 0.9586, + "predictedPositive": 145, + "recall": 0.7722, + "total": 1485, + "trueNegative": 1299, + "truePositive": 139, + "wilsonPrecisionLower95": 0.9127 + }, + "threshold": 0.83 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8354, + "falseNegative": 48, + "falsePositive": 4, + "precision": 0.9706, + "predictedPositive": 136, + "recall": 0.7333, + "total": 1485, + "trueNegative": 1301, + "truePositive": 132, + "wilsonPrecisionLower95": 0.9268 + }, + "threshold": 0.505 + } + }, + "metrics": { + "f1": 0.7994, + "falseNegative": 113, + "falsePositive": 11, + "precision": 0.9574, + "predictedPositive": 258, + "recall": 0.6861, + "total": 2970, + "trueNegative": 2599, + "truePositive": 247, + "wilsonPrecisionLower95": 0.9253 + }, + "threshold": 0.815 + }, + "invitation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8302, + "falseNegative": 24, + "falsePositive": 3, + "precision": 0.9565, + "predictedPositive": 69, + "recall": 0.7333, + "total": 1485, + "trueNegative": 1392, + "truePositive": 66, + "wilsonPrecisionLower95": 0.8798 + }, + "threshold": 0.935 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6765, + "falseNegative": 44, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 46, + "recall": 0.5111, + "total": 1485, + "trueNegative": 1395, + "truePositive": 46, + "wilsonPrecisionLower95": 0.9229 + }, + "threshold": 0.505 + } + }, + "metrics": { + "f1": 0.5476, + "falseNegative": 111, + "falsePositive": 3, + "precision": 0.9583, + "predictedPositive": 72, + "recall": 0.3833, + "total": 2970, + "trueNegative": 2787, + "truePositive": 69, + "wilsonPrecisionLower95": 0.8845 + }, + "threshold": 0.935 + }, + "question": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8445, + "falseNegative": 76, + "falsePositive": 12, + "precision": 0.9522, + "predictedPositive": 251, + "recall": 0.7587, + "total": 1485, + "trueNegative": 1158, + "truePositive": 239, + "wilsonPrecisionLower95": 0.9183 + }, + "threshold": 0.725 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8536, + "falseNegative": 76, + "falsePositive": 6, + "precision": 0.9755, + "predictedPositive": 245, + "recall": 0.7587, + "total": 1485, + "trueNegative": 1164, + "truePositive": 239, + "wilsonPrecisionLower95": 0.9476 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.8729, + "falseNegative": 122, + "falsePositive": 26, + "precision": 0.9513, + "predictedPositive": 534, + "recall": 0.8063, + "total": 2970, + "trueNegative": 2314, + "truePositive": 508, + "wilsonPrecisionLower95": 0.9296 + }, + "threshold": 0.515 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.948, + "falseNegative": 52, + "falsePositive": 46, + "precision": 0.951, + "predictedPositive": 939, + "recall": 0.945, + "total": 1485, + "trueNegative": 494, + "truePositive": 893, + "wilsonPrecisionLower95": 0.9353 + }, + "threshold": 0.7 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9424, + "falseNegative": 62, + "falsePositive": 46, + "precision": 0.9505, + "predictedPositive": 929, + "recall": 0.9344, + "total": 1485, + "trueNegative": 494, + "truePositive": 883, + "wilsonPrecisionLower95": 0.9346 + }, + "threshold": 0.695 + } + }, + "metrics": { + "f1": 0.9454, + "falseNegative": 115, + "falsePositive": 90, + "precision": 0.9517, + "predictedPositive": 1865, + "recall": 0.9392, + "total": 2970, + "trueNegative": 990, + "truePositive": 1775, + "wilsonPrecisionLower95": 0.9411 + }, + "threshold": 0.7 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 45, + "recall": 1.0, + "total": 1485, + "trueNegative": 1440, + "truePositive": 45, + "wilsonPrecisionLower95": 0.9213 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.967, + "falseNegative": 1, + "falsePositive": 2, + "precision": 0.9565, + "predictedPositive": 46, + "recall": 0.9778, + "total": 1485, + "trueNegative": 1438, + "truePositive": 44, + "wilsonPrecisionLower95": 0.8547 + }, + "threshold": 0.93 + } + }, + "metrics": { + "f1": 0.978, + "falseNegative": 1, + "falsePositive": 3, + "precision": 0.9674, + "predictedPositive": 92, + "recall": 0.9889, + "total": 2970, + "trueNegative": 2877, + "truePositive": 89, + "wilsonPrecisionLower95": 0.9085 + }, + "threshold": 0.91 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8403, + "falseNegative": 89, + "falsePositive": 14, + "precision": 0.9509, + "predictedPositive": 285, + "recall": 0.7528, + "total": 1485, + "trueNegative": 1111, + "truePositive": 271, + "wilsonPrecisionLower95": 0.9192 + }, + "threshold": 0.835 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8278, + "falseNegative": 98, + "falsePositive": 11, + "precision": 0.9597, + "predictedPositive": 273, + "recall": 0.7278, + "total": 1485, + "trueNegative": 1114, + "truePositive": 262, + "wilsonPrecisionLower95": 0.9293 + }, + "threshold": 0.745 + } + }, + "metrics": { + "f1": 0.8637, + "falseNegative": 153, + "falsePositive": 26, + "precision": 0.9562, + "predictedPositive": 593, + "recall": 0.7875, + "total": 2970, + "trueNegative": 2224, + "truePositive": 567, + "wilsonPrecisionLower95": 0.9365 + }, + "threshold": 0.755 + } + }, + "trainingRecords": 15345, + "validation": { + "intentsMeeting95Precision": 8, + "macroF1": 0.559, + "macroPrecision": 0.9784, + "macroRecall": 0.4293, + "minimumPrecision": 0.9412, + "perIntent": { + "blessing": { + "f1": 0.3636, + "falseNegative": 70, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 20, + "recall": 0.2222, + "total": 2970, + "trueNegative": 2880, + "truePositive": 20, + "wilsonPrecisionLower95": 0.8389 + }, + "complaint": { + "f1": 0.5542, + "falseNegative": 222, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 138, + "recall": 0.3833, + "total": 2970, + "trueNegative": 2610, + "truePositive": 138, + "wilsonPrecisionLower95": 0.9729 + }, + "confirmationDecision": { + "f1": 0.6023, + "falseNegative": 102, + "falsePositive": 1, + "precision": 0.9873, + "predictedPositive": 79, + "recall": 0.4333, + "total": 2970, + "trueNegative": 2789, + "truePositive": 78, + "wilsonPrecisionLower95": 0.9317 + }, + "followUpReminder": { + "f1": 0.2884, + "falseNegative": 299, + "falsePositive": 2, + "precision": 0.9683, + "predictedPositive": 63, + "recall": 0.1694, + "total": 2970, + "trueNegative": 2608, + "truePositive": 61, + "wilsonPrecisionLower95": 0.8914 + }, + "invitation": { + "f1": 0.3256, + "falseNegative": 145, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 35, + "recall": 0.1944, + "total": 2970, + "trueNegative": 2790, + "truePositive": 35, + "wilsonPrecisionLower95": 0.9011 + }, + "question": { + "f1": 0.8386, + "falseNegative": 165, + "falsePositive": 14, + "precision": 0.9708, + "predictedPositive": 479, + "recall": 0.7381, + "total": 2970, + "trueNegative": 2326, + "truePositive": 465, + "wilsonPrecisionLower95": 0.9515 + }, + "replyableMessage": { + "f1": 0.9452, + "falseNegative": 114, + "falsePositive": 92, + "precision": 0.9507, + "predictedPositive": 1868, + "recall": 0.9397, + "total": 2970, + "trueNegative": 988, + "truePositive": 1776, + "wilsonPrecisionLower95": 0.94 + }, + "scheduleNegotiation": { + "f1": 0.5161, + "falseNegative": 58, + "falsePositive": 2, + "precision": 0.9412, + "predictedPositive": 34, + "recall": 0.3556, + "total": 2970, + "trueNegative": 2878, + "truePositive": 32, + "wilsonPrecisionLower95": 0.8091 + }, + "task": { + "f1": 0.5969, + "falseNegative": 412, + "falsePositive": 4, + "precision": 0.9872, + "predictedPositive": 312, + "recall": 0.4278, + "total": 2970, + "trueNegative": 2246, + "truePositive": 308, + "wilsonPrecisionLower95": 0.9675 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 1.5e-05, + "augmentation": "none", + "char_max": 5, + "char_min": 3, + "external_weight": 0.5900000000000001, + "hard_example_weight": 1.0, + "l1_ratio": 0.0, + "max_char_features": 50000, + "max_word_features": 25000, + "min_df": 2, + "pseudo_weight": 0.18, + "round": 2, + "word_max": 2 + }, + "featureCount": 61451, + "pseudoLabelCounts": {}, + "round": 2, + "selectionObjective": 0.6707, + "syntheticRegression": { + "intentsMeeting95Precision": 8, + "macroF1": 0.5397, + "macroPrecision": 0.9762, + "macroRecall": 0.423, + "minimumPrecision": 0.8145, + "perIntent": { + "blessing": { + "f1": 0.2178, + "falseNegative": 79, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 11, + "recall": 0.1222, + "total": 2970, + "trueNegative": 2880, + "truePositive": 11, + "wilsonPrecisionLower95": 0.7412 + }, + "complaint": { + "f1": 0.4071, + "falseNegative": 268, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 92, + "recall": 0.2556, + "total": 2970, + "trueNegative": 2610, + "truePositive": 92, + "wilsonPrecisionLower95": 0.9599 + }, + "confirmationDecision": { + "f1": 0.6645, + "falseNegative": 79, + "falsePositive": 23, + "precision": 0.8145, + "predictedPositive": 124, + "recall": 0.5611, + "total": 2970, + "trueNegative": 2767, + "truePositive": 101, + "wilsonPrecisionLower95": 0.737 + }, + "followUpReminder": { + "f1": 0.3857, + "falseNegative": 274, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 86, + "recall": 0.2389, + "total": 2970, + "trueNegative": 2610, + "truePositive": 86, + "wilsonPrecisionLower95": 0.9572 + }, + "invitation": { + "f1": 0.2, + "falseNegative": 160, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 20, + "recall": 0.1111, + "total": 2970, + "trueNegative": 2790, + "truePositive": 20, + "wilsonPrecisionLower95": 0.8389 + }, + "question": { + "f1": 0.8046, + "falseNegative": 206, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 424, + "recall": 0.673, + "total": 2970, + "trueNegative": 2340, + "truePositive": 424, + "wilsonPrecisionLower95": 0.991 + }, + "replyableMessage": { + "f1": 0.9038, + "falseNegative": 293, + "falsePositive": 47, + "precision": 0.9714, + "predictedPositive": 1644, + "recall": 0.845, + "total": 2970, + "trueNegative": 1033, + "truePositive": 1597, + "wilsonPrecisionLower95": 0.9622 + }, + "scheduleNegotiation": { + "f1": 0.8387, + "falseNegative": 25, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 65, + "recall": 0.7222, + "total": 2970, + "trueNegative": 2880, + "truePositive": 65, + "wilsonPrecisionLower95": 0.9442 + }, + "task": { + "f1": 0.4348, + "falseNegative": 520, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 200, + "recall": 0.2778, + "total": 2970, + "trueNegative": 2250, + "truePositive": 200, + "wilsonPrecisionLower95": 0.9812 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9048, + "falseNegative": 7, + "falsePositive": 1, + "precision": 0.9744, + "predictedPositive": 39, + "recall": 0.8444, + "total": 1485, + "trueNegative": 1439, + "truePositive": 38, + "wilsonPrecisionLower95": 0.8682 + }, + "threshold": 0.59 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 180, + "recall": 1.0, + "total": 1485, + "trueNegative": 1305, + "truePositive": 180, + "wilsonPrecisionLower95": 0.9791 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9226, + "falseNegative": 19, + "falsePositive": 8, + "precision": 0.9527, + "predictedPositive": 169, + "recall": 0.8944, + "total": 1485, + "trueNegative": 1297, + "truePositive": 161, + "wilsonPrecisionLower95": 0.9094 + }, + "threshold": 0.555 + } + }, + "metrics": { + "f1": 0.9649, + "falseNegative": 16, + "falsePositive": 9, + "precision": 0.9745, + "predictedPositive": 353, + "recall": 0.9556, + "total": 2970, + "trueNegative": 2601, + "truePositive": 344, + "wilsonPrecisionLower95": 0.9523 + }, + "threshold": 0.5 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.954, + "falseNegative": 7, + "falsePositive": 1, + "precision": 0.9881, + "predictedPositive": 84, + "recall": 0.9222, + "total": 1485, + "trueNegative": 1394, + "truePositive": 83, + "wilsonPrecisionLower95": 0.9356 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7183, + "falseNegative": 39, + "falsePositive": 1, + "precision": 0.9808, + "predictedPositive": 52, + "recall": 0.5667, + "total": 1485, + "trueNegative": 1394, + "truePositive": 51, + "wilsonPrecisionLower95": 0.8988 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.8481, + "falseNegative": 46, + "falsePositive": 2, + "precision": 0.9853, + "predictedPositive": 136, + "recall": 0.7444, + "total": 2970, + "trueNegative": 2788, + "truePositive": 134, + "wilsonPrecisionLower95": 0.948 + }, + "threshold": 0.5 + }, + "followUpReminder": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8328, + "falseNegative": 48, + "falsePositive": 5, + "precision": 0.9635, + "predictedPositive": 137, + "recall": 0.7333, + "total": 1485, + "trueNegative": 1300, + "truePositive": 132, + "wilsonPrecisionLower95": 0.9174 + }, + "threshold": 0.91 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6879, + "falseNegative": 83, + "falsePositive": 5, + "precision": 0.951, + "predictedPositive": 102, + "recall": 0.5389, + "total": 1485, + "trueNegative": 1300, + "truePositive": 97, + "wilsonPrecisionLower95": 0.8903 + }, + "threshold": 0.765 + } + }, + "metrics": { + "f1": 0.6243, + "falseNegative": 193, + "falsePositive": 8, + "precision": 0.9543, + "predictedPositive": 175, + "recall": 0.4639, + "total": 2970, + "trueNegative": 2602, + "truePositive": 167, + "wilsonPrecisionLower95": 0.9124 + }, + "threshold": 0.905 + }, + "invitation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9059, + "falseNegative": 13, + "falsePositive": 3, + "precision": 0.9625, + "predictedPositive": 80, + "recall": 0.8556, + "total": 1485, + "trueNegative": 1392, + "truePositive": 77, + "wilsonPrecisionLower95": 0.8955 + }, + "threshold": 0.86 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.6113, + "falseNegative": 99, + "falsePositive": 4, + "precision": 0.9529, + "predictedPositive": 85, + "recall": 0.45, + "total": 2970, + "trueNegative": 2786, + "truePositive": 81, + "wilsonPrecisionLower95": 0.8852 + }, + "threshold": 0.855 + }, + "question": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8505, + "falseNegative": 76, + "falsePositive": 8, + "precision": 0.9676, + "predictedPositive": 247, + "recall": 0.7587, + "total": 1485, + "trueNegative": 1162, + "truePositive": 239, + "wilsonPrecisionLower95": 0.9374 + }, + "threshold": 0.705 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7308, + "falseNegative": 129, + "falsePositive": 8, + "precision": 0.9588, + "predictedPositive": 194, + "recall": 0.5905, + "total": 1485, + "trueNegative": 1162, + "truePositive": 186, + "wilsonPrecisionLower95": 0.9208 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.7813, + "falseNegative": 212, + "falsePositive": 22, + "precision": 0.95, + "predictedPositive": 440, + "recall": 0.6635, + "total": 2970, + "trueNegative": 2318, + "truePositive": 418, + "wilsonPrecisionLower95": 0.9255 + }, + "threshold": 0.57 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9629, + "falseNegative": 23, + "falsePositive": 48, + "precision": 0.9505, + "predictedPositive": 970, + "recall": 0.9757, + "total": 1485, + "trueNegative": 492, + "truePositive": 922, + "wilsonPrecisionLower95": 0.935 + }, + "threshold": 0.62 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8526, + "falseNegative": 216, + "falsePositive": 36, + "precision": 0.9529, + "predictedPositive": 765, + "recall": 0.7714, + "total": 1485, + "trueNegative": 504, + "truePositive": 729, + "wilsonPrecisionLower95": 0.9355 + }, + "threshold": 0.84 + } + }, + "metrics": { + "f1": 0.9224, + "falseNegative": 197, + "falsePositive": 88, + "precision": 0.9506, + "predictedPositive": 1781, + "recall": 0.8958, + "total": 2970, + "trueNegative": 992, + "truePositive": 1693, + "wilsonPrecisionLower95": 0.9395 + }, + "threshold": 0.775 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9783, + "falseNegative": 0, + "falsePositive": 2, + "precision": 0.9574, + "predictedPositive": 47, + "recall": 1.0, + "total": 1485, + "trueNegative": 1438, + "truePositive": 45, + "wilsonPrecisionLower95": 0.8575 + }, + "threshold": 0.765 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9778, + "falseNegative": 1, + "falsePositive": 1, + "precision": 0.9778, + "predictedPositive": 45, + "recall": 0.9778, + "total": 1485, + "trueNegative": 1439, + "truePositive": 44, + "wilsonPrecisionLower95": 0.8843 + }, + "threshold": 0.855 + } + }, + "metrics": { + "f1": 0.978, + "falseNegative": 1, + "falsePositive": 3, + "precision": 0.9674, + "predictedPositive": 92, + "recall": 0.9889, + "total": 2970, + "trueNegative": 2877, + "truePositive": 89, + "wilsonPrecisionLower95": 0.9085 + }, + "threshold": 0.855 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8541, + "falseNegative": 82, + "falsePositive": 13, + "precision": 0.9553, + "predictedPositive": 291, + "recall": 0.7722, + "total": 1485, + "trueNegative": 1112, + "truePositive": 278, + "wilsonPrecisionLower95": 0.9251 + }, + "threshold": 0.84 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.5418, + "falseNegative": 224, + "falsePositive": 6, + "precision": 0.9577, + "predictedPositive": 142, + "recall": 0.3778, + "total": 1485, + "trueNegative": 1119, + "truePositive": 136, + "wilsonPrecisionLower95": 0.9109 + }, + "threshold": 0.91 + } + }, + "metrics": { + "f1": 0.7246, + "falseNegative": 299, + "falsePositive": 21, + "precision": 0.9525, + "predictedPositive": 442, + "recall": 0.5847, + "total": 2970, + "trueNegative": 2229, + "truePositive": 421, + "wilsonPrecisionLower95": 0.9285 + }, + "threshold": 0.88 + } + }, + "trainingRecords": 15345, + "validation": { + "intentsMeeting95Precision": 9, + "macroF1": 0.5203, + "macroPrecision": 0.9875, + "macroRecall": 0.3893, + "minimumPrecision": 0.9516, + "perIntent": { + "blessing": { + "f1": 0.2524, + "falseNegative": 77, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 13, + "recall": 0.1444, + "total": 2970, + "trueNegative": 2880, + "truePositive": 13, + "wilsonPrecisionLower95": 0.7719 + }, + "complaint": { + "f1": 0.5455, + "falseNegative": 225, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 135, + "recall": 0.375, + "total": 2970, + "trueNegative": 2610, + "truePositive": 135, + "wilsonPrecisionLower95": 0.9723 + }, + "confirmationDecision": { + "f1": 0.6312, + "falseNegative": 97, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 83, + "recall": 0.4611, + "total": 2970, + "trueNegative": 2790, + "truePositive": 83, + "wilsonPrecisionLower95": 0.9558 + }, + "followUpReminder": { + "f1": 0.2476, + "falseNegative": 309, + "falsePositive": 1, + "precision": 0.9808, + "predictedPositive": 52, + "recall": 0.1417, + "total": 2970, + "trueNegative": 2609, + "truePositive": 51, + "wilsonPrecisionLower95": 0.8988 + }, + "invitation": { + "f1": 0.3019, + "falseNegative": 148, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 32, + "recall": 0.1778, + "total": 2970, + "trueNegative": 2790, + "truePositive": 32, + "wilsonPrecisionLower95": 0.8928 + }, + "question": { + "f1": 0.7851, + "falseNegative": 219, + "falsePositive": 6, + "precision": 0.9856, + "predictedPositive": 417, + "recall": 0.6524, + "total": 2970, + "trueNegative": 2334, + "truePositive": 411, + "wilsonPrecisionLower95": 0.969 + }, + "replyableMessage": { + "f1": 0.9109, + "falseNegative": 239, + "falsePositive": 84, + "precision": 0.9516, + "predictedPositive": 1735, + "recall": 0.8735, + "total": 2970, + "trueNegative": 996, + "truePositive": 1651, + "wilsonPrecisionLower95": 0.9404 + }, + "scheduleNegotiation": { + "f1": 0.5203, + "falseNegative": 58, + "falsePositive": 1, + "precision": 0.9697, + "predictedPositive": 33, + "recall": 0.3556, + "total": 2970, + "trueNegative": 2879, + "truePositive": 32, + "wilsonPrecisionLower95": 0.8468 + }, + "task": { + "f1": 0.4874, + "falseNegative": 488, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 232, + "recall": 0.3222, + "total": 2970, + "trueNegative": 2250, + "truePositive": 232, + "wilsonPrecisionLower95": 0.9837 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 1e-05, + "augmentation": "none", + "char_max": 6, + "char_min": 2, + "external_weight": 0.6100000000000001, + "hard_example_weight": 1.2, + "l1_ratio": 0.05, + "max_char_features": 55000, + "max_word_features": 25000, + "min_df": 2, + "pseudo_weight": 0.195, + "round": 3, + "word_max": 2 + }, + "featureCount": 66451, + "pseudoLabelCounts": {}, + "round": 3, + "selectionObjective": 0.6612, + "syntheticRegression": { + "intentsMeeting95Precision": 7, + "macroF1": 0.5557, + "macroPrecision": 0.9751, + "macroRecall": 0.4386, + "minimumPrecision": 0.8115, + "perIntent": { + "blessing": { + "f1": 0.0645, + "falseNegative": 87, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 3, + "recall": 0.0333, + "total": 2970, + "trueNegative": 2880, + "truePositive": 3, + "wilsonPrecisionLower95": 0.4385 + }, + "complaint": { + "f1": 0.4176, + "falseNegative": 265, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 95, + "recall": 0.2639, + "total": 2970, + "trueNegative": 2610, + "truePositive": 95, + "wilsonPrecisionLower95": 0.9611 + }, + "confirmationDecision": { + "f1": 0.6556, + "falseNegative": 81, + "falsePositive": 23, + "precision": 0.8115, + "predictedPositive": 122, + "recall": 0.55, + "total": 2970, + "trueNegative": 2767, + "truePositive": 99, + "wilsonPrecisionLower95": 0.733 + }, + "followUpReminder": { + "f1": 0.4211, + "falseNegative": 264, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 96, + "recall": 0.2667, + "total": 2970, + "trueNegative": 2610, + "truePositive": 96, + "wilsonPrecisionLower95": 0.9615 + }, + "invitation": { + "f1": 0.4071, + "falseNegative": 134, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 46, + "recall": 0.2556, + "total": 2970, + "trueNegative": 2790, + "truePositive": 46, + "wilsonPrecisionLower95": 0.9229 + }, + "question": { + "f1": 0.8408, + "falseNegative": 173, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 457, + "recall": 0.7254, + "total": 2970, + "trueNegative": 2340, + "truePositive": 457, + "wilsonPrecisionLower95": 0.9917 + }, + "replyableMessage": { + "f1": 0.931, + "falseNegative": 189, + "falsePositive": 63, + "precision": 0.9643, + "predictedPositive": 1764, + "recall": 0.9, + "total": 2970, + "trueNegative": 1017, + "truePositive": 1701, + "wilsonPrecisionLower95": 0.9546 + }, + "scheduleNegotiation": { + "f1": 0.7671, + "falseNegative": 34, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 56, + "recall": 0.6222, + "total": 2970, + "trueNegative": 2880, + "truePositive": 56, + "wilsonPrecisionLower95": 0.9358 + }, + "task": { + "f1": 0.4969, + "falseNegative": 482, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 238, + "recall": 0.3306, + "total": 2970, + "trueNegative": 2250, + "truePositive": 238, + "wilsonPrecisionLower95": 0.9841 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 180, + "recall": 1.0, + "total": 1485, + "trueNegative": 1305, + "truePositive": 180, + "wilsonPrecisionLower95": 0.9791 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9371, + "falseNegative": 16, + "falsePositive": 6, + "precision": 0.9647, + "predictedPositive": 170, + "recall": 0.9111, + "total": 1485, + "trueNegative": 1299, + "truePositive": 164, + "wilsonPrecisionLower95": 0.9251 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.969, + "falseNegative": 16, + "falsePositive": 6, + "precision": 0.9829, + "predictedPositive": 350, + "recall": 0.9556, + "total": 2970, + "trueNegative": 2604, + "truePositive": 344, + "wilsonPrecisionLower95": 0.9631 + }, + "threshold": 0.5 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9727, + "falseNegative": 1, + "falsePositive": 4, + "precision": 0.957, + "predictedPositive": 93, + "recall": 0.9889, + "total": 1485, + "trueNegative": 1391, + "truePositive": 89, + "wilsonPrecisionLower95": 0.8946 + }, + "threshold": 0.585 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7815, + "falseNegative": 31, + "falsePositive": 2, + "precision": 0.9672, + "predictedPositive": 61, + "recall": 0.6556, + "total": 1485, + "trueNegative": 1393, + "truePositive": 59, + "wilsonPrecisionLower95": 0.8881 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.8701, + "falseNegative": 36, + "falsePositive": 7, + "precision": 0.9536, + "predictedPositive": 151, + "recall": 0.8, + "total": 2970, + "trueNegative": 2783, + "truePositive": 144, + "wilsonPrecisionLower95": 0.9074 + }, + "threshold": 0.555 + }, + "followUpReminder": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8454, + "falseNegative": 46, + "falsePositive": 3, + "precision": 0.9781, + "predictedPositive": 137, + "recall": 0.7444, + "total": 1485, + "trueNegative": 1302, + "truePositive": 134, + "wilsonPrecisionLower95": 0.9376 + }, + "threshold": 0.97 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7525, + "falseNegative": 69, + "falsePositive": 4, + "precision": 0.9652, + "predictedPositive": 115, + "recall": 0.6167, + "total": 1485, + "trueNegative": 1301, + "truePositive": 111, + "wilsonPrecisionLower95": 0.914 + }, + "threshold": 0.785 + } + }, + "metrics": { + "f1": 0.703, + "falseNegative": 160, + "falsePositive": 9, + "precision": 0.9569, + "predictedPositive": 209, + "recall": 0.5556, + "total": 2970, + "trueNegative": 2601, + "truePositive": 200, + "wilsonPrecisionLower95": 0.9202 + }, + "threshold": 0.955 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7974, + "falseNegative": 29, + "falsePositive": 2, + "precision": 0.9683, + "predictedPositive": 63, + "recall": 0.6778, + "total": 1485, + "trueNegative": 1393, + "truePositive": 61, + "wilsonPrecisionLower95": 0.8914 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8259, + "falseNegative": 85, + "falsePositive": 12, + "precision": 0.9504, + "predictedPositive": 242, + "recall": 0.7302, + "total": 1485, + "trueNegative": 1158, + "truePositive": 230, + "wilsonPrecisionLower95": 0.9153 + }, + "threshold": 0.865 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8194, + "falseNegative": 95, + "falsePositive": 2, + "precision": 0.991, + "predictedPositive": 222, + "recall": 0.6984, + "total": 1485, + "trueNegative": 1168, + "truePositive": 220, + "wilsonPrecisionLower95": 0.9678 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.7959, + "falseNegative": 199, + "falsePositive": 22, + "precision": 0.9514, + "predictedPositive": 453, + "recall": 0.6841, + "total": 2970, + "trueNegative": 2318, + "truePositive": 431, + "wilsonPrecisionLower95": 0.9276 + }, + "threshold": 0.74 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9496, + "falseNegative": 50, + "falsePositive": 45, + "precision": 0.9521, + "predictedPositive": 940, + "recall": 0.9471, + "total": 1485, + "trueNegative": 495, + "truePositive": 895, + "wilsonPrecisionLower95": 0.9365 + }, + "threshold": 0.705 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9209, + "falseNegative": 101, + "falsePositive": 44, + "precision": 0.9505, + "predictedPositive": 888, + "recall": 0.8931, + "total": 1485, + "trueNegative": 496, + "truePositive": 844, + "wilsonPrecisionLower95": 0.9341 + }, + "threshold": 0.78 + } + }, + "metrics": { + "f1": 0.9318, + "falseNegative": 163, + "falsePositive": 90, + "precision": 0.9505, + "predictedPositive": 1817, + "recall": 0.9138, + "total": 2970, + "trueNegative": 990, + "truePositive": 1727, + "wilsonPrecisionLower95": 0.9395 + }, + "threshold": 0.77 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 45, + "recall": 1.0, + "total": 1485, + "trueNegative": 1440, + "truePositive": 45, + "wilsonPrecisionLower95": 0.9213 + }, + "threshold": 0.745 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9783, + "falseNegative": 0, + "falsePositive": 2, + "precision": 0.9574, + "predictedPositive": 47, + "recall": 1.0, + "total": 1485, + "trueNegative": 1438, + "truePositive": 45, + "wilsonPrecisionLower95": 0.8575 + }, + "threshold": 0.93 + } + }, + "metrics": { + "f1": 0.9778, + "falseNegative": 2, + "falsePositive": 2, + "precision": 0.9778, + "predictedPositive": 90, + "recall": 0.9778, + "total": 2970, + "trueNegative": 2878, + "truePositive": 88, + "wilsonPrecisionLower95": 0.9226 + }, + "threshold": 0.93 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8488, + "falseNegative": 85, + "falsePositive": 13, + "precision": 0.9549, + "predictedPositive": 288, + "recall": 0.7639, + "total": 1485, + "trueNegative": 1112, + "truePositive": 275, + "wilsonPrecisionLower95": 0.9243 + }, + "threshold": 0.865 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8006, + "falseNegative": 111, + "falsePositive": 13, + "precision": 0.9504, + "predictedPositive": 262, + "recall": 0.6917, + "total": 1485, + "trueNegative": 1112, + "truePositive": 249, + "wilsonPrecisionLower95": 0.917 + }, + "threshold": 0.8 + } + }, + "metrics": { + "f1": 0.8301, + "falseNegative": 190, + "falsePositive": 27, + "precision": 0.9515, + "predictedPositive": 557, + "recall": 0.7361, + "total": 2970, + "trueNegative": 2223, + "truePositive": 530, + "wilsonPrecisionLower95": 0.9304 + }, + "threshold": 0.82 + } + }, + "trainingRecords": 15345, + "validation": { + "intentsMeeting95Precision": 6, + "macroF1": 0.5094, + "macroPrecision": 0.9758, + "macroRecall": 0.3953, + "minimumPrecision": 0.9412, + "perIntent": { + "blessing": { + "f1": 0.1443, + "falseNegative": 83, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 7, + "recall": 0.0778, + "total": 2970, + "trueNegative": 2880, + "truePositive": 7, + "wilsonPrecisionLower95": 0.6457 + }, + "complaint": { + "f1": 0.5455, + "falseNegative": 225, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 135, + "recall": 0.375, + "total": 2970, + "trueNegative": 2610, + "truePositive": 135, + "wilsonPrecisionLower95": 0.9723 + }, + "confirmationDecision": { + "f1": 0.6617, + "falseNegative": 91, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 89, + "recall": 0.4944, + "total": 2970, + "trueNegative": 2790, + "truePositive": 89, + "wilsonPrecisionLower95": 0.9586 + }, + "followUpReminder": { + "f1": 0.2464, + "falseNegative": 309, + "falsePositive": 3, + "precision": 0.9444, + "predictedPositive": 54, + "recall": 0.1417, + "total": 2970, + "trueNegative": 2607, + "truePositive": 51, + "wilsonPrecisionLower95": 0.8489 + }, + "invitation": { + "f1": 0.1347, + "falseNegative": 167, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 13, + "recall": 0.0722, + "total": 2970, + "trueNegative": 2790, + "truePositive": 13, + "wilsonPrecisionLower95": 0.7719 + }, + "question": { + "f1": 0.8158, + "falseNegative": 187, + "falsePositive": 13, + "precision": 0.9715, + "predictedPositive": 456, + "recall": 0.7032, + "total": 2970, + "trueNegative": 2327, + "truePositive": 443, + "wilsonPrecisionLower95": 0.9518 + }, + "replyableMessage": { + "f1": 0.9354, + "falseNegative": 151, + "falsePositive": 89, + "precision": 0.9513, + "predictedPositive": 1828, + "recall": 0.9201, + "total": 2970, + "trueNegative": 991, + "truePositive": 1739, + "wilsonPrecisionLower95": 0.9405 + }, + "scheduleNegotiation": { + "f1": 0.5161, + "falseNegative": 58, + "falsePositive": 2, + "precision": 0.9412, + "predictedPositive": 34, + "recall": 0.3556, + "total": 2970, + "trueNegative": 2878, + "truePositive": 32, + "wilsonPrecisionLower95": 0.8091 + }, + "task": { + "f1": 0.585, + "falseNegative": 419, + "falsePositive": 8, + "precision": 0.9741, + "predictedPositive": 309, + "recall": 0.4181, + "total": 2970, + "trueNegative": 2242, + "truePositive": 301, + "wilsonPrecisionLower95": 0.9498 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 8e-06, + "augmentation": "none", + "char_max": 6, + "char_min": 3, + "external_weight": 0.63, + "hard_example_weight": 1.2, + "l1_ratio": 0.05, + "max_char_features": 60000, + "max_word_features": 30000, + "min_df": 2, + "pseudo_weight": 0.21, + "round": 4, + "word_max": 3 + }, + "featureCount": 79458, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 0 + }, + "complaint": { + "negative": 97, + "positive": 0 + }, + "confirmationDecision": { + "negative": 51, + "positive": 2 + }, + "followUpReminder": { + "negative": 250, + "positive": 0 + }, + "invitation": { + "negative": 250, + "positive": 0 + }, + "question": { + "negative": 36, + "positive": 1 + }, + "replyableMessage": { + "negative": 3, + "positive": 130 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 0 + }, + "task": { + "negative": 85, + "positive": 0 + } + }, + "round": 4, + "selectionObjective": 0.6534, + "syntheticRegression": { + "intentsMeeting95Precision": 7, + "macroF1": 0.5001, + "macroPrecision": 0.9696, + "macroRecall": 0.3844, + "minimumPrecision": 0.7742, + "perIntent": { + "blessing": { + "f1": 0.0645, + "falseNegative": 87, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 3, + "recall": 0.0333, + "total": 2970, + "trueNegative": 2880, + "truePositive": 3, + "wilsonPrecisionLower95": 0.4385 + }, + "complaint": { + "f1": 0.4211, + "falseNegative": 264, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 96, + "recall": 0.2667, + "total": 2970, + "trueNegative": 2610, + "truePositive": 96, + "wilsonPrecisionLower95": 0.9615 + }, + "confirmationDecision": { + "f1": 0.5275, + "falseNegative": 108, + "falsePositive": 21, + "precision": 0.7742, + "predictedPositive": 93, + "recall": 0.4, + "total": 2970, + "trueNegative": 2769, + "truePositive": 72, + "wilsonPrecisionLower95": 0.6793 + }, + "followUpReminder": { + "f1": 0.4071, + "falseNegative": 268, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 92, + "recall": 0.2556, + "total": 2970, + "trueNegative": 2610, + "truePositive": 92, + "wilsonPrecisionLower95": 0.9599 + }, + "invitation": { + "f1": 0.2, + "falseNegative": 160, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 20, + "recall": 0.1111, + "total": 2970, + "trueNegative": 2790, + "truePositive": 20, + "wilsonPrecisionLower95": 0.8389 + }, + "question": { + "f1": 0.7349, + "falseNegative": 264, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 366, + "recall": 0.581, + "total": 2970, + "trueNegative": 2340, + "truePositive": 366, + "wilsonPrecisionLower95": 0.9896 + }, + "replyableMessage": { + "f1": 0.8816, + "falseNegative": 353, + "falsePositive": 60, + "precision": 0.9624, + "predictedPositive": 1597, + "recall": 0.8132, + "total": 2970, + "trueNegative": 1020, + "truePositive": 1537, + "wilsonPrecisionLower95": 0.9519 + }, + "scheduleNegotiation": { + "f1": 0.8462, + "falseNegative": 24, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 66, + "recall": 0.7333, + "total": 2970, + "trueNegative": 2880, + "truePositive": 66, + "wilsonPrecisionLower95": 0.945 + }, + "task": { + "f1": 0.4184, + "falseNegative": 529, + "falsePositive": 2, + "precision": 0.9896, + "predictedPositive": 193, + "recall": 0.2653, + "total": 2970, + "trueNegative": 2248, + "truePositive": 191, + "wilsonPrecisionLower95": 0.963 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9972, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 179, + "recall": 0.9944, + "total": 1485, + "trueNegative": 1305, + "truePositive": 179, + "wilsonPrecisionLower95": 0.979 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9253, + "falseNegative": 19, + "falsePositive": 7, + "precision": 0.9583, + "predictedPositive": 168, + "recall": 0.8944, + "total": 1485, + "trueNegative": 1298, + "truePositive": 161, + "wilsonPrecisionLower95": 0.9165 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.9618, + "falseNegative": 20, + "falsePositive": 7, + "precision": 0.9798, + "predictedPositive": 347, + "recall": 0.9444, + "total": 2970, + "trueNegative": 2603, + "truePositive": 340, + "wilsonPrecisionLower95": 0.959 + }, + "threshold": 0.5 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8765, + "falseNegative": 19, + "falsePositive": 1, + "precision": 0.9861, + "predictedPositive": 72, + "recall": 0.7889, + "total": 1485, + "trueNegative": 1394, + "truePositive": 71, + "wilsonPrecisionLower95": 0.9254 + }, + "threshold": 0.945 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6569, + "falseNegative": 45, + "falsePositive": 2, + "precision": 0.9574, + "predictedPositive": 47, + "recall": 0.5, + "total": 1485, + "trueNegative": 1393, + "truePositive": 45, + "wilsonPrecisionLower95": 0.8575 + }, + "threshold": 0.64 + } + }, + "metrics": { + "f1": 0.7108, + "falseNegative": 78, + "falsePositive": 5, + "precision": 0.9533, + "predictedPositive": 107, + "recall": 0.5667, + "total": 2970, + "trueNegative": 2785, + "truePositive": 102, + "wilsonPrecisionLower95": 0.8952 + }, + "threshold": 0.91 + }, + "followUpReminder": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8, + "falseNegative": 56, + "falsePositive": 6, + "precision": 0.9538, + "predictedPositive": 130, + "recall": 0.6889, + "total": 1485, + "trueNegative": 1299, + "truePositive": 124, + "wilsonPrecisionLower95": 0.903 + }, + "threshold": 0.98 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6165, + "falseNegative": 98, + "falsePositive": 4, + "precision": 0.9535, + "predictedPositive": 86, + "recall": 0.4556, + "total": 1485, + "trueNegative": 1301, + "truePositive": 82, + "wilsonPrecisionLower95": 0.8864 + }, + "threshold": 0.825 + } + }, + "metrics": { + "f1": 0.5912, + "falseNegative": 206, + "falsePositive": 7, + "precision": 0.9565, + "predictedPositive": 161, + "recall": 0.4278, + "total": 2970, + "trueNegative": 2603, + "truePositive": 154, + "wilsonPrecisionLower95": 0.913 + }, + "threshold": 0.98 + }, + "invitation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8302, + "falseNegative": 24, + "falsePositive": 3, + "precision": 0.9565, + "predictedPositive": 69, + "recall": 0.7333, + "total": 1485, + "trueNegative": 1392, + "truePositive": 66, + "wilsonPrecisionLower95": 0.8798 + }, + "threshold": 0.98 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.536, + "falseNegative": 113, + "falsePositive": 3, + "precision": 0.9571, + "predictedPositive": 70, + "recall": 0.3722, + "total": 2970, + "trueNegative": 2787, + "truePositive": 67, + "wilsonPrecisionLower95": 0.8814 + }, + "threshold": 0.995 + }, + "question": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.7813, + "falseNegative": 106, + "falsePositive": 11, + "precision": 0.95, + "predictedPositive": 220, + "recall": 0.6635, + "total": 1485, + "trueNegative": 1159, + "truePositive": 209, + "wilsonPrecisionLower95": 0.9127 + }, + "threshold": 0.865 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6844, + "falseNegative": 148, + "falsePositive": 6, + "precision": 0.9653, + "predictedPositive": 173, + "recall": 0.5302, + "total": 1485, + "trueNegative": 1164, + "truePositive": 167, + "wilsonPrecisionLower95": 0.9264 + }, + "threshold": 0.63 + } + }, + "metrics": { + "f1": 0.75, + "falseNegative": 240, + "falsePositive": 20, + "precision": 0.9512, + "predictedPositive": 410, + "recall": 0.619, + "total": 2970, + "trueNegative": 2320, + "truePositive": 390, + "wilsonPrecisionLower95": 0.9259 + }, + "threshold": 0.74 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9441, + "falseNegative": 59, + "falsePositive": 46, + "precision": 0.9506, + "predictedPositive": 932, + "recall": 0.9376, + "total": 1485, + "trueNegative": 494, + "truePositive": 886, + "wilsonPrecisionLower95": 0.9348 + }, + "threshold": 0.76 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8342, + "falseNegative": 243, + "falsePositive": 36, + "precision": 0.9512, + "predictedPositive": 738, + "recall": 0.7429, + "total": 1485, + "trueNegative": 504, + "truePositive": 702, + "wilsonPrecisionLower95": 0.9332 + }, + "threshold": 0.885 + } + }, + "metrics": { + "f1": 0.9056, + "falseNegative": 255, + "falsePositive": 86, + "precision": 0.95, + "predictedPositive": 1721, + "recall": 0.8651, + "total": 2970, + "trueNegative": 994, + "truePositive": 1635, + "wilsonPrecisionLower95": 0.9387 + }, + "threshold": 0.855 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 45, + "recall": 1.0, + "total": 1485, + "trueNegative": 1440, + "truePositive": 45, + "wilsonPrecisionLower95": 0.9213 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9318, + "falseNegative": 4, + "falsePositive": 2, + "precision": 0.9535, + "predictedPositive": 43, + "recall": 0.9111, + "total": 1485, + "trueNegative": 1438, + "truePositive": 41, + "wilsonPrecisionLower95": 0.8454 + }, + "threshold": 0.945 + } + }, + "metrics": { + "f1": 0.9727, + "falseNegative": 1, + "falsePositive": 4, + "precision": 0.957, + "predictedPositive": 93, + "recall": 0.9889, + "total": 2970, + "trueNegative": 2876, + "truePositive": 89, + "wilsonPrecisionLower95": 0.8946 + }, + "threshold": 0.62 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.851, + "falseNegative": 83, + "falsePositive": 14, + "precision": 0.9519, + "predictedPositive": 291, + "recall": 0.7694, + "total": 1485, + "trueNegative": 1111, + "truePositive": 277, + "wilsonPrecisionLower95": 0.9209 + }, + "threshold": 0.87 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.4886, + "falseNegative": 242, + "falsePositive": 5, + "precision": 0.9593, + "predictedPositive": 123, + "recall": 0.3278, + "total": 1485, + "trueNegative": 1120, + "truePositive": 118, + "wilsonPrecisionLower95": 0.9084 + }, + "threshold": 0.935 + } + }, + "metrics": { + "f1": 0.6978, + "falseNegative": 324, + "falsePositive": 19, + "precision": 0.9542, + "predictedPositive": 415, + "recall": 0.55, + "total": 2970, + "trueNegative": 2231, + "truePositive": 396, + "wilsonPrecisionLower95": 0.9296 + }, + "threshold": 0.915 + } + }, + "trainingRecords": 15345, + "validation": { + "intentsMeeting95Precision": 8, + "macroF1": 0.4964, + "macroPrecision": 0.9825, + "macroRecall": 0.3647, + "minimumPrecision": 0.9412, + "perIntent": { + "blessing": { + "f1": 0.2, + "falseNegative": 80, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 10, + "recall": 0.1111, + "total": 2970, + "trueNegative": 2880, + "truePositive": 10, + "wilsonPrecisionLower95": 0.7225 + }, + "complaint": { + "f1": 0.5425, + "falseNegative": 226, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 134, + "recall": 0.3722, + "total": 2970, + "trueNegative": 2610, + "truePositive": 134, + "wilsonPrecisionLower95": 0.9721 + }, + "confirmationDecision": { + "f1": 0.5542, + "falseNegative": 111, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 69, + "recall": 0.3833, + "total": 2970, + "trueNegative": 2790, + "truePositive": 69, + "wilsonPrecisionLower95": 0.9473 + }, + "followUpReminder": { + "f1": 0.2602, + "falseNegative": 306, + "falsePositive": 1, + "precision": 0.9818, + "predictedPositive": 55, + "recall": 0.15, + "total": 2970, + "trueNegative": 2609, + "truePositive": 54, + "wilsonPrecisionLower95": 0.9039 + }, + "invitation": { + "f1": 0.3019, + "falseNegative": 148, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 32, + "recall": 0.1778, + "total": 2970, + "trueNegative": 2790, + "truePositive": 32, + "wilsonPrecisionLower95": 0.8928 + }, + "question": { + "f1": 0.7302, + "falseNegative": 262, + "falsePositive": 10, + "precision": 0.9735, + "predictedPositive": 378, + "recall": 0.5841, + "total": 2970, + "trueNegative": 2330, + "truePositive": 368, + "wilsonPrecisionLower95": 0.952 + }, + "replyableMessage": { + "f1": 0.8921, + "falseNegative": 302, + "falsePositive": 82, + "precision": 0.9509, + "predictedPositive": 1670, + "recall": 0.8402, + "total": 2970, + "trueNegative": 998, + "truePositive": 1588, + "wilsonPrecisionLower95": 0.9395 + }, + "scheduleNegotiation": { + "f1": 0.5161, + "falseNegative": 58, + "falsePositive": 2, + "precision": 0.9412, + "predictedPositive": 34, + "recall": 0.3556, + "total": 2970, + "trueNegative": 2878, + "truePositive": 32, + "wilsonPrecisionLower95": 0.8091 + }, + "task": { + "f1": 0.4708, + "falseNegative": 498, + "falsePositive": 1, + "precision": 0.9955, + "predictedPositive": 223, + "recall": 0.3083, + "total": 2970, + "trueNegative": 2249, + "truePositive": 222, + "wilsonPrecisionLower95": 0.975 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 1.2e-05, + "augmentation": "none", + "char_max": 5, + "char_min": 2, + "external_weight": 0.65, + "hard_example_weight": 1.4, + "l1_ratio": 0.1, + "max_char_features": 60000, + "max_word_features": 32000, + "min_df": 1, + "pseudo_weight": 0.22499999999999998, + "round": 5, + "word_max": 3 + }, + "featureCount": 92000, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 0 + }, + "complaint": { + "negative": 88, + "positive": 0 + }, + "confirmationDecision": { + "negative": 96, + "positive": 3 + }, + "followUpReminder": { + "negative": 250, + "positive": 0 + }, + "invitation": { + "negative": 250, + "positive": 0 + }, + "question": { + "negative": 40, + "positive": 1 + }, + "replyableMessage": { + "negative": 3, + "positive": 150 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 0 + }, + "task": { + "negative": 97, + "positive": 0 + } + }, + "round": 5, + "selectionObjective": 0.6361, + "syntheticRegression": { + "intentsMeeting95Precision": 7, + "macroF1": 0.5097, + "macroPrecision": 0.9762, + "macroRecall": 0.3951, + "minimumPrecision": 0.8197, + "perIntent": { + "blessing": { + "f1": 0.022, + "falseNegative": 89, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 1, + "recall": 0.0111, + "total": 2970, + "trueNegative": 2880, + "truePositive": 1, + "wilsonPrecisionLower95": 0.2065 + }, + "complaint": { + "f1": 0.4176, + "falseNegative": 265, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 95, + "recall": 0.2639, + "total": 2970, + "trueNegative": 2610, + "truePositive": 95, + "wilsonPrecisionLower95": 0.9611 + }, + "confirmationDecision": { + "f1": 0.6623, + "falseNegative": 80, + "falsePositive": 22, + "precision": 0.8197, + "predictedPositive": 122, + "recall": 0.5556, + "total": 2970, + "trueNegative": 2768, + "truePositive": 100, + "wilsonPrecisionLower95": 0.742 + }, + "followUpReminder": { + "f1": 0.341, + "falseNegative": 286, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 74, + "recall": 0.2056, + "total": 2970, + "trueNegative": 2610, + "truePositive": 74, + "wilsonPrecisionLower95": 0.9507 + }, + "invitation": { + "f1": 0.2609, + "falseNegative": 153, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 27, + "recall": 0.15, + "total": 2970, + "trueNegative": 2790, + "truePositive": 27, + "wilsonPrecisionLower95": 0.8754 + }, + "question": { + "f1": 0.8147, + "falseNegative": 197, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 433, + "recall": 0.6873, + "total": 2970, + "trueNegative": 2340, + "truePositive": 433, + "wilsonPrecisionLower95": 0.9912 + }, + "replyableMessage": { + "f1": 0.912, + "falseNegative": 257, + "falsePositive": 58, + "precision": 0.9657, + "predictedPositive": 1691, + "recall": 0.864, + "total": 2970, + "trueNegative": 1022, + "truePositive": 1633, + "wilsonPrecisionLower95": 0.9559 + }, + "scheduleNegotiation": { + "f1": 0.6364, + "falseNegative": 48, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 42, + "recall": 0.4667, + "total": 2970, + "trueNegative": 2880, + "truePositive": 42, + "wilsonPrecisionLower95": 0.9162 + }, + "task": { + "f1": 0.52, + "falseNegative": 467, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 253, + "recall": 0.3514, + "total": 2970, + "trueNegative": 2250, + "truePositive": 253, + "wilsonPrecisionLower95": 0.985 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 180, + "recall": 1.0, + "total": 1485, + "trueNegative": 1305, + "truePositive": 180, + "wilsonPrecisionLower95": 0.9791 + }, + "threshold": 0.69 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9288, + "falseNegative": 17, + "falsePositive": 8, + "precision": 0.9532, + "predictedPositive": 171, + "recall": 0.9056, + "total": 1485, + "trueNegative": 1297, + "truePositive": 163, + "wilsonPrecisionLower95": 0.9104 + }, + "threshold": 0.51 + } + }, + "metrics": { + "f1": 0.9569, + "falseNegative": 16, + "falsePositive": 15, + "precision": 0.9582, + "predictedPositive": 359, + "recall": 0.9556, + "total": 2970, + "trueNegative": 2595, + "truePositive": 344, + "wilsonPrecisionLower95": 0.9322 + }, + "threshold": 0.5 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8589, + "falseNegative": 20, + "falsePositive": 3, + "precision": 0.9589, + "predictedPositive": 73, + "recall": 0.7778, + "total": 1485, + "trueNegative": 1392, + "truePositive": 70, + "wilsonPrecisionLower95": 0.886 + }, + "threshold": 0.57 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6763, + "falseNegative": 43, + "falsePositive": 2, + "precision": 0.9592, + "predictedPositive": 49, + "recall": 0.5222, + "total": 1485, + "trueNegative": 1393, + "truePositive": 47, + "wilsonPrecisionLower95": 0.8629 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.76, + "falseNegative": 66, + "falsePositive": 6, + "precision": 0.95, + "predictedPositive": 120, + "recall": 0.6333, + "total": 2970, + "trueNegative": 2784, + "truePositive": 114, + "wilsonPrecisionLower95": 0.8952 + }, + "threshold": 0.56 + }, + "followUpReminder": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8, + "falseNegative": 56, + "falsePositive": 6, + "precision": 0.9538, + "predictedPositive": 130, + "recall": 0.6889, + "total": 1485, + "trueNegative": 1299, + "truePositive": 124, + "wilsonPrecisionLower95": 0.903 + }, + "threshold": 0.96 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7708, + "falseNegative": 64, + "falsePositive": 5, + "precision": 0.9587, + "predictedPositive": 121, + "recall": 0.6444, + "total": 1485, + "trueNegative": 1300, + "truePositive": 116, + "wilsonPrecisionLower95": 0.9069 + }, + "threshold": 0.58 + } + }, + "metrics": { + "f1": 0.6892, + "falseNegative": 166, + "falsePositive": 9, + "precision": 0.9557, + "predictedPositive": 203, + "recall": 0.5389, + "total": 2970, + "trueNegative": 2601, + "truePositive": 194, + "wilsonPrecisionLower95": 0.9179 + }, + "threshold": 0.915 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.5203, + "falseNegative": 58, + "falsePositive": 1, + "precision": 0.9697, + "predictedPositive": 33, + "recall": 0.3556, + "total": 1485, + "trueNegative": 1394, + "truePositive": 32, + "wilsonPrecisionLower95": 0.8468 + }, + "threshold": 0.88 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8225, + "falseNegative": 88, + "falsePositive": 10, + "precision": 0.9578, + "predictedPositive": 237, + "recall": 0.7206, + "total": 1485, + "trueNegative": 1160, + "truePositive": 227, + "wilsonPrecisionLower95": 0.9241 + }, + "threshold": 0.84 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8388, + "falseNegative": 86, + "falsePositive": 2, + "precision": 0.9913, + "predictedPositive": 231, + "recall": 0.727, + "total": 1485, + "trueNegative": 1168, + "truePositive": 229, + "wilsonPrecisionLower95": 0.969 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.8591, + "falseNegative": 136, + "falsePositive": 26, + "precision": 0.95, + "predictedPositive": 520, + "recall": 0.7841, + "total": 2970, + "trueNegative": 2314, + "truePositive": 494, + "wilsonPrecisionLower95": 0.9277 + }, + "threshold": 0.505 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9541, + "falseNegative": 40, + "falsePositive": 47, + "precision": 0.9506, + "predictedPositive": 952, + "recall": 0.9577, + "total": 1485, + "trueNegative": 493, + "truePositive": 905, + "wilsonPrecisionLower95": 0.935 + }, + "threshold": 0.69 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8807, + "falseNegative": 170, + "falsePositive": 40, + "precision": 0.9509, + "predictedPositive": 815, + "recall": 0.8201, + "total": 1485, + "trueNegative": 500, + "truePositive": 775, + "wilsonPrecisionLower95": 0.9339 + }, + "threshold": 0.8 + } + }, + "metrics": { + "f1": 0.914, + "falseNegative": 227, + "falsePositive": 86, + "precision": 0.9508, + "predictedPositive": 1749, + "recall": 0.8799, + "total": 2970, + "trueNegative": 994, + "truePositive": 1663, + "wilsonPrecisionLower95": 0.9397 + }, + "threshold": 0.78 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9888, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 44, + "recall": 0.9778, + "total": 1485, + "trueNegative": 1440, + "truePositive": 44, + "wilsonPrecisionLower95": 0.9197 + }, + "threshold": 0.635 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.8129, + "falseNegative": 27, + "falsePositive": 2, + "precision": 0.9692, + "predictedPositive": 65, + "recall": 0.7, + "total": 2970, + "trueNegative": 2878, + "truePositive": 63, + "wilsonPrecisionLower95": 0.8946 + }, + "threshold": 0.995 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8185, + "falseNegative": 103, + "falsePositive": 11, + "precision": 0.959, + "predictedPositive": 268, + "recall": 0.7139, + "total": 1485, + "trueNegative": 1114, + "truePositive": 257, + "wilsonPrecisionLower95": 0.928 + }, + "threshold": 0.83 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7702, + "falseNegative": 127, + "falsePositive": 12, + "precision": 0.951, + "predictedPositive": 245, + "recall": 0.6472, + "total": 1485, + "trueNegative": 1113, + "truePositive": 233, + "wilsonPrecisionLower95": 0.9164 + }, + "threshold": 0.765 + } + }, + "metrics": { + "f1": 0.7816, + "falseNegative": 244, + "falsePositive": 22, + "precision": 0.9558, + "predictedPositive": 498, + "recall": 0.6611, + "total": 2970, + "trueNegative": 2228, + "truePositive": 476, + "wilsonPrecisionLower95": 0.934 + }, + "threshold": 0.805 + } + }, + "trainingRecords": 15345, + "validation": { + "intentsMeeting95Precision": 7, + "macroF1": 0.4633, + "macroPrecision": 0.987, + "macroRecall": 0.3584, + "minimumPrecision": 0.9508, + "perIntent": { + "blessing": { + "f1": 0.0851, + "falseNegative": 86, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 4, + "recall": 0.0444, + "total": 2970, + "trueNegative": 2880, + "truePositive": 4, + "wilsonPrecisionLower95": 0.5101 + }, + "complaint": { + "f1": 0.5484, + "falseNegative": 224, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 136, + "recall": 0.3778, + "total": 2970, + "trueNegative": 2610, + "truePositive": 136, + "wilsonPrecisionLower95": 0.9725 + }, + "confirmationDecision": { + "f1": 0.616, + "falseNegative": 99, + "falsePositive": 2, + "precision": 0.9759, + "predictedPositive": 83, + "recall": 0.45, + "total": 2970, + "trueNegative": 2788, + "truePositive": 81, + "wilsonPrecisionLower95": 0.9163 + }, + "followUpReminder": { + "f1": 0.2524, + "falseNegative": 308, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 52, + "recall": 0.1444, + "total": 2970, + "trueNegative": 2610, + "truePositive": 52, + "wilsonPrecisionLower95": 0.9312 + }, + "invitation": { + "f1": 0.0645, + "falseNegative": 174, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 6, + "recall": 0.0333, + "total": 2970, + "trueNegative": 2790, + "truePositive": 6, + "wilsonPrecisionLower95": 0.6097 + }, + "question": { + "f1": 0.8249, + "falseNegative": 180, + "falsePositive": 11, + "precision": 0.9761, + "predictedPositive": 461, + "recall": 0.7143, + "total": 2970, + "trueNegative": 2329, + "truePositive": 450, + "wilsonPrecisionLower95": 0.9578 + }, + "replyableMessage": { + "f1": 0.9188, + "falseNegative": 210, + "falsePositive": 87, + "precision": 0.9508, + "predictedPositive": 1767, + "recall": 0.8889, + "total": 2970, + "trueNegative": 993, + "truePositive": 1680, + "wilsonPrecisionLower95": 0.9397 + }, + "scheduleNegotiation": { + "f1": 0.2857, + "falseNegative": 75, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 15, + "recall": 0.1667, + "total": 2970, + "trueNegative": 2880, + "truePositive": 15, + "wilsonPrecisionLower95": 0.7961 + }, + "task": { + "f1": 0.5737, + "falseNegative": 428, + "falsePositive": 6, + "precision": 0.9799, + "predictedPositive": 298, + "recall": 0.4056, + "total": 2970, + "trueNegative": 2244, + "truePositive": 292, + "wilsonPrecisionLower95": 0.9568 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 8e-06, + "augmentation": "punctuation", + "char_max": 6, + "char_min": 2, + "external_weight": 0.67, + "hard_example_weight": 1.5, + "l1_ratio": 0.1, + "max_char_features": 65000, + "max_word_features": 28000, + "min_df": 2, + "pseudo_weight": 0.24, + "round": 6, + "word_max": 2 + }, + "featureCount": 78354, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 0 + }, + "complaint": { + "negative": 146, + "positive": 0 + }, + "confirmationDecision": { + "negative": 88, + "positive": 2 + }, + "followUpReminder": { + "negative": 250, + "positive": 0 + }, + "invitation": { + "negative": 250, + "positive": 0 + }, + "question": { + "negative": 36, + "positive": 5 + }, + "replyableMessage": { + "negative": 2, + "positive": 130 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 0 + }, + "task": { + "negative": 83, + "positive": 0 + } + }, + "round": 6, + "selectionObjective": 0.6598, + "syntheticRegression": { + "intentsMeeting95Precision": 7, + "macroF1": 0.5285, + "macroPrecision": 0.9709, + "macroRecall": 0.4124, + "minimumPrecision": 0.78, + "perIntent": { + "blessing": { + "f1": 0.0435, + "falseNegative": 88, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 2, + "recall": 0.0222, + "total": 2970, + "trueNegative": 2880, + "truePositive": 2, + "wilsonPrecisionLower95": 0.3424 + }, + "complaint": { + "f1": 0.4211, + "falseNegative": 264, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 96, + "recall": 0.2667, + "total": 2970, + "trueNegative": 2610, + "truePositive": 96, + "wilsonPrecisionLower95": 0.9615 + }, + "confirmationDecision": { + "f1": 0.5571, + "falseNegative": 102, + "falsePositive": 22, + "precision": 0.78, + "predictedPositive": 100, + "recall": 0.4333, + "total": 2970, + "trueNegative": 2768, + "truePositive": 78, + "wilsonPrecisionLower95": 0.6893 + }, + "followUpReminder": { + "f1": 0.3636, + "falseNegative": 280, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 80, + "recall": 0.2222, + "total": 2970, + "trueNegative": 2610, + "truePositive": 80, + "wilsonPrecisionLower95": 0.9542 + }, + "invitation": { + "f1": 0.3333, + "falseNegative": 144, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 36, + "recall": 0.2, + "total": 2970, + "trueNegative": 2790, + "truePositive": 36, + "wilsonPrecisionLower95": 0.9036 + }, + "question": { + "f1": 0.8224, + "falseNegative": 190, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 440, + "recall": 0.6984, + "total": 2970, + "trueNegative": 2340, + "truePositive": 440, + "wilsonPrecisionLower95": 0.9913 + }, + "replyableMessage": { + "f1": 0.8888, + "falseNegative": 335, + "falsePositive": 54, + "precision": 0.9664, + "predictedPositive": 1609, + "recall": 0.8228, + "total": 2970, + "trueNegative": 1026, + "truePositive": 1555, + "wilsonPrecisionLower95": 0.9565 + }, + "scheduleNegotiation": { + "f1": 0.8387, + "falseNegative": 25, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 65, + "recall": 0.7222, + "total": 2970, + "trueNegative": 2880, + "truePositive": 65, + "wilsonPrecisionLower95": 0.9442 + }, + "task": { + "f1": 0.488, + "falseNegative": 487, + "falsePositive": 2, + "precision": 0.9915, + "predictedPositive": 235, + "recall": 0.3236, + "total": 2970, + "trueNegative": 2248, + "truePositive": 233, + "wilsonPrecisionLower95": 0.9695 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9834, + "falseNegative": 2, + "falsePositive": 4, + "precision": 0.978, + "predictedPositive": 182, + "recall": 0.9889, + "total": 1485, + "trueNegative": 1301, + "truePositive": 178, + "wilsonPrecisionLower95": 0.9449 + }, + "threshold": 0.555 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9494, + "falseNegative": 11, + "falsePositive": 7, + "precision": 0.9602, + "predictedPositive": 176, + "recall": 0.9389, + "total": 1485, + "trueNegative": 1298, + "truePositive": 169, + "wilsonPrecisionLower95": 0.9202 + }, + "threshold": 0.535 + } + }, + "metrics": { + "f1": 0.9652, + "falseNegative": 13, + "falsePositive": 12, + "precision": 0.9666, + "predictedPositive": 359, + "recall": 0.9639, + "total": 2970, + "trueNegative": 2598, + "truePositive": 347, + "wilsonPrecisionLower95": 0.9425 + }, + "threshold": 0.535 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8, + "falseNegative": 28, + "falsePositive": 3, + "precision": 0.9538, + "predictedPositive": 65, + "recall": 0.6889, + "total": 1485, + "trueNegative": 1392, + "truePositive": 62, + "wilsonPrecisionLower95": 0.8729 + }, + "threshold": 0.695 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.731, + "falseNegative": 37, + "falsePositive": 2, + "precision": 0.9636, + "predictedPositive": 55, + "recall": 0.5889, + "total": 1485, + "trueNegative": 1393, + "truePositive": 53, + "wilsonPrecisionLower95": 0.8768 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.7354, + "falseNegative": 73, + "falsePositive": 4, + "precision": 0.964, + "predictedPositive": 111, + "recall": 0.5944, + "total": 2970, + "trueNegative": 2786, + "truePositive": 107, + "wilsonPrecisionLower95": 0.911 + }, + "threshold": 0.695 + }, + "followUpReminder": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8632, + "falseNegative": 38, + "falsePositive": 7, + "precision": 0.953, + "predictedPositive": 149, + "recall": 0.7889, + "total": 1485, + "trueNegative": 1298, + "truePositive": 142, + "wilsonPrecisionLower95": 0.9062 + }, + "threshold": 0.9 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.4938, + "falseNegative": 120, + "falsePositive": 3, + "precision": 0.9524, + "predictedPositive": 63, + "recall": 0.3333, + "total": 1485, + "trueNegative": 1302, + "truePositive": 60, + "wilsonPrecisionLower95": 0.8691 + }, + "threshold": 0.845 + } + }, + "metrics": { + "f1": 0.7018, + "falseNegative": 160, + "falsePositive": 10, + "precision": 0.9524, + "predictedPositive": 210, + "recall": 0.5556, + "total": 2970, + "trueNegative": 2600, + "truePositive": 200, + "wilsonPrecisionLower95": 0.9146 + }, + "threshold": 0.885 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7222, + "falseNegative": 38, + "falsePositive": 2, + "precision": 0.963, + "predictedPositive": 54, + "recall": 0.5778, + "total": 1485, + "trueNegative": 1393, + "truePositive": 52, + "wilsonPrecisionLower95": 0.8746 + }, + "threshold": 0.665 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8167, + "falseNegative": 90, + "falsePositive": 11, + "precision": 0.9534, + "predictedPositive": 236, + "recall": 0.7143, + "total": 1485, + "trueNegative": 1159, + "truePositive": 225, + "wilsonPrecisionLower95": 0.9185 + }, + "threshold": 0.865 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8205, + "falseNegative": 91, + "falsePositive": 7, + "precision": 0.9697, + "predictedPositive": 231, + "recall": 0.7111, + "total": 1485, + "trueNegative": 1163, + "truePositive": 224, + "wilsonPrecisionLower95": 0.9388 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.804, + "falseNegative": 191, + "falsePositive": 23, + "precision": 0.9502, + "predictedPositive": 462, + "recall": 0.6968, + "total": 2970, + "trueNegative": 2317, + "truePositive": 439, + "wilsonPrecisionLower95": 0.9264 + }, + "threshold": 0.68 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9268, + "falseNegative": 90, + "falsePositive": 45, + "precision": 0.95, + "predictedPositive": 900, + "recall": 0.9048, + "total": 1485, + "trueNegative": 495, + "truePositive": 855, + "wilsonPrecisionLower95": 0.9338 + }, + "threshold": 0.86 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8789, + "falseNegative": 176, + "falsePositive": 36, + "precision": 0.9553, + "predictedPositive": 805, + "recall": 0.8138, + "total": 1485, + "trueNegative": 504, + "truePositive": 769, + "wilsonPrecisionLower95": 0.9387 + }, + "threshold": 0.85 + } + }, + "metrics": { + "f1": 0.9046, + "falseNegative": 259, + "falsePositive": 85, + "precision": 0.9505, + "predictedPositive": 1716, + "recall": 0.863, + "total": 2970, + "trueNegative": 995, + "truePositive": 1631, + "wilsonPrecisionLower95": 0.9392 + }, + "threshold": 0.85 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9773, + "falseNegative": 2, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 43, + "recall": 0.9556, + "total": 1485, + "trueNegative": 1440, + "truePositive": 43, + "wilsonPrecisionLower95": 0.918 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9888, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 44, + "recall": 0.9778, + "total": 1485, + "trueNegative": 1440, + "truePositive": 44, + "wilsonPrecisionLower95": 0.9197 + }, + "threshold": 0.97 + } + }, + "metrics": { + "f1": 0.9497, + "falseNegative": 5, + "falsePositive": 4, + "precision": 0.9551, + "predictedPositive": 89, + "recall": 0.9444, + "total": 2970, + "trueNegative": 2876, + "truePositive": 85, + "wilsonPrecisionLower95": 0.8901 + }, + "threshold": 0.91 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.827, + "falseNegative": 97, + "falsePositive": 13, + "precision": 0.9529, + "predictedPositive": 276, + "recall": 0.7306, + "total": 1485, + "trueNegative": 1112, + "truePositive": 263, + "wilsonPrecisionLower95": 0.9211 + }, + "threshold": 0.89 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7902, + "falseNegative": 117, + "falsePositive": 12, + "precision": 0.9529, + "predictedPositive": 255, + "recall": 0.675, + "total": 1485, + "trueNegative": 1113, + "truePositive": 243, + "wilsonPrecisionLower95": 0.9196 + }, + "threshold": 0.785 + } + }, + "metrics": { + "f1": 0.7783, + "falseNegative": 246, + "falsePositive": 24, + "precision": 0.9518, + "predictedPositive": 498, + "recall": 0.6583, + "total": 2970, + "trueNegative": 2226, + "truePositive": 474, + "wilsonPrecisionLower95": 0.9293 + }, + "threshold": 0.85 + } + }, + "trainingRecords": 17767, + "validation": { + "intentsMeeting95Precision": 8, + "macroF1": 0.5015, + "macroPrecision": 0.9886, + "macroRecall": 0.3796, + "minimumPrecision": 0.9525, + "perIntent": { + "blessing": { + "f1": 0.1633, + "falseNegative": 82, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 8, + "recall": 0.0889, + "total": 2970, + "trueNegative": 2880, + "truePositive": 8, + "wilsonPrecisionLower95": 0.6756 + }, + "complaint": { + "f1": 0.5484, + "falseNegative": 224, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 136, + "recall": 0.3778, + "total": 2970, + "trueNegative": 2610, + "truePositive": 136, + "wilsonPrecisionLower95": 0.9725 + }, + "confirmationDecision": { + "f1": 0.5714, + "falseNegative": 108, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 72, + "recall": 0.4, + "total": 2970, + "trueNegative": 2790, + "truePositive": 72, + "wilsonPrecisionLower95": 0.9493 + }, + "followUpReminder": { + "f1": 0.2396, + "falseNegative": 311, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 49, + "recall": 0.1361, + "total": 2970, + "trueNegative": 2610, + "truePositive": 49, + "wilsonPrecisionLower95": 0.9273 + }, + "invitation": { + "f1": 0.191, + "falseNegative": 161, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 19, + "recall": 0.1056, + "total": 2970, + "trueNegative": 2790, + "truePositive": 19, + "wilsonPrecisionLower95": 0.8318 + }, + "question": { + "f1": 0.8126, + "falseNegative": 190, + "falsePositive": 13, + "precision": 0.9713, + "predictedPositive": 453, + "recall": 0.6984, + "total": 2970, + "trueNegative": 2327, + "truePositive": 440, + "wilsonPrecisionLower95": 0.9515 + }, + "replyableMessage": { + "f1": 0.9035, + "falseNegative": 266, + "falsePositive": 81, + "precision": 0.9525, + "predictedPositive": 1705, + "recall": 0.8593, + "total": 2970, + "trueNegative": 999, + "truePositive": 1624, + "wilsonPrecisionLower95": 0.9413 + }, + "scheduleNegotiation": { + "f1": 0.5, + "falseNegative": 60, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 30, + "recall": 0.3333, + "total": 2970, + "trueNegative": 2880, + "truePositive": 30, + "wilsonPrecisionLower95": 0.8865 + }, + "task": { + "f1": 0.5837, + "falseNegative": 420, + "falsePositive": 8, + "precision": 0.974, + "predictedPositive": 308, + "recall": 0.4167, + "total": 2970, + "trueNegative": 2242, + "truePositive": 300, + "wilsonPrecisionLower95": 0.9496 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 6e-06, + "augmentation": "punctuation", + "char_max": 6, + "char_min": 3, + "external_weight": 0.6900000000000001, + "hard_example_weight": 1.5, + "l1_ratio": 0.1, + "max_char_features": 65000, + "max_word_features": 32000, + "min_df": 2, + "pseudo_weight": 0.255, + "round": 7, + "word_max": 3 + }, + "featureCount": 87507, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 0 + }, + "complaint": { + "negative": 141, + "positive": 0 + }, + "confirmationDecision": { + "negative": 95, + "positive": 2 + }, + "followUpReminder": { + "negative": 250, + "positive": 0 + }, + "invitation": { + "negative": 250, + "positive": 0 + }, + "question": { + "negative": 36, + "positive": 5 + }, + "replyableMessage": { + "negative": 2, + "positive": 129 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 0 + }, + "task": { + "negative": 87, + "positive": 0 + } + }, + "round": 7, + "selectionObjective": 0.6286, + "syntheticRegression": { + "intentsMeeting95Precision": 7, + "macroF1": 0.439, + "macroPrecision": 0.9718, + "macroRecall": 0.3285, + "minimumPrecision": 0.8036, + "perIntent": { + "blessing": { + "f1": 0.0851, + "falseNegative": 86, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 4, + "recall": 0.0444, + "total": 2970, + "trueNegative": 2880, + "truePositive": 4, + "wilsonPrecisionLower95": 0.5101 + }, + "complaint": { + "f1": 0.3991, + "falseNegative": 270, + "falsePositive": 1, + "precision": 0.989, + "predictedPositive": 91, + "recall": 0.25, + "total": 2970, + "trueNegative": 2609, + "truePositive": 90, + "wilsonPrecisionLower95": 0.9404 + }, + "confirmationDecision": { + "f1": 0.6164, + "falseNegative": 90, + "falsePositive": 22, + "precision": 0.8036, + "predictedPositive": 112, + "recall": 0.5, + "total": 2970, + "trueNegative": 2768, + "truePositive": 90, + "wilsonPrecisionLower95": 0.7205 + }, + "followUpReminder": { + "f1": 0.2134, + "falseNegative": 317, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 43, + "recall": 0.1194, + "total": 2970, + "trueNegative": 2610, + "truePositive": 43, + "wilsonPrecisionLower95": 0.918 + }, + "invitation": { + "f1": 0.1053, + "falseNegative": 170, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 10, + "recall": 0.0556, + "total": 2970, + "trueNegative": 2790, + "truePositive": 10, + "wilsonPrecisionLower95": 0.7225 + }, + "question": { + "f1": 0.6751, + "falseNegative": 309, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 321, + "recall": 0.5095, + "total": 2970, + "trueNegative": 2340, + "truePositive": 321, + "wilsonPrecisionLower95": 0.9882 + }, + "replyableMessage": { + "f1": 0.8083, + "falseNegative": 572, + "falsePositive": 53, + "precision": 0.9613, + "predictedPositive": 1371, + "recall": 0.6974, + "total": 2970, + "trueNegative": 1027, + "truePositive": 1318, + "wilsonPrecisionLower95": 0.9498 + }, + "scheduleNegotiation": { + "f1": 0.7586, + "falseNegative": 35, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 55, + "recall": 0.6111, + "total": 2970, + "trueNegative": 2880, + "truePositive": 55, + "wilsonPrecisionLower95": 0.9347 + }, + "task": { + "f1": 0.2894, + "falseNegative": 598, + "falsePositive": 1, + "precision": 0.9919, + "predictedPositive": 123, + "recall": 0.1694, + "total": 2970, + "trueNegative": 2249, + "truePositive": 122, + "wilsonPrecisionLower95": 0.9554 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9809, + "falseNegative": 0, + "falsePositive": 7, + "precision": 0.9626, + "predictedPositive": 187, + "recall": 1.0, + "total": 1485, + "trueNegative": 1298, + "truePositive": 180, + "wilsonPrecisionLower95": 0.9248 + }, + "threshold": 0.74 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8974, + "falseNegative": 27, + "falsePositive": 8, + "precision": 0.9503, + "predictedPositive": 161, + "recall": 0.85, + "total": 1485, + "trueNegative": 1297, + "truePositive": 153, + "wilsonPrecisionLower95": 0.905 + }, + "threshold": 0.665 + } + }, + "metrics": { + "f1": 0.9529, + "falseNegative": 16, + "falsePositive": 18, + "precision": 0.9503, + "predictedPositive": 362, + "recall": 0.9556, + "total": 2970, + "trueNegative": 2592, + "truePositive": 344, + "wilsonPrecisionLower95": 0.9228 + }, + "threshold": 0.51 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.7763, + "falseNegative": 31, + "falsePositive": 3, + "precision": 0.9516, + "predictedPositive": 62, + "recall": 0.6556, + "total": 1485, + "trueNegative": 1392, + "truePositive": 59, + "wilsonPrecisionLower95": 0.8671 + }, + "threshold": 0.895 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7568, + "falseNegative": 34, + "falsePositive": 2, + "precision": 0.9655, + "predictedPositive": 58, + "recall": 0.6222, + "total": 1485, + "trueNegative": 1393, + "truePositive": 56, + "wilsonPrecisionLower95": 0.8827 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.6569, + "falseNegative": 90, + "falsePositive": 4, + "precision": 0.9574, + "predictedPositive": 94, + "recall": 0.5, + "total": 2970, + "trueNegative": 2786, + "truePositive": 90, + "wilsonPrecisionLower95": 0.8956 + }, + "threshold": 0.895 + }, + "followUpReminder": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8421, + "falseNegative": 44, + "falsePositive": 7, + "precision": 0.951, + "predictedPositive": 143, + "recall": 0.7556, + "total": 1485, + "trueNegative": 1298, + "truePositive": 136, + "wilsonPrecisionLower95": 0.9024 + }, + "threshold": 0.985 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.5985, + "falseNegative": 205, + "falsePositive": 3, + "precision": 0.981, + "predictedPositive": 158, + "recall": 0.4306, + "total": 2970, + "trueNegative": 2607, + "truePositive": 155, + "wilsonPrecisionLower95": 0.9457 + }, + "threshold": 0.995 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.7143, + "falseNegative": 135, + "falsePositive": 9, + "precision": 0.9524, + "predictedPositive": 189, + "recall": 0.5714, + "total": 1485, + "trueNegative": 1161, + "truePositive": 180, + "wilsonPrecisionLower95": 0.912 + }, + "threshold": 0.965 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7168, + "falseNegative": 134, + "falsePositive": 9, + "precision": 0.9526, + "predictedPositive": 190, + "recall": 0.5746, + "total": 1485, + "trueNegative": 1161, + "truePositive": 181, + "wilsonPrecisionLower95": 0.9124 + }, + "threshold": 0.52 + } + }, + "metrics": { + "f1": 0.7149, + "falseNegative": 269, + "falsePositive": 19, + "precision": 0.95, + "predictedPositive": 380, + "recall": 0.573, + "total": 2970, + "trueNegative": 2321, + "truePositive": 361, + "wilsonPrecisionLower95": 0.9232 + }, + "threshold": 0.795 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8846, + "falseNegative": 163, + "falsePositive": 41, + "precision": 0.9502, + "predictedPositive": 823, + "recall": 0.8275, + "total": 1485, + "trueNegative": 499, + "truePositive": 782, + "wilsonPrecisionLower95": 0.9331 + }, + "threshold": 0.89 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7476, + "falseNegative": 363, + "falsePositive": 30, + "precision": 0.951, + "predictedPositive": 612, + "recall": 0.6159, + "total": 1485, + "trueNegative": 510, + "truePositive": 582, + "wilsonPrecisionLower95": 0.9309 + }, + "threshold": 0.92 + } + }, + "metrics": { + "f1": 0.828, + "falseNegative": 506, + "falsePositive": 69, + "precision": 0.9525, + "predictedPositive": 1453, + "recall": 0.7323, + "total": 2970, + "trueNegative": 1011, + "truePositive": 1384, + "wilsonPrecisionLower95": 0.9403 + }, + "threshold": 0.91 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 45, + "recall": 1.0, + "total": 1485, + "trueNegative": 1440, + "truePositive": 45, + "wilsonPrecisionLower95": 0.9213 + }, + "threshold": 0.66 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 45, + "recall": 1.0, + "total": 1485, + "trueNegative": 1440, + "truePositive": 45, + "wilsonPrecisionLower95": 0.9213 + }, + "threshold": 0.55 + } + }, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 90, + "recall": 1.0, + "total": 2970, + "trueNegative": 2880, + "truePositive": 90, + "wilsonPrecisionLower95": 0.9591 + }, + "threshold": 0.66 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8615, + "falseNegative": 77, + "falsePositive": 14, + "precision": 0.9529, + "predictedPositive": 297, + "recall": 0.7861, + "total": 1485, + "trueNegative": 1111, + "truePositive": 283, + "wilsonPrecisionLower95": 0.9224 + }, + "threshold": 0.805 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.6868, + "falseNegative": 333, + "falsePositive": 20, + "precision": 0.9509, + "predictedPositive": 407, + "recall": 0.5375, + "total": 2970, + "trueNegative": 2230, + "truePositive": 387, + "wilsonPrecisionLower95": 0.9253 + }, + "threshold": 0.925 + } + }, + "trainingRecords": 17708, + "validation": { + "intentsMeeting95Precision": 9, + "macroF1": 0.4573, + "macroPrecision": 0.9836, + "macroRecall": 0.3282, + "minimumPrecision": 0.9505, + "perIntent": { + "blessing": { + "f1": 0.2353, + "falseNegative": 78, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 12, + "recall": 0.1333, + "total": 2970, + "trueNegative": 2880, + "truePositive": 12, + "wilsonPrecisionLower95": 0.7575 + }, + "complaint": { + "f1": 0.5366, + "falseNegative": 228, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 132, + "recall": 0.3667, + "total": 2970, + "trueNegative": 2610, + "truePositive": 132, + "wilsonPrecisionLower95": 0.9717 + }, + "confirmationDecision": { + "f1": 0.5781, + "falseNegative": 106, + "falsePositive": 2, + "precision": 0.9737, + "predictedPositive": 76, + "recall": 0.4111, + "total": 2970, + "trueNegative": 2788, + "truePositive": 74, + "wilsonPrecisionLower95": 0.909 + }, + "followUpReminder": { + "f1": 0.2035, + "falseNegative": 319, + "falsePositive": 2, + "precision": 0.9535, + "predictedPositive": 43, + "recall": 0.1139, + "total": 2970, + "trueNegative": 2608, + "truePositive": 41, + "wilsonPrecisionLower95": 0.8454 + }, + "invitation": { + "f1": 0.1818, + "falseNegative": 162, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 18, + "recall": 0.1, + "total": 2970, + "trueNegative": 2790, + "truePositive": 18, + "wilsonPrecisionLower95": 0.8241 + }, + "question": { + "f1": 0.7078, + "falseNegative": 280, + "falsePositive": 9, + "precision": 0.9749, + "predictedPositive": 359, + "recall": 0.5556, + "total": 2970, + "trueNegative": 2331, + "truePositive": 350, + "wilsonPrecisionLower95": 0.953 + }, + "replyableMessage": { + "f1": 0.8205, + "falseNegative": 526, + "falsePositive": 71, + "precision": 0.9505, + "predictedPositive": 1435, + "recall": 0.7217, + "total": 2970, + "trueNegative": 1009, + "truePositive": 1364, + "wilsonPrecisionLower95": 0.9381 + }, + "scheduleNegotiation": { + "f1": 0.5246, + "falseNegative": 58, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 32, + "recall": 0.3556, + "total": 2970, + "trueNegative": 2880, + "truePositive": 32, + "wilsonPrecisionLower95": 0.8928 + }, + "task": { + "f1": 0.3275, + "falseNegative": 579, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 141, + "recall": 0.1958, + "total": 2970, + "trueNegative": 2250, + "truePositive": 141, + "wilsonPrecisionLower95": 0.9735 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 5e-06, + "augmentation": "punctuation", + "char_max": 7, + "char_min": 2, + "external_weight": 0.7100000000000001, + "hard_example_weight": 1.7, + "l1_ratio": 0.15, + "max_char_features": 70000, + "max_word_features": 28000, + "min_df": 2, + "pseudo_weight": 0.27, + "round": 8, + "word_max": 2 + }, + "featureCount": 83313, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 0 + }, + "complaint": { + "negative": 187, + "positive": 0 + }, + "confirmationDecision": { + "negative": 107, + "positive": 2 + }, + "followUpReminder": { + "negative": 250, + "positive": 0 + }, + "invitation": { + "negative": 250, + "positive": 0 + }, + "question": { + "negative": 40, + "positive": 5 + }, + "replyableMessage": { + "negative": 2, + "positive": 135 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 0 + }, + "task": { + "negative": 94, + "positive": 0 + } + }, + "round": 8, + "selectionObjective": 0.6543, + "syntheticRegression": { + "intentsMeeting95Precision": 7, + "macroF1": 0.5256, + "macroPrecision": 0.9725, + "macroRecall": 0.4039, + "minimumPrecision": 0.8017, + "perIntent": { + "blessing": { + "f1": 0.125, + "falseNegative": 84, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 6, + "recall": 0.0667, + "total": 2970, + "trueNegative": 2880, + "truePositive": 6, + "wilsonPrecisionLower95": 0.6097 + }, + "complaint": { + "f1": 0.4279, + "falseNegative": 262, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 98, + "recall": 0.2722, + "total": 2970, + "trueNegative": 2610, + "truePositive": 98, + "wilsonPrecisionLower95": 0.9623 + }, + "confirmationDecision": { + "f1": 0.6284, + "falseNegative": 87, + "falsePositive": 23, + "precision": 0.8017, + "predictedPositive": 116, + "recall": 0.5167, + "total": 2970, + "trueNegative": 2767, + "truePositive": 93, + "wilsonPrecisionLower95": 0.72 + }, + "followUpReminder": { + "f1": 0.2686, + "falseNegative": 304, + "falsePositive": 1, + "precision": 0.9825, + "predictedPositive": 57, + "recall": 0.1556, + "total": 2970, + "trueNegative": 2609, + "truePositive": 56, + "wilsonPrecisionLower95": 0.9071 + }, + "invitation": { + "f1": 0.341, + "falseNegative": 143, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 37, + "recall": 0.2056, + "total": 2970, + "trueNegative": 2790, + "truePositive": 37, + "wilsonPrecisionLower95": 0.9059 + }, + "question": { + "f1": 0.7525, + "falseNegative": 250, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 380, + "recall": 0.6032, + "total": 2970, + "trueNegative": 2340, + "truePositive": 380, + "wilsonPrecisionLower95": 0.99 + }, + "replyableMessage": { + "f1": 0.8832, + "falseNegative": 366, + "falsePositive": 37, + "precision": 0.9763, + "predictedPositive": 1561, + "recall": 0.8063, + "total": 2970, + "trueNegative": 1043, + "truePositive": 1524, + "wilsonPrecisionLower95": 0.9675 + }, + "scheduleNegotiation": { + "f1": 0.8079, + "falseNegative": 29, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 61, + "recall": 0.6778, + "total": 2970, + "trueNegative": 2880, + "truePositive": 61, + "wilsonPrecisionLower95": 0.9408 + }, + "task": { + "f1": 0.4958, + "falseNegative": 482, + "falsePositive": 2, + "precision": 0.9917, + "predictedPositive": 240, + "recall": 0.3306, + "total": 2970, + "trueNegative": 2248, + "truePositive": 238, + "wilsonPrecisionLower95": 0.9701 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.989, + "falseNegative": 1, + "falsePositive": 3, + "precision": 0.9835, + "predictedPositive": 182, + "recall": 0.9944, + "total": 1485, + "trueNegative": 1302, + "truePositive": 179, + "wilsonPrecisionLower95": 0.9527 + }, + "threshold": 0.97 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9405, + "falseNegative": 14, + "falsePositive": 7, + "precision": 0.9595, + "predictedPositive": 173, + "recall": 0.9222, + "total": 1485, + "trueNegative": 1298, + "truePositive": 166, + "wilsonPrecisionLower95": 0.9188 + }, + "threshold": 0.54 + } + }, + "metrics": { + "f1": 0.9237, + "falseNegative": 39, + "falsePositive": 14, + "precision": 0.9582, + "predictedPositive": 335, + "recall": 0.8917, + "total": 2970, + "trueNegative": 2596, + "truePositive": 321, + "wilsonPrecisionLower95": 0.9311 + }, + "threshold": 0.925 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9613, + "falseNegative": 3, + "falsePositive": 4, + "precision": 0.956, + "predictedPositive": 91, + "recall": 0.9667, + "total": 1485, + "trueNegative": 1391, + "truePositive": 87, + "wilsonPrecisionLower95": 0.8924 + }, + "threshold": 0.72 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.5556, + "falseNegative": 55, + "falsePositive": 1, + "precision": 0.9722, + "predictedPositive": 36, + "recall": 0.3889, + "total": 1485, + "trueNegative": 1394, + "truePositive": 35, + "wilsonPrecisionLower95": 0.8583 + }, + "threshold": 0.905 + } + }, + "metrics": { + "f1": 0.7922, + "falseNegative": 58, + "falsePositive": 6, + "precision": 0.9531, + "predictedPositive": 128, + "recall": 0.6778, + "total": 2970, + "trueNegative": 2784, + "truePositive": 122, + "wilsonPrecisionLower95": 0.9015 + }, + "threshold": 0.88 + }, + "followUpReminder": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8437, + "falseNegative": 45, + "falsePositive": 5, + "precision": 0.9643, + "predictedPositive": 140, + "recall": 0.75, + "total": 1485, + "trueNegative": 1300, + "truePositive": 135, + "wilsonPrecisionLower95": 0.9191 + }, + "threshold": 0.96 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.7063, + "falseNegative": 158, + "falsePositive": 10, + "precision": 0.9528, + "predictedPositive": 212, + "recall": 0.5611, + "total": 2970, + "trueNegative": 2600, + "truePositive": 202, + "wilsonPrecisionLower95": 0.9154 + }, + "threshold": 0.96 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.7157, + "falseNegative": 135, + "falsePositive": 8, + "precision": 0.9574, + "predictedPositive": 188, + "recall": 0.5714, + "total": 1485, + "trueNegative": 1162, + "truePositive": 180, + "wilsonPrecisionLower95": 0.9183 + }, + "threshold": 0.975 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.822, + "falseNegative": 91, + "falsePositive": 6, + "precision": 0.9739, + "predictedPositive": 230, + "recall": 0.7111, + "total": 1485, + "trueNegative": 1164, + "truePositive": 224, + "wilsonPrecisionLower95": 0.9443 + }, + "threshold": 0.505 + } + }, + "metrics": { + "f1": 0.8095, + "falseNegative": 186, + "falsePositive": 23, + "precision": 0.9507, + "predictedPositive": 467, + "recall": 0.7048, + "total": 2970, + "trueNegative": 2317, + "truePositive": 444, + "wilsonPrecisionLower95": 0.9272 + }, + "threshold": 0.71 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8741, + "falseNegative": 181, + "falsePositive": 39, + "precision": 0.9514, + "predictedPositive": 803, + "recall": 0.8085, + "total": 1485, + "trueNegative": 501, + "truePositive": 764, + "wilsonPrecisionLower95": 0.9343 + }, + "threshold": 0.95 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8775, + "falseNegative": 175, + "falsePositive": 40, + "precision": 0.9506, + "predictedPositive": 810, + "recall": 0.8148, + "total": 1485, + "trueNegative": 500, + "truePositive": 770, + "wilsonPrecisionLower95": 0.9335 + }, + "threshold": 0.88 + } + }, + "metrics": { + "f1": 0.8762, + "falseNegative": 354, + "falsePositive": 80, + "precision": 0.9505, + "predictedPositive": 1616, + "recall": 0.8127, + "total": 2970, + "trueNegative": 1000, + "truePositive": 1536, + "wilsonPrecisionLower95": 0.9388 + }, + "threshold": 0.905 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 45, + "recall": 1.0, + "total": 1485, + "trueNegative": 1440, + "truePositive": 45, + "wilsonPrecisionLower95": 0.9213 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9556, + "falseNegative": 2, + "falsePositive": 2, + "precision": 0.9556, + "predictedPositive": 45, + "recall": 0.9556, + "total": 1485, + "trueNegative": 1438, + "truePositive": 43, + "wilsonPrecisionLower95": 0.8517 + }, + "threshold": 0.75 + } + }, + "metrics": { + "f1": 0.9724, + "falseNegative": 2, + "falsePositive": 3, + "precision": 0.967, + "predictedPositive": 91, + "recall": 0.9778, + "total": 2970, + "trueNegative": 2877, + "truePositive": 88, + "wilsonPrecisionLower95": 0.9075 + }, + "threshold": 0.505 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8331, + "falseNegative": 93, + "falsePositive": 14, + "precision": 0.9502, + "predictedPositive": 281, + "recall": 0.7417, + "total": 1485, + "trueNegative": 1111, + "truePositive": 267, + "wilsonPrecisionLower95": 0.9181 + }, + "threshold": 0.95 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7445, + "falseNegative": 140, + "falsePositive": 11, + "precision": 0.9524, + "predictedPositive": 231, + "recall": 0.6111, + "total": 1485, + "trueNegative": 1114, + "truePositive": 220, + "wilsonPrecisionLower95": 0.9168 + }, + "threshold": 0.86 + } + }, + "metrics": { + "f1": 0.7672, + "falseNegative": 257, + "falsePositive": 24, + "precision": 0.9507, + "predictedPositive": 487, + "recall": 0.6431, + "total": 2970, + "trueNegative": 2226, + "truePositive": 463, + "wilsonPrecisionLower95": 0.9277 + }, + "threshold": 0.92 + } + }, + "trainingRecords": 17762, + "validation": { + "intentsMeeting95Precision": 7, + "macroF1": 0.5016, + "macroPrecision": 0.9741, + "macroRecall": 0.3747, + "minimumPrecision": 0.9333, + "perIntent": { + "blessing": { + "f1": 0.2524, + "falseNegative": 77, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 13, + "recall": 0.1444, + "total": 2970, + "trueNegative": 2880, + "truePositive": 13, + "wilsonPrecisionLower95": 0.7719 + }, + "complaint": { + "f1": 0.5513, + "falseNegative": 223, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 137, + "recall": 0.3806, + "total": 2970, + "trueNegative": 2610, + "truePositive": 137, + "wilsonPrecisionLower95": 0.9727 + }, + "confirmationDecision": { + "f1": 0.6107, + "falseNegative": 100, + "falsePositive": 2, + "precision": 0.9756, + "predictedPositive": 82, + "recall": 0.4444, + "total": 2970, + "trueNegative": 2788, + "truePositive": 80, + "wilsonPrecisionLower95": 0.9154 + }, + "followUpReminder": { + "f1": 0.2074, + "falseNegative": 318, + "falsePositive": 3, + "precision": 0.9333, + "predictedPositive": 45, + "recall": 0.1167, + "total": 2970, + "trueNegative": 2607, + "truePositive": 42, + "wilsonPrecisionLower95": 0.8214 + }, + "invitation": { + "f1": 0.1633, + "falseNegative": 164, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 16, + "recall": 0.0889, + "total": 2970, + "trueNegative": 2790, + "truePositive": 16, + "wilsonPrecisionLower95": 0.8064 + }, + "question": { + "f1": 0.759, + "falseNegative": 238, + "falsePositive": 11, + "precision": 0.9727, + "predictedPositive": 403, + "recall": 0.6222, + "total": 2970, + "trueNegative": 2329, + "truePositive": 392, + "wilsonPrecisionLower95": 0.9518 + }, + "replyableMessage": { + "f1": 0.8758, + "falseNegative": 356, + "falsePositive": 79, + "precision": 0.951, + "predictedPositive": 1613, + "recall": 0.8116, + "total": 2970, + "trueNegative": 1001, + "truePositive": 1534, + "wilsonPrecisionLower95": 0.9394 + }, + "scheduleNegotiation": { + "f1": 0.5161, + "falseNegative": 58, + "falsePositive": 2, + "precision": 0.9412, + "predictedPositive": 34, + "recall": 0.3556, + "total": 2970, + "trueNegative": 2878, + "truePositive": 32, + "wilsonPrecisionLower95": 0.8091 + }, + "task": { + "f1": 0.5787, + "falseNegative": 426, + "falsePositive": 2, + "precision": 0.9932, + "predictedPositive": 296, + "recall": 0.4083, + "total": 2970, + "trueNegative": 2248, + "truePositive": 294, + "wilsonPrecisionLower95": 0.9757 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 4e-06, + "augmentation": "punctuation", + "char_max": 7, + "char_min": 3, + "external_weight": 0.73, + "hard_example_weight": 1.7, + "l1_ratio": 0.15, + "max_char_features": 70000, + "max_word_features": 35000, + "min_df": 2, + "pseudo_weight": 0.28500000000000003, + "round": 9, + "word_max": 3 + }, + "featureCount": 92641, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 0 + }, + "complaint": { + "negative": 250, + "positive": 0 + }, + "confirmationDecision": { + "negative": 248, + "positive": 6 + }, + "followUpReminder": { + "negative": 250, + "positive": 1 + }, + "invitation": { + "negative": 250, + "positive": 0 + }, + "question": { + "negative": 146, + "positive": 8 + }, + "replyableMessage": { + "negative": 8, + "positive": 250 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 0 + }, + "task": { + "negative": 208, + "positive": 0 + } + }, + "round": 9, + "selectionObjective": 0.6362, + "syntheticRegression": { + "intentsMeeting95Precision": 6, + "macroF1": 0.473, + "macroPrecision": 0.9657, + "macroRecall": 0.3476, + "minimumPrecision": 0.7955, + "perIntent": { + "blessing": { + "f1": 0.125, + "falseNegative": 84, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 6, + "recall": 0.0667, + "total": 2970, + "trueNegative": 2880, + "truePositive": 6, + "wilsonPrecisionLower95": 0.6097 + }, + "complaint": { + "f1": 0.4141, + "falseNegative": 266, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 94, + "recall": 0.2611, + "total": 2970, + "trueNegative": 2610, + "truePositive": 94, + "wilsonPrecisionLower95": 0.9607 + }, + "confirmationDecision": { + "f1": 0.5224, + "falseNegative": 110, + "falsePositive": 18, + "precision": 0.7955, + "predictedPositive": 88, + "recall": 0.3889, + "total": 2970, + "trueNegative": 2772, + "truePositive": 70, + "wilsonPrecisionLower95": 0.6997 + }, + "followUpReminder": { + "f1": 0.2255, + "falseNegative": 314, + "falsePositive": 2, + "precision": 0.9583, + "predictedPositive": 48, + "recall": 0.1278, + "total": 2970, + "trueNegative": 2608, + "truePositive": 46, + "wilsonPrecisionLower95": 0.8602 + }, + "invitation": { + "f1": 0.4071, + "falseNegative": 134, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 46, + "recall": 0.2556, + "total": 2970, + "trueNegative": 2790, + "truePositive": 46, + "wilsonPrecisionLower95": 0.9229 + }, + "question": { + "f1": 0.6452, + "falseNegative": 330, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 300, + "recall": 0.4762, + "total": 2970, + "trueNegative": 2340, + "truePositive": 300, + "wilsonPrecisionLower95": 0.9874 + }, + "replyableMessage": { + "f1": 0.7919, + "falseNegative": 594, + "falsePositive": 87, + "precision": 0.9371, + "predictedPositive": 1383, + "recall": 0.6857, + "total": 2970, + "trueNegative": 993, + "truePositive": 1296, + "wilsonPrecisionLower95": 0.923 + }, + "scheduleNegotiation": { + "f1": 0.8079, + "falseNegative": 29, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 61, + "recall": 0.6778, + "total": 2970, + "trueNegative": 2880, + "truePositive": 61, + "wilsonPrecisionLower95": 0.9408 + }, + "task": { + "f1": 0.3178, + "falseNegative": 584, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 136, + "recall": 0.1889, + "total": 2970, + "trueNegative": 2250, + "truePositive": 136, + "wilsonPrecisionLower95": 0.9725 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9862, + "falseNegative": 1, + "falsePositive": 4, + "precision": 0.9781, + "predictedPositive": 183, + "recall": 0.9944, + "total": 1485, + "trueNegative": 1301, + "truePositive": 179, + "wilsonPrecisionLower95": 0.9452 + }, + "threshold": 0.74 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9222, + "falseNegative": 20, + "falsePositive": 7, + "precision": 0.9581, + "predictedPositive": 167, + "recall": 0.8889, + "total": 1485, + "trueNegative": 1298, + "truePositive": 160, + "wilsonPrecisionLower95": 0.916 + }, + "threshold": 0.62 + } + }, + "metrics": { + "f1": 0.9528, + "falseNegative": 17, + "falsePositive": 17, + "precision": 0.9528, + "predictedPositive": 360, + "recall": 0.9528, + "total": 2970, + "trueNegative": 2593, + "truePositive": 343, + "wilsonPrecisionLower95": 0.9257 + }, + "threshold": 0.5 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9492, + "falseNegative": 6, + "falsePositive": 3, + "precision": 0.9655, + "predictedPositive": 87, + "recall": 0.9333, + "total": 1485, + "trueNegative": 1392, + "truePositive": 84, + "wilsonPrecisionLower95": 0.9035 + }, + "threshold": 0.965 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7042, + "falseNegative": 40, + "falsePositive": 2, + "precision": 0.9615, + "predictedPositive": 52, + "recall": 0.5556, + "total": 1485, + "trueNegative": 1393, + "truePositive": 50, + "wilsonPrecisionLower95": 0.8702 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.76, + "falseNegative": 66, + "falsePositive": 6, + "precision": 0.95, + "predictedPositive": 120, + "recall": 0.6333, + "total": 2970, + "trueNegative": 2784, + "truePositive": 114, + "wilsonPrecisionLower95": 0.8952 + }, + "threshold": 0.945 + }, + "followUpReminder": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8243, + "falseNegative": 51, + "falsePositive": 4, + "precision": 0.9699, + "predictedPositive": 133, + "recall": 0.7167, + "total": 1485, + "trueNegative": 1301, + "truePositive": 129, + "wilsonPrecisionLower95": 0.9252 + }, + "threshold": 0.99 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.6316, + "falseNegative": 192, + "falsePositive": 4, + "precision": 0.9767, + "predictedPositive": 172, + "recall": 0.4667, + "total": 2970, + "trueNegative": 2606, + "truePositive": 168, + "wilsonPrecisionLower95": 0.9417 + }, + "threshold": 0.995 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.544, + "falseNegative": 56, + "falsePositive": 1, + "precision": 0.9714, + "predictedPositive": 35, + "recall": 0.3778, + "total": 1485, + "trueNegative": 1394, + "truePositive": 34, + "wilsonPrecisionLower95": 0.8547 + }, + "threshold": 0.82 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.6694, + "falseNegative": 153, + "falsePositive": 7, + "precision": 0.9586, + "predictedPositive": 169, + "recall": 0.5143, + "total": 1485, + "trueNegative": 1163, + "truePositive": 162, + "wilsonPrecisionLower95": 0.917 + }, + "threshold": 0.985 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6762, + "falseNegative": 150, + "falsePositive": 8, + "precision": 0.9538, + "predictedPositive": 173, + "recall": 0.5238, + "total": 1485, + "trueNegative": 1162, + "truePositive": 165, + "wilsonPrecisionLower95": 0.9114 + }, + "threshold": 0.64 + } + }, + "metrics": { + "f1": 0.5392, + "falseNegative": 393, + "falsePositive": 12, + "precision": 0.9518, + "predictedPositive": 249, + "recall": 0.3762, + "total": 2970, + "trueNegative": 2328, + "truePositive": 237, + "wilsonPrecisionLower95": 0.9177 + }, + "threshold": 0.965 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9407, + "falseNegative": 65, + "falsePositive": 46, + "precision": 0.9503, + "predictedPositive": 926, + "recall": 0.9312, + "total": 1485, + "trueNegative": 494, + "truePositive": 880, + "wilsonPrecisionLower95": 0.9344 + }, + "threshold": 0.745 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6118, + "falseNegative": 521, + "falsePositive": 17, + "precision": 0.9615, + "predictedPositive": 441, + "recall": 0.4487, + "total": 1485, + "trueNegative": 523, + "truePositive": 424, + "wilsonPrecisionLower95": 0.9391 + }, + "threshold": 0.965 + } + }, + "metrics": { + "f1": 0.8037, + "falseNegative": 576, + "falsePositive": 66, + "precision": 0.9522, + "predictedPositive": 1380, + "recall": 0.6952, + "total": 2970, + "trueNegative": 1014, + "truePositive": 1314, + "wilsonPrecisionLower95": 0.9396 + }, + "threshold": 0.95 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9286, + "falseNegative": 6, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 39, + "recall": 0.8667, + "total": 1485, + "trueNegative": 1440, + "truePositive": 39, + "wilsonPrecisionLower95": 0.9103 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9302, + "falseNegative": 5, + "falsePositive": 1, + "precision": 0.9756, + "predictedPositive": 41, + "recall": 0.8889, + "total": 1485, + "trueNegative": 1439, + "truePositive": 40, + "wilsonPrecisionLower95": 0.874 + }, + "threshold": 0.61 + } + }, + "metrics": { + "f1": 0.9294, + "falseNegative": 11, + "falsePositive": 1, + "precision": 0.9875, + "predictedPositive": 80, + "recall": 0.8778, + "total": 2970, + "trueNegative": 2879, + "truePositive": 79, + "wilsonPrecisionLower95": 0.9325 + }, + "threshold": 0.61 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8403, + "falseNegative": 89, + "falsePositive": 14, + "precision": 0.9509, + "predictedPositive": 285, + "recall": 0.7528, + "total": 1485, + "trueNegative": 1111, + "truePositive": 271, + "wilsonPrecisionLower95": 0.9192 + }, + "threshold": 0.93 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.2384, + "falseNegative": 311, + "falsePositive": 2, + "precision": 0.9608, + "predictedPositive": 51, + "recall": 0.1361, + "total": 1485, + "trueNegative": 1123, + "truePositive": 49, + "wilsonPrecisionLower95": 0.8678 + }, + "threshold": 0.995 + } + }, + "metrics": { + "f1": 0.6415, + "falseNegative": 372, + "falsePositive": 17, + "precision": 0.9534, + "predictedPositive": 365, + "recall": 0.4833, + "total": 2970, + "trueNegative": 2233, + "truePositive": 348, + "wilsonPrecisionLower95": 0.9267 + }, + "threshold": 0.975 + } + }, + "trainingRecords": 17823, + "validation": { + "intentsMeeting95Precision": 8, + "macroF1": 0.4697, + "macroPrecision": 0.9844, + "macroRecall": 0.3337, + "minimumPrecision": 0.9268, + "perIntent": { + "blessing": { + "f1": 0.2857, + "falseNegative": 75, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 15, + "recall": 0.1667, + "total": 2970, + "trueNegative": 2880, + "truePositive": 15, + "wilsonPrecisionLower95": 0.7961 + }, + "complaint": { + "f1": 0.5366, + "falseNegative": 228, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 132, + "recall": 0.3667, + "total": 2970, + "trueNegative": 2610, + "truePositive": 132, + "wilsonPrecisionLower95": 0.9717 + }, + "confirmationDecision": { + "f1": 0.6415, + "falseNegative": 95, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 85, + "recall": 0.4722, + "total": 2970, + "trueNegative": 2790, + "truePositive": 85, + "wilsonPrecisionLower95": 0.9568 + }, + "followUpReminder": { + "f1": 0.1895, + "falseNegative": 322, + "falsePositive": 3, + "precision": 0.9268, + "predictedPositive": 41, + "recall": 0.1056, + "total": 2970, + "trueNegative": 2607, + "truePositive": 38, + "wilsonPrecisionLower95": 0.8057 + }, + "invitation": { + "f1": 0.2266, + "falseNegative": 157, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 23, + "recall": 0.1278, + "total": 2970, + "trueNegative": 2790, + "truePositive": 23, + "wilsonPrecisionLower95": 0.8569 + }, + "question": { + "f1": 0.6701, + "falseNegative": 309, + "falsePositive": 7, + "precision": 0.9787, + "predictedPositive": 328, + "recall": 0.5095, + "total": 2970, + "trueNegative": 2333, + "truePositive": 321, + "wilsonPrecisionLower95": 0.9566 + }, + "replyableMessage": { + "f1": 0.8007, + "falseNegative": 586, + "falsePositive": 63, + "precision": 0.9539, + "predictedPositive": 1367, + "recall": 0.6899, + "total": 2970, + "trueNegative": 1017, + "truePositive": 1304, + "wilsonPrecisionLower95": 0.9415 + }, + "scheduleNegotiation": { + "f1": 0.5, + "falseNegative": 60, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 30, + "recall": 0.3333, + "total": 2970, + "trueNegative": 2880, + "truePositive": 30, + "wilsonPrecisionLower95": 0.8865 + }, + "task": { + "f1": 0.3766, + "falseNegative": 553, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 167, + "recall": 0.2319, + "total": 2970, + "trueNegative": 2250, + "truePositive": 167, + "wilsonPrecisionLower95": 0.9775 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 6e-06, + "augmentation": "punctuation", + "char_max": 6, + "char_min": 2, + "external_weight": 0.75, + "hard_example_weight": 1.8, + "l1_ratio": 0.2, + "max_char_features": 75000, + "max_word_features": 38000, + "min_df": 1, + "pseudo_weight": 0.3, + "round": 10, + "word_max": 3 + }, + "featureCount": 113000, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 0 + }, + "complaint": { + "negative": 250, + "positive": 0 + }, + "confirmationDecision": { + "negative": 250, + "positive": 11 + }, + "followUpReminder": { + "negative": 250, + "positive": 4 + }, + "invitation": { + "negative": 250, + "positive": 0 + }, + "question": { + "negative": 182, + "positive": 10 + }, + "replyableMessage": { + "negative": 11, + "positive": 250 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 0 + }, + "task": { + "negative": 250, + "positive": 1 + } + }, + "round": 10, + "selectionObjective": 0.6131, + "syntheticRegression": { + "intentsMeeting95Precision": 7, + "macroF1": 0.4796, + "macroPrecision": 0.972, + "macroRecall": 0.3442, + "minimumPrecision": 0.8198, + "perIntent": { + "blessing": { + "f1": 0.125, + "falseNegative": 84, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 6, + "recall": 0.0667, + "total": 2970, + "trueNegative": 2880, + "truePositive": 6, + "wilsonPrecisionLower95": 0.6097 + }, + "complaint": { + "f1": 0.4148, + "falseNegative": 265, + "falsePositive": 3, + "precision": 0.9694, + "predictedPositive": 98, + "recall": 0.2639, + "total": 2970, + "trueNegative": 2607, + "truePositive": 95, + "wilsonPrecisionLower95": 0.9138 + }, + "confirmationDecision": { + "f1": 0.6254, + "falseNegative": 89, + "falsePositive": 20, + "precision": 0.8198, + "predictedPositive": 111, + "recall": 0.5056, + "total": 2970, + "trueNegative": 2770, + "truePositive": 91, + "wilsonPrecisionLower95": 0.738 + }, + "followUpReminder": { + "f1": 0.2768, + "falseNegative": 302, + "falsePositive": 1, + "precision": 0.9831, + "predictedPositive": 59, + "recall": 0.1611, + "total": 2970, + "trueNegative": 2609, + "truePositive": 58, + "wilsonPrecisionLower95": 0.91 + }, + "invitation": { + "f1": 0.3636, + "falseNegative": 140, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 40, + "recall": 0.2222, + "total": 2970, + "trueNegative": 2790, + "truePositive": 40, + "wilsonPrecisionLower95": 0.9124 + }, + "question": { + "f1": 0.5383, + "falseNegative": 398, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 232, + "recall": 0.3683, + "total": 2970, + "trueNegative": 2340, + "truePositive": 232, + "wilsonPrecisionLower95": 0.9837 + }, + "replyableMessage": { + "f1": 0.7216, + "falseNegative": 808, + "falsePositive": 27, + "precision": 0.9757, + "predictedPositive": 1109, + "recall": 0.5725, + "total": 2970, + "trueNegative": 1053, + "truePositive": 1082, + "wilsonPrecisionLower95": 0.9648 + }, + "scheduleNegotiation": { + "f1": 0.7586, + "falseNegative": 35, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 55, + "recall": 0.6111, + "total": 2970, + "trueNegative": 2880, + "truePositive": 55, + "wilsonPrecisionLower95": 0.9347 + }, + "task": { + "f1": 0.4921, + "falseNegative": 485, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 235, + "recall": 0.3264, + "total": 2970, + "trueNegative": 2250, + "truePositive": 235, + "wilsonPrecisionLower95": 0.9839 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9807, + "falseNegative": 2, + "falsePositive": 5, + "precision": 0.9727, + "predictedPositive": 183, + "recall": 0.9889, + "total": 1485, + "trueNegative": 1300, + "truePositive": 178, + "wilsonPrecisionLower95": 0.9376 + }, + "threshold": 0.765 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9288, + "falseNegative": 17, + "falsePositive": 8, + "precision": 0.9532, + "predictedPositive": 171, + "recall": 0.9056, + "total": 1485, + "trueNegative": 1297, + "truePositive": 163, + "wilsonPrecisionLower95": 0.9104 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.9455, + "falseNegative": 22, + "falsePositive": 17, + "precision": 0.9521, + "predictedPositive": 355, + "recall": 0.9389, + "total": 2970, + "trueNegative": 2593, + "truePositive": 338, + "wilsonPrecisionLower95": 0.9247 + }, + "threshold": 0.595 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.7568, + "falseNegative": 34, + "falsePositive": 2, + "precision": 0.9655, + "predictedPositive": 58, + "recall": 0.6222, + "total": 1485, + "trueNegative": 1393, + "truePositive": 56, + "wilsonPrecisionLower95": 0.8827 + }, + "threshold": 0.955 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7733, + "falseNegative": 32, + "falsePositive": 2, + "precision": 0.9667, + "predictedPositive": 60, + "recall": 0.6444, + "total": 1485, + "trueNegative": 1393, + "truePositive": 58, + "wilsonPrecisionLower95": 0.8864 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.7723, + "falseNegative": 63, + "falsePositive": 6, + "precision": 0.9512, + "predictedPositive": 123, + "recall": 0.65, + "total": 2970, + "trueNegative": 2784, + "truePositive": 117, + "wilsonPrecisionLower95": 0.8977 + }, + "threshold": 0.765 + }, + "followUpReminder": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8483, + "falseNegative": 43, + "falsePositive": 6, + "precision": 0.958, + "predictedPositive": 143, + "recall": 0.7611, + "total": 1485, + "trueNegative": 1299, + "truePositive": 137, + "wilsonPrecisionLower95": 0.9115 + }, + "threshold": 0.995 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.3529, + "falseNegative": 141, + "falsePositive": 2, + "precision": 0.9512, + "predictedPositive": 41, + "recall": 0.2167, + "total": 1485, + "trueNegative": 1303, + "truePositive": 39, + "wilsonPrecisionLower95": 0.8386 + }, + "threshold": 0.99 + } + }, + "metrics": { + "f1": 0.6292, + "falseNegative": 192, + "falsePositive": 6, + "precision": 0.9655, + "predictedPositive": 174, + "recall": 0.4667, + "total": 2970, + "trueNegative": 2604, + "truePositive": 168, + "wilsonPrecisionLower95": 0.9268 + }, + "threshold": 0.995 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.3894, + "falseNegative": 68, + "falsePositive": 1, + "precision": 0.9565, + "predictedPositive": 23, + "recall": 0.2444, + "total": 1485, + "trueNegative": 1394, + "truePositive": 22, + "wilsonPrecisionLower95": 0.7901 + }, + "threshold": 0.98 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8175, + "falseNegative": 91, + "falsePositive": 9, + "precision": 0.9614, + "predictedPositive": 233, + "recall": 0.7111, + "total": 1485, + "trueNegative": 1161, + "truePositive": 224, + "wilsonPrecisionLower95": 0.9282 + }, + "threshold": 0.6 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.7182, + "falseNegative": 401, + "falsePositive": 26, + "precision": 0.9544, + "predictedPositive": 570, + "recall": 0.5757, + "total": 1485, + "trueNegative": 514, + "truePositive": 544, + "wilsonPrecisionLower95": 0.934 + }, + "threshold": 0.985 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6667, + "falseNegative": 460, + "falsePositive": 25, + "precision": 0.951, + "predictedPositive": 510, + "recall": 0.5132, + "total": 1485, + "trueNegative": 515, + "truePositive": 485, + "wilsonPrecisionLower95": 0.9286 + }, + "threshold": 0.945 + } + }, + "metrics": { + "f1": 0.7151, + "falseNegative": 807, + "falsePositive": 56, + "precision": 0.9508, + "predictedPositive": 1139, + "recall": 0.573, + "total": 2970, + "trueNegative": 1024, + "truePositive": 1083, + "wilsonPrecisionLower95": 0.9367 + }, + "threshold": 0.96 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9412, + "falseNegative": 5, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 40, + "recall": 0.8889, + "total": 1485, + "trueNegative": 1440, + "truePositive": 40, + "wilsonPrecisionLower95": 0.9124 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8052, + "falseNegative": 14, + "falsePositive": 1, + "precision": 0.9688, + "predictedPositive": 32, + "recall": 0.6889, + "total": 1485, + "trueNegative": 1439, + "truePositive": 31, + "wilsonPrecisionLower95": 0.8426 + }, + "threshold": 0.98 + } + }, + "metrics": { + "f1": 0.8375, + "falseNegative": 23, + "falsePositive": 3, + "precision": 0.9571, + "predictedPositive": 70, + "recall": 0.7444, + "total": 2970, + "trueNegative": 2877, + "truePositive": 67, + "wilsonPrecisionLower95": 0.8814 + }, + "threshold": 0.92 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.7783, + "falseNegative": 123, + "falsePositive": 12, + "precision": 0.9518, + "predictedPositive": 249, + "recall": 0.6583, + "total": 1485, + "trueNegative": 1113, + "truePositive": 237, + "wilsonPrecisionLower95": 0.9177 + }, + "threshold": 0.84 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7994, + "falseNegative": 113, + "falsePositive": 11, + "precision": 0.9574, + "predictedPositive": 258, + "recall": 0.6861, + "total": 1485, + "trueNegative": 1114, + "truePositive": 247, + "wilsonPrecisionLower95": 0.9253 + }, + "threshold": 0.755 + } + }, + "metrics": { + "f1": 0.7977, + "falseNegative": 225, + "falsePositive": 26, + "precision": 0.9501, + "predictedPositive": 521, + "recall": 0.6875, + "total": 2970, + "trueNegative": 2224, + "truePositive": 495, + "wilsonPrecisionLower95": 0.9279 + }, + "threshold": 0.77 + } + }, + "trainingRecords": 17763, + "validation": { + "intentsMeeting95Precision": 7, + "macroF1": 0.4424, + "macroPrecision": 0.9693, + "macroRecall": 0.3051, + "minimumPrecision": 0.8889, + "perIntent": { + "blessing": { + "f1": 0.1818, + "falseNegative": 81, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 9, + "recall": 0.1, + "total": 2970, + "trueNegative": 2880, + "truePositive": 9, + "wilsonPrecisionLower95": 0.7009 + }, + "complaint": { + "f1": 0.5414, + "falseNegative": 226, + "falsePositive": 1, + "precision": 0.9926, + "predictedPositive": 135, + "recall": 0.3722, + "total": 2970, + "trueNegative": 2609, + "truePositive": 134, + "wilsonPrecisionLower95": 0.9592 + }, + "confirmationDecision": { + "f1": 0.5946, + "falseNegative": 103, + "falsePositive": 2, + "precision": 0.9747, + "predictedPositive": 79, + "recall": 0.4278, + "total": 2970, + "trueNegative": 2788, + "truePositive": 77, + "wilsonPrecisionLower95": 0.9123 + }, + "followUpReminder": { + "f1": 0.1975, + "falseNegative": 320, + "falsePositive": 5, + "precision": 0.8889, + "predictedPositive": 45, + "recall": 0.1111, + "total": 2970, + "trueNegative": 2605, + "truePositive": 40, + "wilsonPrecisionLower95": 0.765 + }, + "invitation": { + "f1": 0.209, + "falseNegative": 159, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 21, + "recall": 0.1167, + "total": 2970, + "trueNegative": 2790, + "truePositive": 21, + "wilsonPrecisionLower95": 0.8454 + }, + "question": { + "f1": 0.5216, + "falseNegative": 407, + "falsePositive": 2, + "precision": 0.9911, + "predictedPositive": 225, + "recall": 0.354, + "total": 2970, + "trueNegative": 2338, + "truePositive": 223, + "wilsonPrecisionLower95": 0.9682 + }, + "replyableMessage": { + "f1": 0.6929, + "falseNegative": 861, + "falsePositive": 51, + "precision": 0.9528, + "predictedPositive": 1080, + "recall": 0.5444, + "total": 2970, + "trueNegative": 1029, + "truePositive": 1029, + "wilsonPrecisionLower95": 0.9384 + }, + "scheduleNegotiation": { + "f1": 0.4706, + "falseNegative": 62, + "falsePositive": 1, + "precision": 0.9655, + "predictedPositive": 29, + "recall": 0.3111, + "total": 2970, + "trueNegative": 2879, + "truePositive": 28, + "wilsonPrecisionLower95": 0.8282 + }, + "task": { + "f1": 0.5725, + "falseNegative": 426, + "falsePositive": 13, + "precision": 0.9577, + "predictedPositive": 307, + "recall": 0.4083, + "total": 2970, + "trueNegative": 2237, + "truePositive": 294, + "wilsonPrecisionLower95": 0.9289 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 5e-06, + "augmentation": "prefix", + "char_max": 6, + "char_min": 2, + "external_weight": 0.77, + "hard_example_weight": 1.8, + "l1_ratio": 0.1, + "max_char_features": 70000, + "max_word_features": 30000, + "min_df": 2, + "pseudo_weight": 0.31499999999999995, + "round": 11, + "word_max": 2 + }, + "featureCount": 81451, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 0 + }, + "complaint": { + "negative": 250, + "positive": 0 + }, + "confirmationDecision": { + "negative": 250, + "positive": 14 + }, + "followUpReminder": { + "negative": 250, + "positive": 2 + }, + "invitation": { + "negative": 250, + "positive": 1 + }, + "question": { + "negative": 233, + "positive": 0 + }, + "replyableMessage": { + "negative": 8, + "positive": 250 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 0 + }, + "task": { + "negative": 250, + "positive": 0 + } + }, + "round": 11, + "selectionObjective": 0.6682, + "syntheticRegression": { + "intentsMeeting95Precision": 6, + "macroF1": 0.5232, + "macroPrecision": 0.9846, + "macroRecall": 0.4153, + "minimumPrecision": 0.93, + "perIntent": { + "blessing": { + "f1": 0.125, + "falseNegative": 84, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 6, + "recall": 0.0667, + "total": 2970, + "trueNegative": 2880, + "truePositive": 6, + "wilsonPrecisionLower95": 0.6097 + }, + "complaint": { + "f1": 0.4245, + "falseNegative": 263, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 97, + "recall": 0.2694, + "total": 2970, + "trueNegative": 2610, + "truePositive": 97, + "wilsonPrecisionLower95": 0.9619 + }, + "confirmationDecision": { + "f1": 0.6643, + "falseNegative": 87, + "falsePositive": 7, + "precision": 0.93, + "predictedPositive": 100, + "recall": 0.5167, + "total": 2970, + "trueNegative": 2783, + "truePositive": 93, + "wilsonPrecisionLower95": 0.8625 + }, + "followUpReminder": { + "f1": 0.0902, + "falseNegative": 343, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 17, + "recall": 0.0472, + "total": 2970, + "trueNegative": 2610, + "truePositive": 17, + "wilsonPrecisionLower95": 0.8157 + }, + "invitation": { + "f1": 0.341, + "falseNegative": 143, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 37, + "recall": 0.2056, + "total": 2970, + "trueNegative": 2790, + "truePositive": 37, + "wilsonPrecisionLower95": 0.9059 + }, + "question": { + "f1": 0.8191, + "falseNegative": 193, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 437, + "recall": 0.6937, + "total": 2970, + "trueNegative": 2340, + "truePositive": 437, + "wilsonPrecisionLower95": 0.9913 + }, + "replyableMessage": { + "f1": 0.8975, + "falseNegative": 253, + "falsePositive": 121, + "precision": 0.9312, + "predictedPositive": 1758, + "recall": 0.8661, + "total": 2970, + "trueNegative": 959, + "truePositive": 1637, + "wilsonPrecisionLower95": 0.9184 + }, + "scheduleNegotiation": { + "f1": 0.8535, + "falseNegative": 23, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 67, + "recall": 0.7444, + "total": 2970, + "trueNegative": 2880, + "truePositive": 67, + "wilsonPrecisionLower95": 0.9458 + }, + "task": { + "f1": 0.4937, + "falseNegative": 484, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 236, + "recall": 0.3278, + "total": 2970, + "trueNegative": 2250, + "truePositive": 236, + "wilsonPrecisionLower95": 0.984 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9917, + "falseNegative": 0, + "falsePositive": 3, + "precision": 0.9836, + "predictedPositive": 183, + "recall": 1.0, + "total": 1485, + "trueNegative": 1302, + "truePositive": 180, + "wilsonPrecisionLower95": 0.9529 + }, + "threshold": 0.99 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9318, + "falseNegative": 16, + "falsePositive": 8, + "precision": 0.9535, + "predictedPositive": 172, + "recall": 0.9111, + "total": 1485, + "trueNegative": 1297, + "truePositive": 164, + "wilsonPrecisionLower95": 0.9109 + }, + "threshold": 0.84 + } + }, + "metrics": { + "f1": 0.9529, + "falseNegative": 16, + "falsePositive": 18, + "precision": 0.9503, + "predictedPositive": 362, + "recall": 0.9556, + "total": 2970, + "trueNegative": 2592, + "truePositive": 344, + "wilsonPrecisionLower95": 0.9228 + }, + "threshold": 0.84 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.931, + "falseNegative": 9, + "falsePositive": 3, + "precision": 0.9643, + "predictedPositive": 84, + "recall": 0.9, + "total": 1485, + "trueNegative": 1392, + "truePositive": 81, + "wilsonPrecisionLower95": 0.9002 + }, + "threshold": 0.97 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7222, + "falseNegative": 38, + "falsePositive": 2, + "precision": 0.963, + "predictedPositive": 54, + "recall": 0.5778, + "total": 1485, + "trueNegative": 1393, + "truePositive": 52, + "wilsonPrecisionLower95": 0.8746 + }, + "threshold": 0.86 + } + }, + "metrics": { + "f1": 0.8077, + "falseNegative": 54, + "falsePositive": 6, + "precision": 0.9545, + "predictedPositive": 132, + "recall": 0.7, + "total": 2970, + "trueNegative": 2784, + "truePositive": 126, + "wilsonPrecisionLower95": 0.9044 + }, + "threshold": 0.95 + }, + "followUpReminder": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.4833, + "falseNegative": 61, + "falsePositive": 1, + "precision": 0.9667, + "predictedPositive": 30, + "recall": 0.3222, + "total": 1485, + "trueNegative": 1394, + "truePositive": 29, + "wilsonPrecisionLower95": 0.8333 + }, + "threshold": 0.965 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.7993, + "falseNegative": 98, + "falsePositive": 11, + "precision": 0.9518, + "predictedPositive": 228, + "recall": 0.6889, + "total": 1485, + "trueNegative": 1159, + "truePositive": 217, + "wilsonPrecisionLower95": 0.9157 + }, + "threshold": 0.92 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7572, + "falseNegative": 117, + "falsePositive": 10, + "precision": 0.9519, + "predictedPositive": 208, + "recall": 0.6286, + "total": 1485, + "trueNegative": 1160, + "truePositive": 198, + "wilsonPrecisionLower95": 0.9138 + }, + "threshold": 0.735 + } + }, + "metrics": { + "f1": 0.797, + "falseNegative": 198, + "falsePositive": 22, + "precision": 0.9515, + "predictedPositive": 454, + "recall": 0.6857, + "total": 2970, + "trueNegative": 2318, + "truePositive": 432, + "wilsonPrecisionLower95": 0.9277 + }, + "threshold": 0.8 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9373, + "falseNegative": 71, + "falsePositive": 46, + "precision": 0.95, + "predictedPositive": 920, + "recall": 0.9249, + "total": 1485, + "trueNegative": 494, + "truePositive": 874, + "wilsonPrecisionLower95": 0.934 + }, + "threshold": 0.79 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8754, + "falseNegative": 179, + "falsePositive": 39, + "precision": 0.9516, + "predictedPositive": 805, + "recall": 0.8106, + "total": 1485, + "trueNegative": 501, + "truePositive": 766, + "wilsonPrecisionLower95": 0.9345 + }, + "threshold": 0.91 + } + }, + "metrics": { + "f1": 0.9105, + "falseNegative": 242, + "falsePositive": 82, + "precision": 0.9526, + "predictedPositive": 1730, + "recall": 0.872, + "total": 2970, + "trueNegative": 998, + "truePositive": 1648, + "wilsonPrecisionLower95": 0.9415 + }, + "threshold": 0.895 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9773, + "falseNegative": 2, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 43, + "recall": 0.9556, + "total": 1485, + "trueNegative": 1440, + "truePositive": 43, + "wilsonPrecisionLower95": 0.918 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9318, + "falseNegative": 4, + "falsePositive": 2, + "precision": 0.9535, + "predictedPositive": 43, + "recall": 0.9111, + "total": 1485, + "trueNegative": 1438, + "truePositive": 41, + "wilsonPrecisionLower95": 0.8454 + }, + "threshold": 0.9 + } + }, + "metrics": { + "f1": 0.9551, + "falseNegative": 5, + "falsePositive": 3, + "precision": 0.9659, + "predictedPositive": 88, + "recall": 0.9444, + "total": 2970, + "trueNegative": 2877, + "truePositive": 85, + "wilsonPrecisionLower95": 0.9045 + }, + "threshold": 0.885 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8315, + "falseNegative": 96, + "falsePositive": 11, + "precision": 0.96, + "predictedPositive": 275, + "recall": 0.7333, + "total": 1485, + "trueNegative": 1114, + "truePositive": 264, + "wilsonPrecisionLower95": 0.9298 + }, + "threshold": 0.985 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8349, + "falseNegative": 92, + "falsePositive": 14, + "precision": 0.9504, + "predictedPositive": 282, + "recall": 0.7444, + "total": 1485, + "trueNegative": 1111, + "truePositive": 268, + "wilsonPrecisionLower95": 0.9184 + }, + "threshold": 0.845 + } + }, + "metrics": { + "f1": 0.7688, + "falseNegative": 256, + "falsePositive": 23, + "precision": 0.9528, + "predictedPositive": 487, + "recall": 0.6444, + "total": 2970, + "trueNegative": 2227, + "truePositive": 464, + "wilsonPrecisionLower95": 0.9301 + }, + "threshold": 0.975 + } + }, + "trainingRecords": 15345, + "validation": { + "intentsMeeting95Precision": 8, + "macroF1": 0.5184, + "macroPrecision": 0.982, + "macroRecall": 0.3943, + "minimumPrecision": 0.9412, + "perIntent": { + "blessing": { + "f1": 0.2692, + "falseNegative": 76, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 14, + "recall": 0.1556, + "total": 2970, + "trueNegative": 2880, + "truePositive": 14, + "wilsonPrecisionLower95": 0.7847 + }, + "complaint": { + "f1": 0.5542, + "falseNegative": 222, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 138, + "recall": 0.3833, + "total": 2970, + "trueNegative": 2610, + "truePositive": 138, + "wilsonPrecisionLower95": 0.9729 + }, + "confirmationDecision": { + "f1": 0.6642, + "falseNegative": 90, + "falsePositive": 1, + "precision": 0.989, + "predictedPositive": 91, + "recall": 0.5, + "total": 2970, + "trueNegative": 2789, + "truePositive": 90, + "wilsonPrecisionLower95": 0.9404 + }, + "followUpReminder": { + "f1": 0.1347, + "falseNegative": 334, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 26, + "recall": 0.0722, + "total": 2970, + "trueNegative": 2610, + "truePositive": 26, + "wilsonPrecisionLower95": 0.8713 + }, + "invitation": { + "f1": 0.2524, + "falseNegative": 154, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 26, + "recall": 0.1444, + "total": 2970, + "trueNegative": 2790, + "truePositive": 26, + "wilsonPrecisionLower95": 0.8713 + }, + "question": { + "f1": 0.7757, + "falseNegative": 222, + "falsePositive": 14, + "precision": 0.9668, + "predictedPositive": 422, + "recall": 0.6476, + "total": 2970, + "trueNegative": 2326, + "truePositive": 408, + "wilsonPrecisionLower95": 0.9451 + }, + "replyableMessage": { + "f1": 0.9073, + "falseNegative": 250, + "falsePositive": 85, + "precision": 0.9507, + "predictedPositive": 1725, + "recall": 0.8677, + "total": 2970, + "trueNegative": 995, + "truePositive": 1640, + "wilsonPrecisionLower95": 0.9395 + }, + "scheduleNegotiation": { + "f1": 0.5161, + "falseNegative": 58, + "falsePositive": 2, + "precision": 0.9412, + "predictedPositive": 34, + "recall": 0.3556, + "total": 2970, + "trueNegative": 2878, + "truePositive": 32, + "wilsonPrecisionLower95": 0.8091 + }, + "task": { + "f1": 0.592, + "falseNegative": 416, + "falsePositive": 3, + "precision": 0.9902, + "predictedPositive": 307, + "recall": 0.4222, + "total": 2970, + "trueNegative": 2247, + "truePositive": 304, + "wilsonPrecisionLower95": 0.9717 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 4e-06, + "augmentation": "prefix", + "char_max": 6, + "char_min": 3, + "external_weight": 0.79, + "hard_example_weight": 2.0, + "l1_ratio": 0.15, + "max_char_features": 75000, + "max_word_features": 35000, + "min_df": 2, + "pseudo_weight": 0.32999999999999996, + "round": 12, + "word_max": 3 + }, + "featureCount": 94458, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 0 + }, + "complaint": { + "negative": 250, + "positive": 0 + }, + "confirmationDecision": { + "negative": 250, + "positive": 15 + }, + "followUpReminder": { + "negative": 250, + "positive": 0 + }, + "invitation": { + "negative": 250, + "positive": 0 + }, + "question": { + "negative": 232, + "positive": 0 + }, + "replyableMessage": { + "negative": 10, + "positive": 250 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 0 + }, + "task": { + "negative": 250, + "positive": 0 + } + }, + "round": 12, + "selectionObjective": 0.6224, + "syntheticRegression": { + "intentsMeeting95Precision": 7, + "macroF1": 0.4599, + "macroPrecision": 0.9738, + "macroRecall": 0.3356, + "minimumPrecision": 0.8053, + "perIntent": { + "blessing": { + "f1": 0.1053, + "falseNegative": 85, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 5, + "recall": 0.0556, + "total": 2970, + "trueNegative": 2880, + "truePositive": 5, + "wilsonPrecisionLower95": 0.5655 + }, + "complaint": { + "f1": 0.3964, + "falseNegative": 271, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 89, + "recall": 0.2472, + "total": 2970, + "trueNegative": 2610, + "truePositive": 89, + "wilsonPrecisionLower95": 0.9586 + }, + "confirmationDecision": { + "f1": 0.6212, + "falseNegative": 89, + "falsePositive": 22, + "precision": 0.8053, + "predictedPositive": 113, + "recall": 0.5056, + "total": 2970, + "trueNegative": 2768, + "truePositive": 91, + "wilsonPrecisionLower95": 0.7228 + }, + "followUpReminder": { + "f1": 0.2651, + "falseNegative": 305, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 55, + "recall": 0.1528, + "total": 2970, + "trueNegative": 2610, + "truePositive": 55, + "wilsonPrecisionLower95": 0.9347 + }, + "invitation": { + "f1": 0.341, + "falseNegative": 143, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 37, + "recall": 0.2056, + "total": 2970, + "trueNegative": 2790, + "truePositive": 37, + "wilsonPrecisionLower95": 0.9059 + }, + "question": { + "f1": 0.427, + "falseNegative": 459, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 171, + "recall": 0.2714, + "total": 2970, + "trueNegative": 2340, + "truePositive": 171, + "wilsonPrecisionLower95": 0.978 + }, + "replyableMessage": { + "f1": 0.7888, + "falseNegative": 624, + "falsePositive": 54, + "precision": 0.9591, + "predictedPositive": 1320, + "recall": 0.6698, + "total": 2970, + "trueNegative": 1026, + "truePositive": 1266, + "wilsonPrecisionLower95": 0.947 + }, + "scheduleNegotiation": { + "f1": 0.8, + "falseNegative": 30, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 60, + "recall": 0.6667, + "total": 2970, + "trueNegative": 2880, + "truePositive": 60, + "wilsonPrecisionLower95": 0.9398 + }, + "task": { + "f1": 0.3946, + "falseNegative": 543, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 177, + "recall": 0.2458, + "total": 2970, + "trueNegative": 2250, + "truePositive": 177, + "wilsonPrecisionLower95": 0.9788 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9917, + "falseNegative": 0, + "falsePositive": 3, + "precision": 0.9836, + "predictedPositive": 183, + "recall": 1.0, + "total": 1485, + "trueNegative": 1302, + "truePositive": 180, + "wilsonPrecisionLower95": 0.9529 + }, + "threshold": 0.885 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8974, + "falseNegative": 27, + "falsePositive": 8, + "precision": 0.9503, + "predictedPositive": 161, + "recall": 0.85, + "total": 1485, + "trueNegative": 1297, + "truePositive": 153, + "wilsonPrecisionLower95": 0.905 + }, + "threshold": 0.9 + } + }, + "metrics": { + "f1": 0.9515, + "falseNegative": 17, + "falsePositive": 18, + "precision": 0.9501, + "predictedPositive": 361, + "recall": 0.9528, + "total": 2970, + "trueNegative": 2592, + "truePositive": 343, + "wilsonPrecisionLower95": 0.9226 + }, + "threshold": 0.77 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9944, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 89, + "recall": 0.9889, + "total": 1485, + "trueNegative": 1395, + "truePositive": 89, + "wilsonPrecisionLower95": 0.9586 + }, + "threshold": 0.915 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.4959, + "falseNegative": 60, + "falsePositive": 1, + "precision": 0.9677, + "predictedPositive": 31, + "recall": 0.3333, + "total": 1485, + "trueNegative": 1394, + "truePositive": 30, + "wilsonPrecisionLower95": 0.8381 + }, + "threshold": 0.985 + } + }, + "metrics": { + "f1": 0.8217, + "falseNegative": 51, + "falsePositive": 5, + "precision": 0.9627, + "predictedPositive": 134, + "recall": 0.7167, + "total": 2970, + "trueNegative": 2785, + "truePositive": 129, + "wilsonPrecisionLower95": 0.9156 + }, + "threshold": 0.915 + }, + "followUpReminder": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.5647, + "falseNegative": 108, + "falsePositive": 3, + "precision": 0.96, + "predictedPositive": 75, + "recall": 0.4, + "total": 1485, + "trueNegative": 1302, + "truePositive": 72, + "wilsonPrecisionLower95": 0.8889 + }, + "threshold": 0.995 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6542, + "falseNegative": 158, + "falsePositive": 8, + "precision": 0.9515, + "predictedPositive": 165, + "recall": 0.4984, + "total": 1485, + "trueNegative": 1162, + "truePositive": 157, + "wilsonPrecisionLower95": 0.9073 + }, + "threshold": 0.805 + } + }, + "metrics": { + "f1": 0.5711, + "falseNegative": 373, + "falsePositive": 13, + "precision": 0.9519, + "predictedPositive": 270, + "recall": 0.4079, + "total": 2970, + "trueNegative": 2327, + "truePositive": 257, + "wilsonPrecisionLower95": 0.9194 + }, + "threshold": 0.985 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8956, + "falseNegative": 147, + "falsePositive": 39, + "precision": 0.9534, + "predictedPositive": 837, + "recall": 0.8444, + "total": 1485, + "trueNegative": 501, + "truePositive": 798, + "wilsonPrecisionLower95": 0.9369 + }, + "threshold": 0.975 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6735, + "falseNegative": 453, + "falsePositive": 24, + "precision": 0.9535, + "predictedPositive": 516, + "recall": 0.5206, + "total": 1485, + "trueNegative": 516, + "truePositive": 492, + "wilsonPrecisionLower95": 0.9317 + }, + "threshold": 0.985 + } + }, + "metrics": { + "f1": 0.7866, + "falseNegative": 626, + "falsePositive": 60, + "precision": 0.9547, + "predictedPositive": 1324, + "recall": 0.6688, + "total": 2970, + "trueNegative": 1020, + "truePositive": 1264, + "wilsonPrecisionLower95": 0.9421 + }, + "threshold": 0.985 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9888, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 44, + "recall": 0.9778, + "total": 1485, + "trueNegative": 1440, + "truePositive": 44, + "wilsonPrecisionLower95": 0.9197 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8052, + "falseNegative": 14, + "falsePositive": 1, + "precision": 0.9688, + "predictedPositive": 32, + "recall": 0.6889, + "total": 1485, + "trueNegative": 1439, + "truePositive": 31, + "wilsonPrecisionLower95": 0.8426 + }, + "threshold": 0.94 + } + }, + "metrics": { + "f1": 0.9364, + "falseNegative": 9, + "falsePositive": 2, + "precision": 0.9759, + "predictedPositive": 83, + "recall": 0.9, + "total": 2970, + "trueNegative": 2878, + "truePositive": 81, + "wilsonPrecisionLower95": 0.9163 + }, + "threshold": 0.715 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8475, + "falseNegative": 85, + "falsePositive": 14, + "precision": 0.9516, + "predictedPositive": 289, + "recall": 0.7639, + "total": 1485, + "trueNegative": 1111, + "truePositive": 275, + "wilsonPrecisionLower95": 0.9203 + }, + "threshold": 0.985 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.501, + "falseNegative": 238, + "falsePositive": 5, + "precision": 0.9606, + "predictedPositive": 127, + "recall": 0.3389, + "total": 1485, + "trueNegative": 1120, + "truePositive": 122, + "wilsonPrecisionLower95": 0.9111 + }, + "threshold": 0.985 + } + }, + "metrics": { + "f1": 0.6989, + "falseNegative": 323, + "falsePositive": 19, + "precision": 0.9543, + "predictedPositive": 416, + "recall": 0.5514, + "total": 2970, + "trueNegative": 2231, + "truePositive": 397, + "wilsonPrecisionLower95": 0.9298 + }, + "threshold": 0.985 + } + }, + "trainingRecords": 15345, + "validation": { + "intentsMeeting95Precision": 9, + "macroF1": 0.445, + "macroPrecision": 0.9912, + "macroRecall": 0.307, + "minimumPrecision": 0.9534, + "perIntent": { + "blessing": { + "f1": 0.2857, + "falseNegative": 75, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 15, + "recall": 0.1667, + "total": 2970, + "trueNegative": 2880, + "truePositive": 15, + "wilsonPrecisionLower95": 0.7961 + }, + "complaint": { + "f1": 0.5355, + "falseNegative": 228, + "falsePositive": 1, + "precision": 0.9925, + "predictedPositive": 133, + "recall": 0.3667, + "total": 2970, + "trueNegative": 2609, + "truePositive": 132, + "wilsonPrecisionLower95": 0.9586 + }, + "confirmationDecision": { + "f1": 0.5827, + "falseNegative": 106, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 74, + "recall": 0.4111, + "total": 2970, + "trueNegative": 2790, + "truePositive": 74, + "wilsonPrecisionLower95": 0.9507 + }, + "followUpReminder": { + "f1": 0.2045, + "falseNegative": 319, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 41, + "recall": 0.1139, + "total": 2970, + "trueNegative": 2610, + "truePositive": 41, + "wilsonPrecisionLower95": 0.9143 + }, + "invitation": { + "f1": 0.2266, + "falseNegative": 157, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 23, + "recall": 0.1278, + "total": 2970, + "trueNegative": 2790, + "truePositive": 23, + "wilsonPrecisionLower95": 0.8569 + }, + "question": { + "f1": 0.42, + "falseNegative": 462, + "falsePositive": 2, + "precision": 0.9882, + "predictedPositive": 170, + "recall": 0.2667, + "total": 2970, + "trueNegative": 2338, + "truePositive": 168, + "wilsonPrecisionLower95": 0.9581 + }, + "replyableMessage": { + "f1": 0.7956, + "falseNegative": 600, + "falsePositive": 63, + "precision": 0.9534, + "predictedPositive": 1353, + "recall": 0.6825, + "total": 2970, + "trueNegative": 1017, + "truePositive": 1290, + "wilsonPrecisionLower95": 0.9409 + }, + "scheduleNegotiation": { + "f1": 0.4874, + "falseNegative": 61, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 29, + "recall": 0.3222, + "total": 2970, + "trueNegative": 2880, + "truePositive": 29, + "wilsonPrecisionLower95": 0.883 + }, + "task": { + "f1": 0.4666, + "falseNegative": 500, + "falsePositive": 3, + "precision": 0.9865, + "predictedPositive": 223, + "recall": 0.3056, + "total": 2970, + "trueNegative": 2247, + "truePositive": 220, + "wilsonPrecisionLower95": 0.9612 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 3e-06, + "augmentation": "prefix", + "char_max": 7, + "char_min": 2, + "external_weight": 0.81, + "hard_example_weight": 2.0, + "l1_ratio": 0.15, + "max_char_features": 80000, + "max_word_features": 38000, + "min_df": 2, + "pseudo_weight": 0.345, + "round": 13, + "word_max": 3 + }, + "featureCount": 99458, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 0 + }, + "complaint": { + "negative": 250, + "positive": 1 + }, + "confirmationDecision": { + "negative": 250, + "positive": 18 + }, + "followUpReminder": { + "negative": 250, + "positive": 0 + }, + "invitation": { + "negative": 250, + "positive": 0 + }, + "question": { + "negative": 250, + "positive": 0 + }, + "replyableMessage": { + "negative": 12, + "positive": 250 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 0 + }, + "task": { + "negative": 250, + "positive": 0 + } + }, + "round": 13, + "selectionObjective": 0.6381, + "syntheticRegression": { + "intentsMeeting95Precision": 6, + "macroF1": 0.4906, + "macroPrecision": 0.9736, + "macroRecall": 0.3748, + "minimumPrecision": 0.8182, + "perIntent": { + "blessing": { + "f1": 0.1443, + "falseNegative": 83, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 7, + "recall": 0.0778, + "total": 2970, + "trueNegative": 2880, + "truePositive": 7, + "wilsonPrecisionLower95": 0.6457 + }, + "complaint": { + "f1": 0.4279, + "falseNegative": 262, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 98, + "recall": 0.2722, + "total": 2970, + "trueNegative": 2610, + "truePositive": 98, + "wilsonPrecisionLower95": 0.9623 + }, + "confirmationDecision": { + "f1": 0.6207, + "falseNegative": 90, + "falsePositive": 20, + "precision": 0.8182, + "predictedPositive": 110, + "recall": 0.5, + "total": 2970, + "trueNegative": 2770, + "truePositive": 90, + "wilsonPrecisionLower95": 0.7358 + }, + "followUpReminder": { + "f1": 0.1201, + "falseNegative": 337, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 23, + "recall": 0.0639, + "total": 2970, + "trueNegative": 2610, + "truePositive": 23, + "wilsonPrecisionLower95": 0.8569 + }, + "invitation": { + "f1": 0.3486, + "falseNegative": 142, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 38, + "recall": 0.2111, + "total": 2970, + "trueNegative": 2790, + "truePositive": 38, + "wilsonPrecisionLower95": 0.9082 + }, + "question": { + "f1": 0.5124, + "falseNegative": 413, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 217, + "recall": 0.3444, + "total": 2970, + "trueNegative": 2340, + "truePositive": 217, + "wilsonPrecisionLower95": 0.9826 + }, + "replyableMessage": { + "f1": 0.9179, + "falseNegative": 202, + "falsePositive": 100, + "precision": 0.9441, + "predictedPositive": 1788, + "recall": 0.8931, + "total": 2970, + "trueNegative": 980, + "truePositive": 1688, + "wilsonPrecisionLower95": 0.9324 + }, + "scheduleNegotiation": { + "f1": 0.7755, + "falseNegative": 33, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 57, + "recall": 0.6333, + "total": 2970, + "trueNegative": 2880, + "truePositive": 57, + "wilsonPrecisionLower95": 0.9369 + }, + "task": { + "f1": 0.5484, + "falseNegative": 448, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 272, + "recall": 0.3778, + "total": 2970, + "trueNegative": 2250, + "truePositive": 272, + "wilsonPrecisionLower95": 0.9861 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9727, + "falseNegative": 2, + "falsePositive": 8, + "precision": 0.957, + "predictedPositive": 186, + "recall": 0.9889, + "total": 1485, + "trueNegative": 1297, + "truePositive": 178, + "wilsonPrecisionLower95": 0.9174 + }, + "threshold": 0.98 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.95, + "falseNegative": 9, + "falsePositive": 9, + "precision": 0.95, + "predictedPositive": 180, + "recall": 0.95, + "total": 1485, + "trueNegative": 1296, + "truePositive": 171, + "wilsonPrecisionLower95": 0.9077 + }, + "threshold": 0.905 + } + }, + "metrics": { + "f1": 0.9529, + "falseNegative": 16, + "falsePositive": 18, + "precision": 0.9503, + "predictedPositive": 362, + "recall": 0.9556, + "total": 2970, + "trueNegative": 2592, + "truePositive": 344, + "wilsonPrecisionLower95": 0.9228 + }, + "threshold": 0.965 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.828, + "falseNegative": 25, + "falsePositive": 2, + "precision": 0.9701, + "predictedPositive": 67, + "recall": 0.7222, + "total": 1485, + "trueNegative": 1393, + "truePositive": 65, + "wilsonPrecisionLower95": 0.8975 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6763, + "falseNegative": 43, + "falsePositive": 2, + "precision": 0.9592, + "predictedPositive": 49, + "recall": 0.5222, + "total": 1485, + "trueNegative": 1393, + "truePositive": 47, + "wilsonPrecisionLower95": 0.8629 + }, + "threshold": 0.93 + } + }, + "metrics": { + "f1": 0.7855, + "falseNegative": 61, + "falsePositive": 4, + "precision": 0.9675, + "predictedPositive": 123, + "recall": 0.6611, + "total": 2970, + "trueNegative": 2786, + "truePositive": 119, + "wilsonPrecisionLower95": 0.9194 + }, + "threshold": 0.82 + }, + "followUpReminder": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7452, + "falseNegative": 122, + "falsePositive": 10, + "precision": 0.9507, + "predictedPositive": 203, + "recall": 0.6127, + "total": 1485, + "trueNegative": 1160, + "truePositive": 193, + "wilsonPrecisionLower95": 0.9117 + }, + "threshold": 0.915 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9441, + "falseNegative": 59, + "falsePositive": 46, + "precision": 0.9506, + "predictedPositive": 932, + "recall": 0.9376, + "total": 1485, + "trueNegative": 494, + "truePositive": 886, + "wilsonPrecisionLower95": 0.9348 + }, + "threshold": 0.935 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9067, + "falseNegative": 129, + "falsePositive": 39, + "precision": 0.9544, + "predictedPositive": 855, + "recall": 0.8635, + "total": 1485, + "trueNegative": 501, + "truePositive": 816, + "wilsonPrecisionLower95": 0.9383 + }, + "threshold": 0.905 + } + }, + "metrics": { + "f1": 0.9267, + "falseNegative": 182, + "falsePositive": 88, + "precision": 0.951, + "predictedPositive": 1796, + "recall": 0.9037, + "total": 2970, + "trueNegative": 992, + "truePositive": 1708, + "wilsonPrecisionLower95": 0.94 + }, + "threshold": 0.905 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9888, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 44, + "recall": 0.9778, + "total": 1485, + "trueNegative": 1440, + "truePositive": 44, + "wilsonPrecisionLower95": 0.9197 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9655, + "falseNegative": 3, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 42, + "recall": 0.9333, + "total": 1485, + "trueNegative": 1440, + "truePositive": 42, + "wilsonPrecisionLower95": 0.9162 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.9773, + "falseNegative": 4, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 86, + "recall": 0.9556, + "total": 2970, + "trueNegative": 2880, + "truePositive": 86, + "wilsonPrecisionLower95": 0.9572 + }, + "threshold": 0.5 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.7876, + "falseNegative": 119, + "falsePositive": 11, + "precision": 0.9563, + "predictedPositive": 252, + "recall": 0.6694, + "total": 1485, + "trueNegative": 1114, + "truePositive": 241, + "wilsonPrecisionLower95": 0.9235 + }, + "threshold": 0.995 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8416, + "falseNegative": 89, + "falsePositive": 13, + "precision": 0.9542, + "predictedPositive": 284, + "recall": 0.7528, + "total": 1485, + "trueNegative": 1112, + "truePositive": 271, + "wilsonPrecisionLower95": 0.9233 + }, + "threshold": 0.895 + } + }, + "metrics": { + "f1": 0.6962, + "falseNegative": 327, + "falsePositive": 16, + "precision": 0.9609, + "predictedPositive": 409, + "recall": 0.5458, + "total": 2970, + "trueNegative": 2234, + "truePositive": 393, + "wilsonPrecisionLower95": 0.9374 + }, + "threshold": 0.995 + } + }, + "trainingRecords": 15346, + "validation": { + "intentsMeeting95Precision": 9, + "macroF1": 0.4673, + "macroPrecision": 0.991, + "macroRecall": 0.3426, + "minimumPrecision": 0.9524, + "perIntent": { + "blessing": { + "f1": 0.2178, + "falseNegative": 79, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 11, + "recall": 0.1222, + "total": 2970, + "trueNegative": 2880, + "truePositive": 11, + "wilsonPrecisionLower95": 0.7412 + }, + "complaint": { + "f1": 0.5531, + "falseNegative": 222, + "falsePositive": 1, + "precision": 0.9928, + "predictedPositive": 139, + "recall": 0.3833, + "total": 2970, + "trueNegative": 2609, + "truePositive": 138, + "wilsonPrecisionLower95": 0.9604 + }, + "confirmationDecision": { + "f1": 0.5657, + "falseNegative": 109, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 71, + "recall": 0.3944, + "total": 2970, + "trueNegative": 2790, + "truePositive": 71, + "wilsonPrecisionLower95": 0.9487 + }, + "followUpReminder": { + "f1": 0.1538, + "falseNegative": 330, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 30, + "recall": 0.0833, + "total": 2970, + "trueNegative": 2610, + "truePositive": 30, + "wilsonPrecisionLower95": 0.8865 + }, + "invitation": { + "f1": 0.2, + "falseNegative": 160, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 20, + "recall": 0.1111, + "total": 2970, + "trueNegative": 2790, + "truePositive": 20, + "wilsonPrecisionLower95": 0.8389 + }, + "question": { + "f1": 0.4934, + "falseNegative": 423, + "falsePositive": 2, + "precision": 0.9904, + "predictedPositive": 209, + "recall": 0.3286, + "total": 2970, + "trueNegative": 2338, + "truePositive": 207, + "wilsonPrecisionLower95": 0.9658 + }, + "replyableMessage": { + "f1": 0.9258, + "falseNegative": 188, + "falsePositive": 85, + "precision": 0.9524, + "predictedPositive": 1787, + "recall": 0.9005, + "total": 2970, + "trueNegative": 995, + "truePositive": 1702, + "wilsonPrecisionLower95": 0.9416 + }, + "scheduleNegotiation": { + "f1": 0.5124, + "falseNegative": 59, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 31, + "recall": 0.3444, + "total": 2970, + "trueNegative": 2880, + "truePositive": 31, + "wilsonPrecisionLower95": 0.8897 + }, + "task": { + "f1": 0.584, + "falseNegative": 421, + "falsePositive": 5, + "precision": 0.9836, + "predictedPositive": 304, + "recall": 0.4153, + "total": 2970, + "trueNegative": 2245, + "truePositive": 299, + "wilsonPrecisionLower95": 0.9621 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 2.5e-06, + "augmentation": "prefix", + "char_max": 7, + "char_min": 3, + "external_weight": 0.8300000000000001, + "hard_example_weight": 2.2, + "l1_ratio": 0.2, + "max_char_features": 80000, + "max_word_features": 40000, + "min_df": 1, + "pseudo_weight": 0.36, + "round": 14, + "word_max": 3 + }, + "featureCount": 120000, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 0 + }, + "complaint": { + "negative": 250, + "positive": 8 + }, + "confirmationDecision": { + "negative": 250, + "positive": 26 + }, + "followUpReminder": { + "negative": 250, + "positive": 0 + }, + "invitation": { + "negative": 250, + "positive": 0 + }, + "question": { + "negative": 250, + "positive": 0 + }, + "replyableMessage": { + "negative": 34, + "positive": 250 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 0 + }, + "task": { + "negative": 250, + "positive": 3 + } + }, + "round": 14, + "selectionObjective": 0.6118, + "syntheticRegression": { + "intentsMeeting95Precision": 6, + "macroF1": 0.4596, + "macroPrecision": 0.9735, + "macroRecall": 0.3465, + "minimumPrecision": 0.8258, + "perIntent": { + "blessing": { + "f1": 0.1443, + "falseNegative": 83, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 7, + "recall": 0.0778, + "total": 2970, + "trueNegative": 2880, + "truePositive": 7, + "wilsonPrecisionLower95": 0.6457 + }, + "complaint": { + "f1": 0.2775, + "falseNegative": 302, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 58, + "recall": 0.1611, + "total": 2970, + "trueNegative": 2610, + "truePositive": 58, + "wilsonPrecisionLower95": 0.9379 + }, + "confirmationDecision": { + "f1": 0.6987, + "falseNegative": 71, + "falsePositive": 23, + "precision": 0.8258, + "predictedPositive": 132, + "recall": 0.6056, + "total": 2970, + "trueNegative": 2767, + "truePositive": 109, + "wilsonPrecisionLower95": 0.7521 + }, + "followUpReminder": { + "f1": 0.3991, + "falseNegative": 270, + "falsePositive": 1, + "precision": 0.989, + "predictedPositive": 91, + "recall": 0.25, + "total": 2970, + "trueNegative": 2609, + "truePositive": 90, + "wilsonPrecisionLower95": 0.9404 + }, + "invitation": { + "f1": 0.3178, + "falseNegative": 146, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 34, + "recall": 0.1889, + "total": 2970, + "trueNegative": 2790, + "truePositive": 34, + "wilsonPrecisionLower95": 0.8985 + }, + "question": { + "f1": 0.7537, + "falseNegative": 249, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 381, + "recall": 0.6048, + "total": 2970, + "trueNegative": 2340, + "truePositive": 381, + "wilsonPrecisionLower95": 0.99 + }, + "replyableMessage": { + "f1": 0.6113, + "falseNegative": 1037, + "falsePositive": 48, + "precision": 0.9467, + "predictedPositive": 901, + "recall": 0.4513, + "total": 2970, + "trueNegative": 1032, + "truePositive": 853, + "wilsonPrecisionLower95": 0.9301 + }, + "scheduleNegotiation": { + "f1": 0.8462, + "falseNegative": 24, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 66, + "recall": 0.7333, + "total": 2970, + "trueNegative": 2880, + "truePositive": 66, + "wilsonPrecisionLower95": 0.945 + }, + "task": { + "f1": 0.0876, + "falseNegative": 687, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 33, + "recall": 0.0458, + "total": 2970, + "trueNegative": 2250, + "truePositive": 33, + "wilsonPrecisionLower95": 0.8957 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9809, + "falseNegative": 0, + "falsePositive": 7, + "precision": 0.9626, + "predictedPositive": 187, + "recall": 1.0, + "total": 1485, + "trueNegative": 1298, + "truePositive": 180, + "wilsonPrecisionLower95": 0.9248 + }, + "threshold": 0.99 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8625, + "falseNegative": 21, + "falsePositive": 1, + "precision": 0.9857, + "predictedPositive": 70, + "recall": 0.7667, + "total": 1485, + "trueNegative": 1394, + "truePositive": 69, + "wilsonPrecisionLower95": 0.9234 + }, + "threshold": 0.905 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.828, + "falseNegative": 25, + "falsePositive": 2, + "precision": 0.9701, + "predictedPositive": 67, + "recall": 0.7222, + "total": 1485, + "trueNegative": 1393, + "truePositive": 65, + "wilsonPrecisionLower95": 0.8975 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.8328, + "falseNegative": 48, + "falsePositive": 5, + "precision": 0.9635, + "predictedPositive": 137, + "recall": 0.7333, + "total": 2970, + "trueNegative": 2785, + "truePositive": 132, + "wilsonPrecisionLower95": 0.9174 + }, + "threshold": 0.685 + }, + "followUpReminder": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8788, + "falseNegative": 35, + "falsePositive": 5, + "precision": 0.9667, + "predictedPositive": 150, + "recall": 0.8056, + "total": 1485, + "trueNegative": 1300, + "truePositive": 145, + "wilsonPrecisionLower95": 0.9243 + }, + "threshold": 0.845 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6113, + "falseNegative": 99, + "falsePositive": 4, + "precision": 0.9529, + "predictedPositive": 85, + "recall": 0.45, + "total": 1485, + "trueNegative": 1301, + "truePositive": 81, + "wilsonPrecisionLower95": 0.8852 + }, + "threshold": 0.97 + } + }, + "metrics": { + "f1": 0.7682, + "falseNegative": 128, + "falsePositive": 12, + "precision": 0.9508, + "predictedPositive": 244, + "recall": 0.6444, + "total": 2970, + "trueNegative": 2598, + "truePositive": 232, + "wilsonPrecisionLower95": 0.916 + }, + "threshold": 0.91 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.847, + "falseNegative": 77, + "falsePositive": 9, + "precision": 0.9636, + "predictedPositive": 247, + "recall": 0.7556, + "total": 1485, + "trueNegative": 1161, + "truePositive": 238, + "wilsonPrecisionLower95": 0.9322 + }, + "threshold": 0.995 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7168, + "falseNegative": 134, + "falsePositive": 9, + "precision": 0.9526, + "predictedPositive": 190, + "recall": 0.5746, + "total": 1485, + "trueNegative": 1161, + "truePositive": 181, + "wilsonPrecisionLower95": 0.9124 + }, + "threshold": 0.865 + } + }, + "metrics": { + "f1": 0.7524, + "falseNegative": 238, + "falsePositive": 20, + "precision": 0.9515, + "predictedPositive": 412, + "recall": 0.6222, + "total": 2970, + "trueNegative": 2320, + "truePositive": 392, + "wilsonPrecisionLower95": 0.9262 + }, + "threshold": 0.975 + }, + "replyableMessage": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9349, + "falseNegative": 76, + "falsePositive": 45, + "precision": 0.9508, + "predictedPositive": 914, + "recall": 0.9196, + "total": 1485, + "trueNegative": 495, + "truePositive": 869, + "wilsonPrecisionLower95": 0.9348 + }, + "threshold": 0.955 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 45, + "recall": 1.0, + "total": 1485, + "trueNegative": 1440, + "truePositive": 45, + "wilsonPrecisionLower95": 0.9213 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.967, + "falseNegative": 1, + "falsePositive": 2, + "precision": 0.9565, + "predictedPositive": 46, + "recall": 0.9778, + "total": 1485, + "trueNegative": 1438, + "truePositive": 44, + "wilsonPrecisionLower95": 0.8547 + }, + "threshold": 0.995 + } + }, + "metrics": { + "f1": 0.9727, + "falseNegative": 1, + "falsePositive": 4, + "precision": 0.957, + "predictedPositive": 93, + "recall": 0.9889, + "total": 2970, + "trueNegative": 2876, + "truePositive": 89, + "wilsonPrecisionLower95": 0.8946 + }, + "threshold": 0.64 + }, + "task": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "trainingRecords": 15345, + "validation": { + "intentsMeeting95Precision": 9, + "macroF1": 0.4308, + "macroPrecision": 0.9856, + "macroRecall": 0.2986, + "minimumPrecision": 0.9516, + "perIntent": { + "blessing": { + "f1": 0.3019, + "falseNegative": 74, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 16, + "recall": 0.1778, + "total": 2970, + "trueNegative": 2880, + "truePositive": 16, + "wilsonPrecisionLower95": 0.8064 + }, + "complaint": { + "f1": 0.4, + "falseNegative": 270, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 90, + "recall": 0.25, + "total": 2970, + "trueNegative": 2610, + "truePositive": 90, + "wilsonPrecisionLower95": 0.9591 + }, + "confirmationDecision": { + "f1": 0.5859, + "falseNegative": 105, + "falsePositive": 1, + "precision": 0.9868, + "predictedPositive": 76, + "recall": 0.4167, + "total": 2970, + "trueNegative": 2789, + "truePositive": 75, + "wilsonPrecisionLower95": 0.9292 + }, + "followUpReminder": { + "f1": 0.3012, + "falseNegative": 296, + "falsePositive": 1, + "precision": 0.9846, + "predictedPositive": 65, + "recall": 0.1778, + "total": 2970, + "trueNegative": 2609, + "truePositive": 64, + "wilsonPrecisionLower95": 0.9179 + }, + "invitation": { + "f1": 0.1726, + "falseNegative": 163, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 17, + "recall": 0.0944, + "total": 2970, + "trueNegative": 2790, + "truePositive": 17, + "wilsonPrecisionLower95": 0.8157 + }, + "question": { + "f1": 0.7782, + "falseNegative": 223, + "falsePositive": 9, + "precision": 0.9784, + "predictedPositive": 416, + "recall": 0.646, + "total": 2970, + "trueNegative": 2331, + "truePositive": 407, + "wilsonPrecisionLower95": 0.9594 + }, + "replyableMessage": { + "f1": 0.6272, + "falseNegative": 1006, + "falsePositive": 45, + "precision": 0.9516, + "predictedPositive": 929, + "recall": 0.4677, + "total": 2970, + "trueNegative": 1035, + "truePositive": 884, + "wilsonPrecisionLower95": 0.9358 + }, + "scheduleNegotiation": { + "f1": 0.5082, + "falseNegative": 59, + "falsePositive": 1, + "precision": 0.9688, + "predictedPositive": 32, + "recall": 0.3444, + "total": 2970, + "trueNegative": 2879, + "truePositive": 31, + "wilsonPrecisionLower95": 0.8426 + }, + "task": { + "f1": 0.2022, + "falseNegative": 639, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 81, + "recall": 0.1125, + "total": 2970, + "trueNegative": 2250, + "truePositive": 81, + "wilsonPrecisionLower95": 0.9547 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 3.5e-06, + "augmentation": "prefix", + "char_max": 6, + "char_min": 2, + "external_weight": 0.8500000000000001, + "hard_example_weight": 2.2, + "l1_ratio": 0.2, + "max_char_features": 85000, + "max_word_features": 42000, + "min_df": 1, + "pseudo_weight": 0.375, + "round": 15, + "word_max": 3 + }, + "featureCount": 127000, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 1 + }, + "complaint": { + "negative": 250, + "positive": 8 + }, + "confirmationDecision": { + "negative": 250, + "positive": 30 + }, + "followUpReminder": { + "negative": 250, + "positive": 0 + }, + "invitation": { + "negative": 250, + "positive": 0 + }, + "question": { + "negative": 250, + "positive": 0 + }, + "replyableMessage": { + "negative": 39, + "positive": 250 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 0 + }, + "task": { + "negative": 250, + "positive": 0 + } + }, + "round": 15, + "selectionObjective": 0.6461, + "syntheticRegression": { + "intentsMeeting95Precision": 8, + "macroF1": 0.4969, + "macroPrecision": 0.9744, + "macroRecall": 0.3753, + "minimumPrecision": 0.8151, + "perIntent": { + "blessing": { + "f1": 0.2, + "falseNegative": 80, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 10, + "recall": 0.1111, + "total": 2970, + "trueNegative": 2880, + "truePositive": 10, + "wilsonPrecisionLower95": 0.7225 + }, + "complaint": { + "f1": 0.4211, + "falseNegative": 264, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 96, + "recall": 0.2667, + "total": 2970, + "trueNegative": 2610, + "truePositive": 96, + "wilsonPrecisionLower95": 0.9615 + }, + "confirmationDecision": { + "f1": 0.6488, + "falseNegative": 83, + "falsePositive": 22, + "precision": 0.8151, + "predictedPositive": 119, + "recall": 0.5389, + "total": 2970, + "trueNegative": 2768, + "truePositive": 97, + "wilsonPrecisionLower95": 0.7359 + }, + "followUpReminder": { + "f1": 0.2651, + "falseNegative": 305, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 55, + "recall": 0.1528, + "total": 2970, + "trueNegative": 2610, + "truePositive": 55, + "wilsonPrecisionLower95": 0.9347 + }, + "invitation": { + "f1": 0.2, + "falseNegative": 160, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 20, + "recall": 0.1111, + "total": 2970, + "trueNegative": 2790, + "truePositive": 20, + "wilsonPrecisionLower95": 0.8389 + }, + "question": { + "f1": 0.5366, + "falseNegative": 399, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 231, + "recall": 0.3667, + "total": 2970, + "trueNegative": 2340, + "truePositive": 231, + "wilsonPrecisionLower95": 0.9836 + }, + "replyableMessage": { + "f1": 0.862, + "falseNegative": 410, + "falsePositive": 64, + "precision": 0.9585, + "predictedPositive": 1544, + "recall": 0.7831, + "total": 2970, + "trueNegative": 1016, + "truePositive": 1480, + "wilsonPrecisionLower95": 0.9474 + }, + "scheduleNegotiation": { + "f1": 0.8235, + "falseNegative": 27, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 63, + "recall": 0.7, + "total": 2970, + "trueNegative": 2880, + "truePositive": 63, + "wilsonPrecisionLower95": 0.9425 + }, + "task": { + "f1": 0.5149, + "falseNegative": 470, + "falsePositive": 1, + "precision": 0.996, + "predictedPositive": 251, + "recall": 0.3472, + "total": 2970, + "trueNegative": 2249, + "truePositive": 250, + "wilsonPrecisionLower95": 0.9778 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9836, + "falseNegative": 0, + "falsePositive": 6, + "precision": 0.9677, + "predictedPositive": 186, + "recall": 1.0, + "total": 1485, + "trueNegative": 1299, + "truePositive": 180, + "wilsonPrecisionLower95": 0.9314 + }, + "threshold": 0.945 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.928, + "falseNegative": 19, + "falsePositive": 6, + "precision": 0.9641, + "predictedPositive": 167, + "recall": 0.8944, + "total": 1485, + "trueNegative": 1299, + "truePositive": 161, + "wilsonPrecisionLower95": 0.9238 + }, + "threshold": 0.77 + } + }, + "metrics": { + "f1": 0.9365, + "falseNegative": 28, + "falsePositive": 17, + "precision": 0.9513, + "predictedPositive": 349, + "recall": 0.9222, + "total": 2970, + "trueNegative": 2593, + "truePositive": 332, + "wilsonPrecisionLower95": 0.9234 + }, + "threshold": 0.88 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.954, + "falseNegative": 7, + "falsePositive": 1, + "precision": 0.9881, + "predictedPositive": 84, + "recall": 0.9222, + "total": 1485, + "trueNegative": 1394, + "truePositive": 83, + "wilsonPrecisionLower95": 0.9356 + }, + "threshold": 0.85 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6906, + "falseNegative": 42, + "falsePositive": 1, + "precision": 0.9796, + "predictedPositive": 49, + "recall": 0.5333, + "total": 1485, + "trueNegative": 1394, + "truePositive": 48, + "wilsonPrecisionLower95": 0.8931 + }, + "threshold": 0.985 + } + }, + "metrics": { + "f1": 0.8598, + "falseNegative": 39, + "falsePositive": 7, + "precision": 0.9527, + "predictedPositive": 148, + "recall": 0.7833, + "total": 2970, + "trueNegative": 2783, + "truePositive": 141, + "wilsonPrecisionLower95": 0.9056 + }, + "threshold": 0.935 + }, + "followUpReminder": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8328, + "falseNegative": 48, + "falsePositive": 5, + "precision": 0.9635, + "predictedPositive": 137, + "recall": 0.7333, + "total": 1485, + "trueNegative": 1300, + "truePositive": 132, + "wilsonPrecisionLower95": 0.9174 + }, + "threshold": 0.99 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9133, + "falseNegative": 11, + "falsePositive": 4, + "precision": 0.9518, + "predictedPositive": 83, + "recall": 0.8778, + "total": 1485, + "trueNegative": 1391, + "truePositive": 79, + "wilsonPrecisionLower95": 0.8825 + }, + "threshold": 0.97 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7548, + "falseNegative": 118, + "falsePositive": 10, + "precision": 0.9517, + "predictedPositive": 207, + "recall": 0.6254, + "total": 1485, + "trueNegative": 1160, + "truePositive": 197, + "wilsonPrecisionLower95": 0.9134 + }, + "threshold": 0.8 + } + }, + "metrics": { + "f1": 0.7678, + "falseNegative": 225, + "falsePositive": 20, + "precision": 0.9529, + "predictedPositive": 425, + "recall": 0.6429, + "total": 2970, + "trueNegative": 2320, + "truePositive": 405, + "wilsonPrecisionLower95": 0.9284 + }, + "threshold": 0.985 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.907, + "falseNegative": 126, + "falsePositive": 42, + "precision": 0.9512, + "predictedPositive": 861, + "recall": 0.8667, + "total": 1485, + "trueNegative": 498, + "truePositive": 819, + "wilsonPrecisionLower95": 0.9347 + }, + "threshold": 0.98 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8252, + "falseNegative": 258, + "falsePositive": 33, + "precision": 0.9542, + "predictedPositive": 720, + "recall": 0.727, + "total": 1485, + "trueNegative": 507, + "truePositive": 687, + "wilsonPrecisionLower95": 0.9363 + }, + "threshold": 0.96 + } + }, + "metrics": { + "f1": 0.8678, + "falseNegative": 384, + "falsePositive": 75, + "precision": 0.9526, + "predictedPositive": 1581, + "recall": 0.7968, + "total": 2970, + "trueNegative": 1005, + "truePositive": 1506, + "wilsonPrecisionLower95": 0.9409 + }, + "threshold": 0.965 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9157, + "falseNegative": 7, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 38, + "recall": 0.8444, + "total": 1485, + "trueNegative": 1440, + "truePositive": 38, + "wilsonPrecisionLower95": 0.9082 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9556, + "falseNegative": 2, + "falsePositive": 2, + "precision": 0.9556, + "predictedPositive": 45, + "recall": 0.9556, + "total": 1485, + "trueNegative": 1438, + "truePositive": 43, + "wilsonPrecisionLower95": 0.8517 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.9364, + "falseNegative": 9, + "falsePositive": 2, + "precision": 0.9759, + "predictedPositive": 83, + "recall": 0.9, + "total": 2970, + "trueNegative": 2878, + "truePositive": 81, + "wilsonPrecisionLower95": 0.9163 + }, + "threshold": 0.5 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8593, + "falseNegative": 79, + "falsePositive": 13, + "precision": 0.9558, + "predictedPositive": 294, + "recall": 0.7806, + "total": 1485, + "trueNegative": 1112, + "truePositive": 281, + "wilsonPrecisionLower95": 0.9258 + }, + "threshold": 0.98 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8166, + "falseNegative": 104, + "falsePositive": 11, + "precision": 0.9588, + "predictedPositive": 267, + "recall": 0.7111, + "total": 1485, + "trueNegative": 1114, + "truePositive": 256, + "wilsonPrecisionLower95": 0.9277 + }, + "threshold": 0.96 + } + }, + "metrics": { + "f1": 0.8434, + "falseNegative": 176, + "falsePositive": 26, + "precision": 0.9544, + "predictedPositive": 570, + "recall": 0.7556, + "total": 2970, + "trueNegative": 2224, + "truePositive": 544, + "wilsonPrecisionLower95": 0.934 + }, + "threshold": 0.965 + } + }, + "trainingRecords": 15345, + "validation": { + "intentsMeeting95Precision": 7, + "macroF1": 0.4919, + "macroPrecision": 0.9716, + "macroRecall": 0.3554, + "minimumPrecision": 0.9024, + "perIntent": { + "blessing": { + "f1": 0.3333, + "falseNegative": 72, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 18, + "recall": 0.2, + "total": 2970, + "trueNegative": 2880, + "truePositive": 18, + "wilsonPrecisionLower95": 0.8241 + }, + "complaint": { + "f1": 0.5455, + "falseNegative": 225, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 135, + "recall": 0.375, + "total": 2970, + "trueNegative": 2610, + "truePositive": 135, + "wilsonPrecisionLower95": 0.9723 + }, + "confirmationDecision": { + "f1": 0.6493, + "falseNegative": 93, + "falsePositive": 1, + "precision": 0.9886, + "predictedPositive": 88, + "recall": 0.4833, + "total": 2970, + "trueNegative": 2789, + "truePositive": 87, + "wilsonPrecisionLower95": 0.9384 + }, + "followUpReminder": { + "f1": 0.1845, + "falseNegative": 323, + "falsePositive": 4, + "precision": 0.9024, + "predictedPositive": 41, + "recall": 0.1028, + "total": 2970, + "trueNegative": 2606, + "truePositive": 37, + "wilsonPrecisionLower95": 0.7745 + }, + "invitation": { + "f1": 0.2775, + "falseNegative": 151, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 29, + "recall": 0.1611, + "total": 2970, + "trueNegative": 2790, + "truePositive": 29, + "wilsonPrecisionLower95": 0.883 + }, + "question": { + "f1": 0.4893, + "falseNegative": 425, + "falsePositive": 3, + "precision": 0.9856, + "predictedPositive": 208, + "recall": 0.3254, + "total": 2970, + "trueNegative": 2337, + "truePositive": 205, + "wilsonPrecisionLower95": 0.9585 + }, + "replyableMessage": { + "f1": 0.8678, + "falseNegative": 384, + "falsePositive": 75, + "precision": 0.9526, + "predictedPositive": 1581, + "recall": 0.7968, + "total": 2970, + "trueNegative": 1005, + "truePositive": 1506, + "wilsonPrecisionLower95": 0.9409 + }, + "scheduleNegotiation": { + "f1": 0.4918, + "falseNegative": 60, + "falsePositive": 2, + "precision": 0.9375, + "predictedPositive": 32, + "recall": 0.3333, + "total": 2970, + "trueNegative": 2878, + "truePositive": 30, + "wilsonPrecisionLower95": 0.7985 + }, + "task": { + "f1": 0.5883, + "falseNegative": 417, + "falsePositive": 7, + "precision": 0.9774, + "predictedPositive": 310, + "recall": 0.4208, + "total": 2970, + "trueNegative": 2243, + "truePositive": 303, + "wilsonPrecisionLower95": 0.9541 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 3e-06, + "augmentation": "numeric", + "char_max": 7, + "char_min": 2, + "external_weight": 0.8700000000000001, + "hard_example_weight": 2.3, + "l1_ratio": 0.15, + "max_char_features": 80000, + "max_word_features": 38000, + "min_df": 2, + "pseudo_weight": 0.39, + "round": 16, + "word_max": 3 + }, + "featureCount": 100252, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 3 + }, + "complaint": { + "negative": 250, + "positive": 11 + }, + "confirmationDecision": { + "negative": 250, + "positive": 25 + }, + "followUpReminder": { + "negative": 250, + "positive": 0 + }, + "invitation": { + "negative": 250, + "positive": 0 + }, + "question": { + "negative": 250, + "positive": 0 + }, + "replyableMessage": { + "negative": 76, + "positive": 250 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 0 + }, + "task": { + "negative": 250, + "positive": 0 + } + }, + "round": 16, + "selectionObjective": 0.6666, + "syntheticRegression": { + "intentsMeeting95Precision": 7, + "macroF1": 0.5355, + "macroPrecision": 0.9681, + "macroRecall": 0.4128, + "minimumPrecision": 0.7609, + "perIntent": { + "blessing": { + "f1": 0.125, + "falseNegative": 84, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 6, + "recall": 0.0667, + "total": 2970, + "trueNegative": 2880, + "truePositive": 6, + "wilsonPrecisionLower95": 0.6097 + }, + "complaint": { + "f1": 0.3673, + "falseNegative": 279, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 81, + "recall": 0.225, + "total": 2970, + "trueNegative": 2610, + "truePositive": 81, + "wilsonPrecisionLower95": 0.9547 + }, + "confirmationDecision": { + "f1": 0.5147, + "falseNegative": 110, + "falsePositive": 22, + "precision": 0.7609, + "predictedPositive": 92, + "recall": 0.3889, + "total": 2970, + "trueNegative": 2768, + "truePositive": 70, + "wilsonPrecisionLower95": 0.6644 + }, + "followUpReminder": { + "f1": 0.3486, + "falseNegative": 284, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 76, + "recall": 0.2111, + "total": 2970, + "trueNegative": 2610, + "truePositive": 76, + "wilsonPrecisionLower95": 0.9519 + }, + "invitation": { + "f1": 0.4211, + "falseNegative": 132, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 48, + "recall": 0.2667, + "total": 2970, + "trueNegative": 2790, + "truePositive": 48, + "wilsonPrecisionLower95": 0.9259 + }, + "question": { + "f1": 0.8117, + "falseNegative": 199, + "falsePositive": 1, + "precision": 0.9977, + "predictedPositive": 432, + "recall": 0.6841, + "total": 2970, + "trueNegative": 2339, + "truePositive": 431, + "wilsonPrecisionLower95": 0.987 + }, + "replyableMessage": { + "f1": 0.8725, + "falseNegative": 371, + "falsePositive": 73, + "precision": 0.9541, + "predictedPositive": 1592, + "recall": 0.8037, + "total": 2970, + "trueNegative": 1007, + "truePositive": 1519, + "wilsonPrecisionLower95": 0.9427 + }, + "scheduleNegotiation": { + "f1": 0.8312, + "falseNegative": 26, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 64, + "recall": 0.7111, + "total": 2970, + "trueNegative": 2880, + "truePositive": 64, + "wilsonPrecisionLower95": 0.9434 + }, + "task": { + "f1": 0.5276, + "falseNegative": 462, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 258, + "recall": 0.3583, + "total": 2970, + "trueNegative": 2250, + "truePositive": 258, + "wilsonPrecisionLower95": 0.9853 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9371, + "falseNegative": 16, + "falsePositive": 6, + "precision": 0.9647, + "predictedPositive": 170, + "recall": 0.9111, + "total": 1485, + "trueNegative": 1299, + "truePositive": 164, + "wilsonPrecisionLower95": 0.9251 + }, + "threshold": 0.995 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8836, + "falseNegative": 32, + "falsePositive": 7, + "precision": 0.9548, + "predictedPositive": 155, + "recall": 0.8222, + "total": 1485, + "trueNegative": 1298, + "truePositive": 148, + "wilsonPrecisionLower95": 0.9097 + }, + "threshold": 0.99 + } + }, + "metrics": { + "f1": 0.8994, + "falseNegative": 56, + "falsePositive": 12, + "precision": 0.962, + "predictedPositive": 316, + "recall": 0.8444, + "total": 2970, + "trueNegative": 2598, + "truePositive": 304, + "wilsonPrecisionLower95": 0.9348 + }, + "threshold": 0.995 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8625, + "falseNegative": 21, + "falsePositive": 1, + "precision": 0.9857, + "predictedPositive": 70, + "recall": 0.7667, + "total": 1485, + "trueNegative": 1394, + "truePositive": 69, + "wilsonPrecisionLower95": 0.9234 + }, + "threshold": 0.915 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.7415, + "falseNegative": 71, + "falsePositive": 5, + "precision": 0.9561, + "predictedPositive": 114, + "recall": 0.6056, + "total": 2970, + "trueNegative": 2785, + "truePositive": 109, + "wilsonPrecisionLower95": 0.9014 + }, + "threshold": 0.96 + }, + "followUpReminder": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8667, + "falseNegative": 37, + "falsePositive": 7, + "precision": 0.9533, + "predictedPositive": 150, + "recall": 0.7944, + "total": 1485, + "trueNegative": 1298, + "truePositive": 143, + "wilsonPrecisionLower95": 0.9068 + }, + "threshold": 0.96 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6031, + "falseNegative": 101, + "falsePositive": 3, + "precision": 0.9634, + "predictedPositive": 82, + "recall": 0.4389, + "total": 1485, + "trueNegative": 1302, + "truePositive": 79, + "wilsonPrecisionLower95": 0.8979 + }, + "threshold": 0.97 + } + }, + "metrics": { + "f1": 0.7492, + "falseNegative": 139, + "falsePositive": 9, + "precision": 0.9609, + "predictedPositive": 230, + "recall": 0.6139, + "total": 2970, + "trueNegative": 2601, + "truePositive": 221, + "wilsonPrecisionLower95": 0.9273 + }, + "threshold": 0.97 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.3604, + "falseNegative": 70, + "falsePositive": 1, + "precision": 0.9524, + "predictedPositive": 21, + "recall": 0.2222, + "total": 1485, + "trueNegative": 1394, + "truePositive": 20, + "wilsonPrecisionLower95": 0.7733 + }, + "threshold": 0.965 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8273, + "falseNegative": 85, + "falsePositive": 11, + "precision": 0.9544, + "predictedPositive": 241, + "recall": 0.7302, + "total": 1485, + "trueNegative": 1159, + "truePositive": 230, + "wilsonPrecisionLower95": 0.9201 + }, + "threshold": 0.985 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.797, + "falseNegative": 99, + "falsePositive": 11, + "precision": 0.9515, + "predictedPositive": 227, + "recall": 0.6857, + "total": 1485, + "trueNegative": 1159, + "truePositive": 216, + "wilsonPrecisionLower95": 0.9153 + }, + "threshold": 0.835 + } + }, + "metrics": { + "f1": 0.8051, + "falseNegative": 190, + "falsePositive": 23, + "precision": 0.9503, + "predictedPositive": 463, + "recall": 0.6984, + "total": 2970, + "trueNegative": 2317, + "truePositive": 440, + "wilsonPrecisionLower95": 0.9266 + }, + "threshold": 0.94 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9343, + "falseNegative": 77, + "falsePositive": 45, + "precision": 0.9507, + "predictedPositive": 913, + "recall": 0.9185, + "total": 1485, + "trueNegative": 495, + "truePositive": 868, + "wilsonPrecisionLower95": 0.9347 + }, + "threshold": 0.91 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8207, + "falseNegative": 263, + "falsePositive": 35, + "precision": 0.9512, + "predictedPositive": 717, + "recall": 0.7217, + "total": 1485, + "trueNegative": 505, + "truePositive": 682, + "wilsonPrecisionLower95": 0.9329 + }, + "threshold": 0.98 + } + }, + "metrics": { + "f1": 0.8572, + "falseNegative": 419, + "falsePositive": 71, + "precision": 0.954, + "predictedPositive": 1542, + "recall": 0.7783, + "total": 2970, + "trueNegative": 1009, + "truePositive": 1471, + "wilsonPrecisionLower95": 0.9423 + }, + "threshold": 0.98 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9888, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 44, + "recall": 0.9778, + "total": 1485, + "trueNegative": 1440, + "truePositive": 44, + "wilsonPrecisionLower95": 0.9197 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.989, + "falseNegative": 0, + "falsePositive": 1, + "precision": 0.9783, + "predictedPositive": 46, + "recall": 1.0, + "total": 1485, + "trueNegative": 1439, + "truePositive": 45, + "wilsonPrecisionLower95": 0.8866 + }, + "threshold": 0.955 + } + }, + "metrics": { + "f1": 0.9778, + "falseNegative": 2, + "falsePositive": 2, + "precision": 0.9778, + "predictedPositive": 90, + "recall": 0.9778, + "total": 2970, + "trueNegative": 2878, + "truePositive": 88, + "wilsonPrecisionLower95": 0.9226 + }, + "threshold": 0.935 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8836, + "falseNegative": 64, + "falsePositive": 14, + "precision": 0.9548, + "predictedPositive": 310, + "recall": 0.8222, + "total": 1485, + "trueNegative": 1111, + "truePositive": 296, + "wilsonPrecisionLower95": 0.9256 + }, + "threshold": 0.975 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7896, + "falseNegative": 118, + "falsePositive": 11, + "precision": 0.9565, + "predictedPositive": 253, + "recall": 0.6722, + "total": 1485, + "trueNegative": 1114, + "truePositive": 242, + "wilsonPrecisionLower95": 0.9238 + }, + "threshold": 0.96 + } + }, + "metrics": { + "f1": 0.8316, + "falseNegative": 189, + "falsePositive": 26, + "precision": 0.9533, + "predictedPositive": 557, + "recall": 0.7375, + "total": 2970, + "trueNegative": 2224, + "truePositive": 531, + "wilsonPrecisionLower95": 0.9325 + }, + "threshold": 0.97 + } + }, + "trainingRecords": 16726, + "validation": { + "intentsMeeting95Precision": 9, + "macroF1": 0.514, + "macroPrecision": 0.9879, + "macroRecall": 0.3812, + "minimumPrecision": 0.9509, + "perIntent": { + "blessing": { + "f1": 0.3636, + "falseNegative": 70, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 20, + "recall": 0.2222, + "total": 2970, + "trueNegative": 2880, + "truePositive": 20, + "wilsonPrecisionLower95": 0.8389 + }, + "complaint": { + "f1": 0.5155, + "falseNegative": 235, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 125, + "recall": 0.3472, + "total": 2970, + "trueNegative": 2610, + "truePositive": 125, + "wilsonPrecisionLower95": 0.9702 + }, + "confirmationDecision": { + "f1": 0.4549, + "falseNegative": 127, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 53, + "recall": 0.2944, + "total": 2970, + "trueNegative": 2790, + "truePositive": 53, + "wilsonPrecisionLower95": 0.9324 + }, + "followUpReminder": { + "f1": 0.2347, + "falseNegative": 312, + "falsePositive": 1, + "precision": 0.9796, + "predictedPositive": 49, + "recall": 0.1333, + "total": 2970, + "trueNegative": 2609, + "truePositive": 48, + "wilsonPrecisionLower95": 0.8931 + }, + "invitation": { + "f1": 0.2609, + "falseNegative": 153, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 27, + "recall": 0.15, + "total": 2970, + "trueNegative": 2790, + "truePositive": 27, + "wilsonPrecisionLower95": 0.8754 + }, + "question": { + "f1": 0.8133, + "falseNegative": 190, + "falsePositive": 12, + "precision": 0.9735, + "predictedPositive": 452, + "recall": 0.6984, + "total": 2970, + "trueNegative": 2328, + "truePositive": 440, + "wilsonPrecisionLower95": 0.9542 + }, + "replyableMessage": { + "f1": 0.8807, + "falseNegative": 340, + "falsePositive": 80, + "precision": 0.9509, + "predictedPositive": 1630, + "recall": 0.8201, + "total": 2970, + "trueNegative": 1000, + "truePositive": 1550, + "wilsonPrecisionLower95": 0.9393 + }, + "scheduleNegotiation": { + "f1": 0.5124, + "falseNegative": 59, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 31, + "recall": 0.3444, + "total": 2970, + "trueNegative": 2880, + "truePositive": 31, + "wilsonPrecisionLower95": 0.8897 + }, + "task": { + "f1": 0.5901, + "falseNegative": 417, + "falsePositive": 4, + "precision": 0.987, + "predictedPositive": 307, + "recall": 0.4208, + "total": 2970, + "trueNegative": 2246, + "truePositive": 303, + "wilsonPrecisionLower95": 0.967 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 2e-06, + "augmentation": "numeric", + "char_max": 7, + "char_min": 3, + "external_weight": 0.8900000000000001, + "hard_example_weight": 2.5, + "l1_ratio": 0.2, + "max_char_features": 85000, + "max_word_features": 42000, + "min_df": 1, + "pseudo_weight": 0.405, + "round": 17, + "word_max": 3 + }, + "featureCount": 127000, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 1 + }, + "complaint": { + "negative": 250, + "positive": 4 + }, + "confirmationDecision": { + "negative": 250, + "positive": 23 + }, + "followUpReminder": { + "negative": 250, + "positive": 3 + }, + "invitation": { + "negative": 250, + "positive": 0 + }, + "question": { + "negative": 250, + "positive": 0 + }, + "replyableMessage": { + "negative": 70, + "positive": 250 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 0 + }, + "task": { + "negative": 250, + "positive": 0 + } + }, + "round": 17, + "selectionObjective": 0.5722, + "syntheticRegression": { + "intentsMeeting95Precision": 7, + "macroF1": 0.3646, + "macroPrecision": 0.9708, + "macroRecall": 0.2459, + "minimumPrecision": 0.7816, + "perIntent": { + "blessing": { + "f1": 0.1633, + "falseNegative": 82, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 8, + "recall": 0.0889, + "total": 2970, + "trueNegative": 2880, + "truePositive": 8, + "wilsonPrecisionLower95": 0.6756 + }, + "complaint": { + "f1": 0.4035, + "falseNegative": 269, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 91, + "recall": 0.2528, + "total": 2970, + "trueNegative": 2610, + "truePositive": 91, + "wilsonPrecisionLower95": 0.9595 + }, + "confirmationDecision": { + "f1": 0.5094, + "falseNegative": 112, + "falsePositive": 19, + "precision": 0.7816, + "predictedPositive": 87, + "recall": 0.3778, + "total": 2970, + "trueNegative": 2771, + "truePositive": 68, + "wilsonPrecisionLower95": 0.6839 + }, + "followUpReminder": { + "f1": 0.1679, + "falseNegative": 327, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 33, + "recall": 0.0917, + "total": 2970, + "trueNegative": 2610, + "truePositive": 33, + "wilsonPrecisionLower95": 0.8957 + }, + "invitation": { + "f1": 0.371, + "falseNegative": 139, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 41, + "recall": 0.2278, + "total": 2970, + "trueNegative": 2790, + "truePositive": 41, + "wilsonPrecisionLower95": 0.9143 + }, + "question": { + "f1": 0.4783, + "falseNegative": 432, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 198, + "recall": 0.3143, + "total": 2970, + "trueNegative": 2340, + "truePositive": 198, + "wilsonPrecisionLower95": 0.981 + }, + "replyableMessage": { + "f1": 0.2549, + "falseNegative": 1612, + "falsePositive": 13, + "precision": 0.9553, + "predictedPositive": 291, + "recall": 0.1471, + "total": 2970, + "trueNegative": 1067, + "truePositive": 278, + "wilsonPrecisionLower95": 0.9251 + }, + "scheduleNegotiation": { + "f1": 0.7671, + "falseNegative": 34, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 56, + "recall": 0.6222, + "total": 2970, + "trueNegative": 2880, + "truePositive": 56, + "wilsonPrecisionLower95": 0.9358 + }, + "task": { + "f1": 0.1656, + "falseNegative": 655, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 65, + "recall": 0.0903, + "total": 2970, + "trueNegative": 2250, + "truePositive": 65, + "wilsonPrecisionLower95": 0.9442 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.989, + "falseNegative": 0, + "falsePositive": 4, + "precision": 0.9783, + "predictedPositive": 184, + "recall": 1.0, + "total": 1485, + "trueNegative": 1301, + "truePositive": 180, + "wilsonPrecisionLower95": 0.9454 + }, + "threshold": 0.995 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8869, + "falseNegative": 31, + "falsePositive": 7, + "precision": 0.9551, + "predictedPositive": 156, + "recall": 0.8278, + "total": 1485, + "trueNegative": 1298, + "truePositive": 149, + "wilsonPrecisionLower95": 0.9103 + }, + "threshold": 0.995 + } + }, + "metrics": { + "f1": 0.944, + "falseNegative": 23, + "falsePositive": 17, + "precision": 0.952, + "predictedPositive": 354, + "recall": 0.9361, + "total": 2970, + "trueNegative": 2593, + "truePositive": 337, + "wilsonPrecisionLower95": 0.9244 + }, + "threshold": 0.975 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8447, + "falseNegative": 22, + "falsePositive": 3, + "precision": 0.9577, + "predictedPositive": 71, + "recall": 0.7556, + "total": 1485, + "trueNegative": 1392, + "truePositive": 68, + "wilsonPrecisionLower95": 0.883 + }, + "threshold": 0.825 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.7723, + "falseNegative": 63, + "falsePositive": 6, + "precision": 0.9512, + "predictedPositive": 123, + "recall": 0.65, + "total": 2970, + "trueNegative": 2784, + "truePositive": 117, + "wilsonPrecisionLower95": 0.8977 + }, + "threshold": 0.965 + }, + "followUpReminder": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6625, + "falseNegative": 155, + "falsePositive": 8, + "precision": 0.9524, + "predictedPositive": 168, + "recall": 0.5079, + "total": 1485, + "trueNegative": 1162, + "truePositive": 160, + "wilsonPrecisionLower95": 0.9089 + }, + "threshold": 0.97 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "replyableMessage": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9655, + "falseNegative": 3, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 42, + "recall": 0.9333, + "total": 1485, + "trueNegative": 1440, + "truePositive": 42, + "wilsonPrecisionLower95": 0.9162 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9425, + "falseNegative": 4, + "falsePositive": 1, + "precision": 0.9762, + "predictedPositive": 42, + "recall": 0.9111, + "total": 1485, + "trueNegative": 1439, + "truePositive": 41, + "wilsonPrecisionLower95": 0.8768 + }, + "threshold": 0.73 + } + }, + "metrics": { + "f1": 0.9486, + "falseNegative": 7, + "falsePositive": 2, + "precision": 0.9765, + "predictedPositive": 85, + "recall": 0.9222, + "total": 2970, + "trueNegative": 2878, + "truePositive": 83, + "wilsonPrecisionLower95": 0.9182 + }, + "threshold": 0.5 + }, + "task": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "trainingRecords": 16820, + "validation": { + "intentsMeeting95Precision": 9, + "macroF1": 0.3704, + "macroPrecision": 0.9857, + "macroRecall": 0.235, + "minimumPrecision": 0.9651, + "perIntent": { + "blessing": { + "f1": 0.3019, + "falseNegative": 74, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 16, + "recall": 0.1778, + "total": 2970, + "trueNegative": 2880, + "truePositive": 16, + "wilsonPrecisionLower95": 0.8064 + }, + "complaint": { + "f1": 0.5325, + "falseNegative": 229, + "falsePositive": 1, + "precision": 0.9924, + "predictedPositive": 132, + "recall": 0.3639, + "total": 2970, + "trueNegative": 2609, + "truePositive": 131, + "wilsonPrecisionLower95": 0.9583 + }, + "confirmationDecision": { + "f1": 0.4329, + "falseNegative": 130, + "falsePositive": 1, + "precision": 0.9804, + "predictedPositive": 51, + "recall": 0.2778, + "total": 2970, + "trueNegative": 2789, + "truePositive": 50, + "wilsonPrecisionLower95": 0.897 + }, + "followUpReminder": { + "f1": 0.204, + "falseNegative": 319, + "falsePositive": 1, + "precision": 0.9762, + "predictedPositive": 42, + "recall": 0.1139, + "total": 2970, + "trueNegative": 2609, + "truePositive": 41, + "wilsonPrecisionLower95": 0.8768 + }, + "invitation": { + "f1": 0.2609, + "falseNegative": 153, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 27, + "recall": 0.15, + "total": 2970, + "trueNegative": 2790, + "truePositive": 27, + "wilsonPrecisionLower95": 0.8754 + }, + "question": { + "f1": 0.5471, + "falseNegative": 392, + "falsePositive": 2, + "precision": 0.9917, + "predictedPositive": 240, + "recall": 0.3778, + "total": 2970, + "trueNegative": 2338, + "truePositive": 238, + "wilsonPrecisionLower95": 0.9701 + }, + "replyableMessage": { + "f1": 0.2757, + "falseNegative": 1586, + "falsePositive": 11, + "precision": 0.9651, + "predictedPositive": 315, + "recall": 0.1608, + "total": 2970, + "trueNegative": 1069, + "truePositive": 304, + "wilsonPrecisionLower95": 0.9386 + }, + "scheduleNegotiation": { + "f1": 0.4706, + "falseNegative": 62, + "falsePositive": 1, + "precision": 0.9655, + "predictedPositive": 29, + "recall": 0.3111, + "total": 2970, + "trueNegative": 2879, + "truePositive": 28, + "wilsonPrecisionLower95": 0.8282 + }, + "task": { + "f1": 0.3079, + "falseNegative": 589, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 131, + "recall": 0.1819, + "total": 2970, + "trueNegative": 2250, + "truePositive": 131, + "wilsonPrecisionLower95": 0.9715 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 1.5e-06, + "augmentation": "numeric", + "char_max": 8, + "char_min": 2, + "external_weight": 0.91, + "hard_example_weight": 2.5, + "l1_ratio": 0.2, + "max_char_features": 90000, + "max_word_features": 45000, + "min_df": 1, + "pseudo_weight": 0.42000000000000004, + "round": 18, + "word_max": 3 + }, + "featureCount": 135000, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 1 + }, + "complaint": { + "negative": 250, + "positive": 8 + }, + "confirmationDecision": { + "negative": 250, + "positive": 27 + }, + "followUpReminder": { + "negative": 250, + "positive": 0 + }, + "invitation": { + "negative": 250, + "positive": 0 + }, + "question": { + "negative": 250, + "positive": 0 + }, + "replyableMessage": { + "negative": 90, + "positive": 32 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 0 + }, + "task": { + "negative": 250, + "positive": 0 + } + }, + "round": 18, + "selectionObjective": 0.6413, + "syntheticRegression": { + "intentsMeeting95Precision": 7, + "macroF1": 0.4676, + "macroPrecision": 0.9604, + "macroRecall": 0.3553, + "minimumPrecision": 0.7403, + "perIntent": { + "blessing": { + "f1": 0.2178, + "falseNegative": 79, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 11, + "recall": 0.1222, + "total": 2970, + "trueNegative": 2880, + "truePositive": 11, + "wilsonPrecisionLower95": 0.7412 + }, + "complaint": { + "f1": 0.3059, + "falseNegative": 295, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 65, + "recall": 0.1806, + "total": 2970, + "trueNegative": 2610, + "truePositive": 65, + "wilsonPrecisionLower95": 0.9442 + }, + "confirmationDecision": { + "f1": 0.4436, + "falseNegative": 123, + "falsePositive": 20, + "precision": 0.7403, + "predictedPositive": 77, + "recall": 0.3167, + "total": 2970, + "trueNegative": 2770, + "truePositive": 57, + "wilsonPrecisionLower95": 0.6326 + }, + "followUpReminder": { + "f1": 0.2844, + "falseNegative": 300, + "falsePositive": 2, + "precision": 0.9677, + "predictedPositive": 62, + "recall": 0.1667, + "total": 2970, + "trueNegative": 2608, + "truePositive": 60, + "wilsonPrecisionLower95": 0.8898 + }, + "invitation": { + "f1": 0.3178, + "falseNegative": 146, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 34, + "recall": 0.1889, + "total": 2970, + "trueNegative": 2790, + "truePositive": 34, + "wilsonPrecisionLower95": 0.8985 + }, + "question": { + "f1": 0.6957, + "falseNegative": 294, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 336, + "recall": 0.5333, + "total": 2970, + "trueNegative": 2340, + "truePositive": 336, + "wilsonPrecisionLower95": 0.9887 + }, + "replyableMessage": { + "f1": 0.8939, + "falseNegative": 273, + "falsePositive": 111, + "precision": 0.9358, + "predictedPositive": 1728, + "recall": 0.8556, + "total": 2970, + "trueNegative": 969, + "truePositive": 1617, + "wilsonPrecisionLower95": 0.9232 + }, + "scheduleNegotiation": { + "f1": 0.8312, + "falseNegative": 26, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 64, + "recall": 0.7111, + "total": 2970, + "trueNegative": 2880, + "truePositive": 64, + "wilsonPrecisionLower95": 0.9434 + }, + "task": { + "f1": 0.2178, + "falseNegative": 632, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 88, + "recall": 0.1222, + "total": 2970, + "trueNegative": 2250, + "truePositive": 88, + "wilsonPrecisionLower95": 0.9582 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9486, + "falseNegative": 7, + "falsePositive": 2, + "precision": 0.9765, + "predictedPositive": 85, + "recall": 0.9222, + "total": 1485, + "trueNegative": 1393, + "truePositive": 83, + "wilsonPrecisionLower95": 0.9182 + }, + "threshold": 0.9 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.8447, + "falseNegative": 44, + "falsePositive": 6, + "precision": 0.9577, + "predictedPositive": 142, + "recall": 0.7556, + "total": 2970, + "trueNegative": 2784, + "truePositive": 136, + "wilsonPrecisionLower95": 0.9109 + }, + "threshold": 0.955 + }, + "followUpReminder": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8822, + "falseNegative": 34, + "falsePositive": 5, + "precision": 0.9669, + "predictedPositive": 151, + "recall": 0.8111, + "total": 1485, + "trueNegative": 1300, + "truePositive": 146, + "wilsonPrecisionLower95": 0.9248 + }, + "threshold": 0.98 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.7641, + "falseNegative": 130, + "falsePositive": 12, + "precision": 0.9504, + "predictedPositive": 242, + "recall": 0.6389, + "total": 2970, + "trueNegative": 2598, + "truePositive": 230, + "wilsonPrecisionLower95": 0.9153 + }, + "threshold": 0.99 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8103, + "falseNegative": 95, + "falsePositive": 8, + "precision": 0.9649, + "predictedPositive": 228, + "recall": 0.6984, + "total": 1485, + "trueNegative": 1162, + "truePositive": 220, + "wilsonPrecisionLower95": 0.9323 + }, + "threshold": 0.915 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9575, + "falseNegative": 33, + "falsePositive": 48, + "precision": 0.95, + "predictedPositive": 960, + "recall": 0.9651, + "total": 1485, + "trueNegative": 492, + "truePositive": 912, + "wilsonPrecisionLower95": 0.9343 + }, + "threshold": 0.845 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8361, + "falseNegative": 241, + "falsePositive": 35, + "precision": 0.9526, + "predictedPositive": 739, + "recall": 0.745, + "total": 1485, + "trueNegative": 505, + "truePositive": 704, + "wilsonPrecisionLower95": 0.9348 + }, + "threshold": 0.98 + } + }, + "metrics": { + "f1": 0.901, + "falseNegative": 274, + "falsePositive": 81, + "precision": 0.9523, + "predictedPositive": 1697, + "recall": 0.855, + "total": 2970, + "trueNegative": 999, + "truePositive": 1616, + "wilsonPrecisionLower95": 0.9411 + }, + "threshold": 0.975 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9655, + "falseNegative": 3, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 42, + "recall": 0.9333, + "total": 1485, + "trueNegative": 1440, + "truePositive": 42, + "wilsonPrecisionLower95": 0.9162 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 45, + "recall": 1.0, + "total": 1485, + "trueNegative": 1440, + "truePositive": 45, + "wilsonPrecisionLower95": 0.9213 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.9831, + "falseNegative": 3, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 87, + "recall": 0.9667, + "total": 2970, + "trueNegative": 2880, + "truePositive": 87, + "wilsonPrecisionLower95": 0.9577 + }, + "threshold": 0.5 + }, + "task": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "trainingRecords": 16841, + "validation": { + "intentsMeeting95Precision": 8, + "macroF1": 0.478, + "macroPrecision": 0.982, + "macroRecall": 0.3466, + "minimumPrecision": 0.9074, + "perIntent": { + "blessing": { + "f1": 0.3486, + "falseNegative": 71, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 19, + "recall": 0.2111, + "total": 2970, + "trueNegative": 2880, + "truePositive": 19, + "wilsonPrecisionLower95": 0.8318 + }, + "complaint": { + "f1": 0.4506, + "falseNegative": 255, + "falsePositive": 1, + "precision": 0.9906, + "predictedPositive": 106, + "recall": 0.2917, + "total": 2970, + "trueNegative": 2609, + "truePositive": 105, + "wilsonPrecisionLower95": 0.9485 + }, + "confirmationDecision": { + "f1": 0.5366, + "falseNegative": 114, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 66, + "recall": 0.3667, + "total": 2970, + "trueNegative": 2790, + "truePositive": 66, + "wilsonPrecisionLower95": 0.945 + }, + "followUpReminder": { + "f1": 0.2367, + "falseNegative": 311, + "falsePositive": 5, + "precision": 0.9074, + "predictedPositive": 54, + "recall": 0.1361, + "total": 2970, + "trueNegative": 2605, + "truePositive": 49, + "wilsonPrecisionLower95": 0.8009 + }, + "invitation": { + "f1": 0.2692, + "falseNegative": 152, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 28, + "recall": 0.1556, + "total": 2970, + "trueNegative": 2790, + "truePositive": 28, + "wilsonPrecisionLower95": 0.8794 + }, + "question": { + "f1": 0.714, + "falseNegative": 278, + "falsePositive": 4, + "precision": 0.9888, + "predictedPositive": 356, + "recall": 0.5587, + "total": 2970, + "trueNegative": 2336, + "truePositive": 352, + "wilsonPrecisionLower95": 0.9715 + }, + "replyableMessage": { + "f1": 0.9005, + "falseNegative": 274, + "falsePositive": 83, + "precision": 0.9511, + "predictedPositive": 1699, + "recall": 0.855, + "total": 2970, + "trueNegative": 997, + "truePositive": 1616, + "wilsonPrecisionLower95": 0.9398 + }, + "scheduleNegotiation": { + "f1": 0.5124, + "falseNegative": 59, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 31, + "recall": 0.3444, + "total": 2970, + "trueNegative": 2880, + "truePositive": 31, + "wilsonPrecisionLower95": 0.8897 + }, + "task": { + "f1": 0.3333, + "falseNegative": 576, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 144, + "recall": 0.2, + "total": 2970, + "trueNegative": 2250, + "truePositive": 144, + "wilsonPrecisionLower95": 0.974 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 1.2e-06, + "augmentation": "numeric", + "char_max": 8, + "char_min": 3, + "external_weight": 0.93, + "hard_example_weight": 2.7, + "l1_ratio": 0.25, + "max_char_features": 95000, + "max_word_features": 45000, + "min_df": 1, + "pseudo_weight": 0.43499999999999994, + "round": 19, + "word_max": 3 + }, + "featureCount": 140000, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 1 + }, + "complaint": { + "negative": 250, + "positive": 2 + }, + "confirmationDecision": { + "negative": 250, + "positive": 36 + }, + "followUpReminder": { + "negative": 250, + "positive": 0 + }, + "invitation": { + "negative": 250, + "positive": 0 + }, + "question": { + "negative": 250, + "positive": 4 + }, + "replyableMessage": { + "negative": 99, + "positive": 32 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 0 + }, + "task": { + "negative": 250, + "positive": 0 + } + }, + "round": 19, + "selectionObjective": 0.6027, + "syntheticRegression": { + "intentsMeeting95Precision": 7, + "macroF1": 0.4167, + "macroPrecision": 0.9816, + "macroRecall": 0.2818, + "minimumPrecision": 0.9094, + "perIntent": { + "blessing": { + "f1": 0.2, + "falseNegative": 80, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 10, + "recall": 0.1111, + "total": 2970, + "trueNegative": 2880, + "truePositive": 10, + "wilsonPrecisionLower95": 0.7225 + }, + "complaint": { + "f1": 0.371, + "falseNegative": 278, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 82, + "recall": 0.2278, + "total": 2970, + "trueNegative": 2610, + "truePositive": 82, + "wilsonPrecisionLower95": 0.9552 + }, + "confirmationDecision": { + "f1": 0.502, + "falseNegative": 118, + "falsePositive": 5, + "precision": 0.9254, + "predictedPositive": 67, + "recall": 0.3444, + "total": 2970, + "trueNegative": 2785, + "truePositive": 62, + "wilsonPrecisionLower95": 0.8369 + }, + "followUpReminder": { + "f1": 0.231, + "falseNegative": 313, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 47, + "recall": 0.1306, + "total": 2970, + "trueNegative": 2610, + "truePositive": 47, + "wilsonPrecisionLower95": 0.9244 + }, + "invitation": { + "f1": 0.3636, + "falseNegative": 140, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 40, + "recall": 0.2222, + "total": 2970, + "trueNegative": 2790, + "truePositive": 40, + "wilsonPrecisionLower95": 0.9124 + }, + "question": { + "f1": 0.4387, + "falseNegative": 453, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 177, + "recall": 0.281, + "total": 2970, + "trueNegative": 2340, + "truePositive": 177, + "wilsonPrecisionLower95": 0.9788 + }, + "replyableMessage": { + "f1": 0.6399, + "falseNegative": 957, + "falsePositive": 93, + "precision": 0.9094, + "predictedPositive": 1026, + "recall": 0.4937, + "total": 2970, + "trueNegative": 987, + "truePositive": 933, + "wilsonPrecisionLower95": 0.8902 + }, + "scheduleNegotiation": { + "f1": 0.7143, + "falseNegative": 40, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 50, + "recall": 0.5556, + "total": 2970, + "trueNegative": 2880, + "truePositive": 50, + "wilsonPrecisionLower95": 0.9287 + }, + "task": { + "f1": 0.2898, + "falseNegative": 598, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 122, + "recall": 0.1694, + "total": 2970, + "trueNegative": 2250, + "truePositive": 122, + "wilsonPrecisionLower95": 0.9695 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "confirmationDecision": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "followUpReminder": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "replyableMessage": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9309, + "falseNegative": 83, + "falsePositive": 45, + "precision": 0.9504, + "predictedPositive": 907, + "recall": 0.9122, + "total": 1485, + "trueNegative": 495, + "truePositive": 862, + "wilsonPrecisionLower95": 0.9343 + }, + "threshold": 0.98 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 45, + "recall": 1.0, + "total": 1485, + "trueNegative": 1440, + "truePositive": 45, + "wilsonPrecisionLower95": 0.9213 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9195, + "falseNegative": 5, + "falsePositive": 2, + "precision": 0.9524, + "predictedPositive": 42, + "recall": 0.8889, + "total": 1485, + "trueNegative": 1438, + "truePositive": 40, + "wilsonPrecisionLower95": 0.8421 + }, + "threshold": 0.945 + } + }, + "metrics": { + "f1": 0.9667, + "falseNegative": 3, + "falsePositive": 3, + "precision": 0.9667, + "predictedPositive": 90, + "recall": 0.9667, + "total": 2970, + "trueNegative": 2877, + "truePositive": 87, + "wilsonPrecisionLower95": 0.9065 + }, + "threshold": 0.83 + }, + "task": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "trainingRecords": 16801, + "validation": { + "intentsMeeting95Precision": 7, + "macroF1": 0.4224, + "macroPrecision": 0.9791, + "macroRecall": 0.2773, + "minimumPrecision": 0.9215, + "perIntent": { + "blessing": { + "f1": 0.4483, + "falseNegative": 64, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 26, + "recall": 0.2889, + "total": 2970, + "trueNegative": 2880, + "truePositive": 26, + "wilsonPrecisionLower95": 0.8713 + }, + "complaint": { + "f1": 0.4648, + "falseNegative": 251, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 109, + "recall": 0.3028, + "total": 2970, + "trueNegative": 2610, + "truePositive": 109, + "wilsonPrecisionLower95": 0.966 + }, + "confirmationDecision": { + "f1": 0.4348, + "falseNegative": 130, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 50, + "recall": 0.2778, + "total": 2970, + "trueNegative": 2790, + "truePositive": 50, + "wilsonPrecisionLower95": 0.9287 + }, + "followUpReminder": { + "f1": 0.2045, + "falseNegative": 319, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 41, + "recall": 0.1139, + "total": 2970, + "trueNegative": 2610, + "truePositive": 41, + "wilsonPrecisionLower95": 0.9143 + }, + "invitation": { + "f1": 0.3019, + "falseNegative": 148, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 32, + "recall": 0.1778, + "total": 2970, + "trueNegative": 2790, + "truePositive": 32, + "wilsonPrecisionLower95": 0.8928 + }, + "question": { + "f1": 0.4287, + "falseNegative": 454, + "falsePositive": 15, + "precision": 0.9215, + "predictedPositive": 191, + "recall": 0.2794, + "total": 2970, + "trueNegative": 2325, + "truePositive": 176, + "wilsonPrecisionLower95": 0.8745 + }, + "replyableMessage": { + "f1": 0.6347, + "falseNegative": 990, + "falsePositive": 46, + "precision": 0.9514, + "predictedPositive": 946, + "recall": 0.4762, + "total": 2970, + "trueNegative": 1034, + "truePositive": 900, + "wilsonPrecisionLower95": 0.9357 + }, + "scheduleNegotiation": { + "f1": 0.5041, + "falseNegative": 59, + "falsePositive": 2, + "precision": 0.9394, + "predictedPositive": 33, + "recall": 0.3444, + "total": 2970, + "trueNegative": 2878, + "truePositive": 31, + "wilsonPrecisionLower95": 0.8039 + }, + "task": { + "f1": 0.3802, + "falseNegative": 551, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 169, + "recall": 0.2347, + "total": 2970, + "trueNegative": 2250, + "truePositive": 169, + "wilsonPrecisionLower95": 0.9778 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 1e-06, + "augmentation": "numeric", + "char_max": 8, + "char_min": 2, + "external_weight": 0.9500000000000001, + "hard_example_weight": 3.0, + "l1_ratio": 0.25, + "max_char_features": 100000, + "max_word_features": 50000, + "min_df": 1, + "pseudo_weight": 0.44999999999999996, + "round": 20, + "word_max": 3 + }, + "featureCount": 150000, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 1 + }, + "complaint": { + "negative": 250, + "positive": 2 + }, + "confirmationDecision": { + "negative": 250, + "positive": 25 + }, + "followUpReminder": { + "negative": 250, + "positive": 0 + }, + "invitation": { + "negative": 250, + "positive": 1 + }, + "question": { + "negative": 250, + "positive": 2 + }, + "replyableMessage": { + "negative": 104, + "positive": 32 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 1 + }, + "task": { + "negative": 250, + "positive": 0 + } + }, + "round": 20, + "selectionObjective": 0.6339, + "syntheticRegression": { + "intentsMeeting95Precision": 8, + "macroF1": 0.471, + "macroPrecision": 0.9861, + "macroRecall": 0.3411, + "minimumPrecision": 0.9231, + "perIntent": { + "blessing": { + "f1": 0.2353, + "falseNegative": 78, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 12, + "recall": 0.1333, + "total": 2970, + "trueNegative": 2880, + "truePositive": 12, + "wilsonPrecisionLower95": 0.7575 + }, + "complaint": { + "f1": 0.2816, + "falseNegative": 301, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 59, + "recall": 0.1639, + "total": 2970, + "trueNegative": 2610, + "truePositive": 59, + "wilsonPrecisionLower95": 0.9389 + }, + "confirmationDecision": { + "f1": 0.6199, + "falseNegative": 96, + "falsePositive": 7, + "precision": 0.9231, + "predictedPositive": 91, + "recall": 0.4667, + "total": 2970, + "trueNegative": 2783, + "truePositive": 84, + "wilsonPrecisionLower95": 0.8496 + }, + "followUpReminder": { + "f1": 0.2353, + "falseNegative": 312, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 48, + "recall": 0.1333, + "total": 2970, + "trueNegative": 2610, + "truePositive": 48, + "wilsonPrecisionLower95": 0.9259 + }, + "invitation": { + "f1": 0.3636, + "falseNegative": 140, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 40, + "recall": 0.2222, + "total": 2970, + "trueNegative": 2790, + "truePositive": 40, + "wilsonPrecisionLower95": 0.9124 + }, + "question": { + "f1": 0.5211, + "falseNegative": 408, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 222, + "recall": 0.3524, + "total": 2970, + "trueNegative": 2340, + "truePositive": 222, + "wilsonPrecisionLower95": 0.983 + }, + "replyableMessage": { + "f1": 0.7688, + "falseNegative": 671, + "falsePositive": 62, + "precision": 0.9516, + "predictedPositive": 1281, + "recall": 0.645, + "total": 2970, + "trueNegative": 1018, + "truePositive": 1219, + "wilsonPrecisionLower95": 0.9384 + }, + "scheduleNegotiation": { + "f1": 0.8387, + "falseNegative": 25, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 65, + "recall": 0.7222, + "total": 2970, + "trueNegative": 2880, + "truePositive": 65, + "wilsonPrecisionLower95": 0.9442 + }, + "task": { + "f1": 0.3747, + "falseNegative": 554, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 166, + "recall": 0.2306, + "total": 2970, + "trueNegative": 2250, + "truePositive": 166, + "wilsonPrecisionLower95": 0.9774 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "confirmationDecision": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.5954, + "falseNegative": 51, + "falsePositive": 2, + "precision": 0.9512, + "predictedPositive": 41, + "recall": 0.4333, + "total": 1485, + "trueNegative": 1393, + "truePositive": 39, + "wilsonPrecisionLower95": 0.8386 + }, + "threshold": 0.995 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "followUpReminder": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8117, + "falseNegative": 279, + "falsePositive": 30, + "precision": 0.9569, + "predictedPositive": 696, + "recall": 0.7048, + "total": 1485, + "trueNegative": 510, + "truePositive": 666, + "wilsonPrecisionLower95": 0.9391 + }, + "threshold": 0.995 + } + }, + "metrics": { + "f1": 0.8736, + "falseNegative": 363, + "falsePositive": 79, + "precision": 0.9508, + "predictedPositive": 1606, + "recall": 0.8079, + "total": 2970, + "trueNegative": 1001, + "truePositive": 1527, + "wilsonPrecisionLower95": 0.9391 + }, + "threshold": 0.995 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 45, + "recall": 1.0, + "total": 1485, + "trueNegative": 1440, + "truePositive": 45, + "wilsonPrecisionLower95": 0.9213 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.967, + "falseNegative": 1, + "falsePositive": 2, + "precision": 0.9565, + "predictedPositive": 46, + "recall": 0.9778, + "total": 1485, + "trueNegative": 1438, + "truePositive": 44, + "wilsonPrecisionLower95": 0.8547 + }, + "threshold": 0.83 + } + }, + "metrics": { + "f1": 0.9834, + "falseNegative": 1, + "falsePositive": 2, + "precision": 0.978, + "predictedPositive": 91, + "recall": 0.9889, + "total": 2970, + "trueNegative": 2878, + "truePositive": 89, + "wilsonPrecisionLower95": 0.9234 + }, + "threshold": 0.83 + }, + "task": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "trainingRecords": 16755, + "validation": { + "intentsMeeting95Precision": 8, + "macroF1": 0.4669, + "macroPrecision": 0.9855, + "macroRecall": 0.3215, + "minimumPrecision": 0.9333, + "perIntent": { + "blessing": { + "f1": 0.3929, + "falseNegative": 68, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 22, + "recall": 0.2444, + "total": 2970, + "trueNegative": 2880, + "truePositive": 22, + "wilsonPrecisionLower95": 0.8513 + }, + "complaint": { + "f1": 0.3929, + "falseNegative": 272, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 88, + "recall": 0.2444, + "total": 2970, + "trueNegative": 2610, + "truePositive": 88, + "wilsonPrecisionLower95": 0.9582 + }, + "confirmationDecision": { + "f1": 0.5366, + "falseNegative": 114, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 66, + "recall": 0.3667, + "total": 2970, + "trueNegative": 2790, + "truePositive": 66, + "wilsonPrecisionLower95": 0.945 + }, + "followUpReminder": { + "f1": 0.2074, + "falseNegative": 318, + "falsePositive": 3, + "precision": 0.9333, + "predictedPositive": 45, + "recall": 0.1167, + "total": 2970, + "trueNegative": 2607, + "truePositive": 42, + "wilsonPrecisionLower95": 0.8214 + }, + "invitation": { + "f1": 0.3178, + "falseNegative": 146, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 34, + "recall": 0.1889, + "total": 2970, + "trueNegative": 2790, + "truePositive": 34, + "wilsonPrecisionLower95": 0.8985 + }, + "question": { + "f1": 0.5568, + "falseNegative": 385, + "falsePositive": 5, + "precision": 0.98, + "predictedPositive": 250, + "recall": 0.3889, + "total": 2970, + "trueNegative": 2335, + "truePositive": 245, + "wilsonPrecisionLower95": 0.954 + }, + "replyableMessage": { + "f1": 0.7934, + "falseNegative": 621, + "falsePositive": 40, + "precision": 0.9694, + "predictedPositive": 1309, + "recall": 0.6714, + "total": 2970, + "trueNegative": 1040, + "truePositive": 1269, + "wilsonPrecisionLower95": 0.9587 + }, + "scheduleNegotiation": { + "f1": 0.5246, + "falseNegative": 58, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 32, + "recall": 0.3556, + "total": 2970, + "trueNegative": 2880, + "truePositive": 32, + "wilsonPrecisionLower95": 0.8928 + }, + "task": { + "f1": 0.4795, + "falseNegative": 492, + "falsePositive": 3, + "precision": 0.987, + "predictedPositive": 231, + "recall": 0.3167, + "total": 2970, + "trueNegative": 2247, + "truePositive": 228, + "wilsonPrecisionLower95": 0.9625 + } + }, + "recordCount": 2970 + } + } +] diff --git a/ModelTraining/ClipboardSemantics/IterativeResearch/PerIntentFineTune/final-report.json b/ModelTraining/ClipboardSemantics/IterativeResearch/PerIntentFineTune/final-report.json new file mode 100644 index 0000000..9fcd3c2 --- /dev/null +++ b/ModelTraining/ClipboardSemantics/IterativeResearch/PerIntentFineTune/final-report.json @@ -0,0 +1,8422 @@ +{ + "baseTrainingRecords": 11500, + "deploymentDecision": "retain-current-deployed-models", + "elapsedSeconds": 188.7732, + "finalEvaluations": { + "golden": { + "byLanguage": { + "en": { + "intentsMeeting95Precision": 1, + "macroF1": 0.847, + "macroPrecision": 0.9225, + "macroRecall": 0.8029, + "minimumPrecision": 0.7692, + "perIntent": { + "blessing": { + "f1": 0.8235, + "falseNegative": 3, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 7, + "recall": 0.7, + "total": 126, + "trueNegative": 116, + "truePositive": 7, + "wilsonPrecisionLower95": 0.6457 + }, + "complaint": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 6, + "recall": 1.0, + "total": 126, + "trueNegative": 120, + "truePositive": 6, + "wilsonPrecisionLower95": 0.6097 + }, + "confirmationDecision": { + "f1": 0.8696, + "falseNegative": 0, + "falsePositive": 3, + "precision": 0.7692, + "predictedPositive": 13, + "recall": 1.0, + "total": 126, + "trueNegative": 113, + "truePositive": 10, + "wilsonPrecisionLower95": 0.4974 + }, + "followUpReminder": { + "f1": 0.8696, + "falseNegative": 3, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 10, + "recall": 0.7692, + "total": 126, + "trueNegative": 113, + "truePositive": 10, + "wilsonPrecisionLower95": 0.7225 + }, + "invitation": { + "f1": 0.8571, + "falseNegative": 2, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 6, + "recall": 0.75, + "total": 126, + "trueNegative": 118, + "truePositive": 6, + "wilsonPrecisionLower95": 0.6097 + }, + "question": { + "f1": 0.8636, + "falseNegative": 4, + "falsePositive": 2, + "precision": 0.9048, + "predictedPositive": 21, + "recall": 0.8261, + "total": 126, + "trueNegative": 101, + "truePositive": 19, + "wilsonPrecisionLower95": 0.7109 + }, + "replyableMessage": { + "f1": 0.8889, + "falseNegative": 6, + "falsePositive": 10, + "precision": 0.8649, + "predictedPositive": 74, + "recall": 0.9143, + "total": 126, + "trueNegative": 46, + "truePositive": 64, + "wilsonPrecisionLower95": 0.7688 + }, + "scheduleNegotiation": { + "f1": 0.8421, + "falseNegative": 2, + "falsePositive": 1, + "precision": 0.8889, + "predictedPositive": 9, + "recall": 0.8, + "total": 126, + "trueNegative": 115, + "truePositive": 8, + "wilsonPrecisionLower95": 0.565 + }, + "task": { + "f1": 0.6087, + "falseNegative": 8, + "falsePositive": 1, + "precision": 0.875, + "predictedPositive": 8, + "recall": 0.4667, + "total": 126, + "trueNegative": 110, + "truePositive": 7, + "wilsonPrecisionLower95": 0.5291 + } + }, + "recordCount": 126 + }, + "zh-Hans": { + "intentsMeeting95Precision": 1, + "macroF1": 0.652, + "macroPrecision": 0.9656, + "macroRecall": 0.5609, + "minimumPrecision": 0.8182, + "perIntent": { + "blessing": { + "f1": 0.3333, + "falseNegative": 8, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 2, + "recall": 0.2, + "total": 126, + "trueNegative": 116, + "truePositive": 2, + "wilsonPrecisionLower95": 0.3424 + }, + "complaint": { + "f1": 0.6667, + "falseNegative": 3, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 3, + "recall": 0.5, + "total": 126, + "trueNegative": 120, + "truePositive": 3, + "wilsonPrecisionLower95": 0.4385 + }, + "confirmationDecision": { + "f1": 0.8571, + "falseNegative": 1, + "falsePositive": 2, + "precision": 0.8182, + "predictedPositive": 11, + "recall": 0.9, + "total": 126, + "trueNegative": 114, + "truePositive": 9, + "wilsonPrecisionLower95": 0.523 + }, + "followUpReminder": { + "f1": 0.8182, + "falseNegative": 4, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 9, + "recall": 0.6923, + "total": 126, + "trueNegative": 113, + "truePositive": 9, + "wilsonPrecisionLower95": 0.7009 + }, + "invitation": { + "f1": 0.5455, + "falseNegative": 5, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 3, + "recall": 0.375, + "total": 126, + "trueNegative": 118, + "truePositive": 3, + "wilsonPrecisionLower95": 0.4385 + }, + "question": { + "f1": 0.9787, + "falseNegative": 0, + "falsePositive": 1, + "precision": 0.9583, + "predictedPositive": 24, + "recall": 1.0, + "total": 126, + "trueNegative": 102, + "truePositive": 23, + "wilsonPrecisionLower95": 0.7976 + }, + "replyableMessage": { + "f1": 0.9143, + "falseNegative": 6, + "falsePositive": 6, + "precision": 0.9143, + "predictedPositive": 70, + "recall": 0.9143, + "total": 126, + "trueNegative": 50, + "truePositive": 64, + "wilsonPrecisionLower95": 0.8253 + }, + "scheduleNegotiation": { + "f1": 0.3333, + "falseNegative": 8, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 2, + "recall": 0.2, + "total": 126, + "trueNegative": 116, + "truePositive": 2, + "wilsonPrecisionLower95": 0.3424 + }, + "task": { + "f1": 0.4211, + "falseNegative": 11, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 4, + "recall": 0.2667, + "total": 126, + "trueNegative": 111, + "truePositive": 4, + "wilsonPrecisionLower95": 0.5101 + } + }, + "recordCount": 126 + } + }, + "bySource": { + "acknowledgment": { + "intentsMeeting95Precision": 0, + "macroF1": 0.0, + "macroPrecision": 0.0, + "macroRecall": 0.0, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 1, + "precision": 0.0, + "predictedPositive": 1, + "recall": 0.0, + "total": 8, + "trueNegative": 7, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 8 + }, + "blessing_boundary": { + "intentsMeeting95Precision": 0, + "macroF1": 0.1482, + "macroPrecision": 0.1667, + "macroRecall": 0.1667, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.6667, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 1, + "recall": 0.5, + "total": 20, + "trueNegative": 18, + "truePositive": 1, + "wilsonPrecisionLower95": 0.2065 + }, + "replyableMessage": { + "f1": 0.6667, + "falseNegative": 0, + "falsePositive": 4, + "precision": 0.5, + "predictedPositive": 8, + "recall": 1.0, + "total": 20, + "trueNegative": 12, + "truePositive": 4, + "wilsonPrecisionLower95": 0.2152 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 2, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 18, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 20 + }, + "blessing_golden": { + "intentsMeeting95Precision": 1, + "macroF1": 0.1772, + "macroPrecision": 0.2222, + "macroRecall": 0.1556, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.6207, + "falseNegative": 11, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 9, + "recall": 0.45, + "total": 20, + "trueNegative": 0, + "truePositive": 9, + "wilsonPrecisionLower95": 0.7009 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.9744, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 19, + "recall": 0.95, + "total": 20, + "trueNegative": 0, + "truePositive": 19, + "wilsonPrecisionLower95": 0.8318 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 20 + }, + "complaint_request": { + "intentsMeeting95Precision": 0, + "macroF1": 0.2703, + "macroPrecision": 0.3333, + "macroRecall": 0.2315, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.6667, + "falseNegative": 3, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 3, + "recall": 0.5, + "total": 6, + "trueNegative": 0, + "truePositive": 3, + "wilsonPrecisionLower95": 0.4385 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.8571, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 3, + "recall": 0.75, + "total": 6, + "trueNegative": 2, + "truePositive": 3, + "wilsonPrecisionLower95": 0.4385 + }, + "replyableMessage": { + "f1": 0.9091, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 5, + "recall": 0.8333, + "total": 6, + "trueNegative": 0, + "truePositive": 5, + "wilsonPrecisionLower95": 0.5655 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 6 + }, + "complaint_statement": { + "intentsMeeting95Precision": 0, + "macroF1": 0.2222, + "macroPrecision": 0.2222, + "macroRecall": 0.2222, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 6, + "recall": 1.0, + "total": 6, + "trueNegative": 0, + "truePositive": 6, + "wilsonPrecisionLower95": 0.6097 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 6, + "recall": 1.0, + "total": 6, + "trueNegative": 0, + "truePositive": 6, + "wilsonPrecisionLower95": 0.6097 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 6 + }, + "confirmation_decision_boundary": { + "intentsMeeting95Precision": 0, + "macroF1": 0.0, + "macroPrecision": 0.0, + "macroRecall": 0.0, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 2, + "precision": 0.0, + "predictedPositive": 2, + "recall": 0.0, + "total": 20, + "trueNegative": 18, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 3, + "precision": 0.0, + "predictedPositive": 3, + "recall": 0.0, + "total": 20, + "trueNegative": 17, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 20 + }, + "confirmation_decision_golden": { + "intentsMeeting95Precision": 2, + "macroF1": 0.2194, + "macroPrecision": 0.2222, + "macroRecall": 0.2167, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.9744, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 19, + "recall": 0.95, + "total": 20, + "trueNegative": 0, + "truePositive": 19, + "wilsonPrecisionLower95": 0.8318 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 20, + "recall": 1.0, + "total": 20, + "trueNegative": 0, + "truePositive": 20, + "wilsonPrecisionLower95": 0.8389 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 20 + }, + "conversational_message": { + "intentsMeeting95Precision": 1, + "macroF1": 0.1111, + "macroPrecision": 0.1111, + "macroRecall": 0.1111, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 10, + "trueNegative": 10, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 10, + "trueNegative": 10, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 10, + "trueNegative": 10, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 10, + "trueNegative": 10, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 10, + "trueNegative": 10, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 10, + "trueNegative": 10, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 10, + "recall": 1.0, + "total": 10, + "trueNegative": 0, + "truePositive": 10, + "wilsonPrecisionLower95": 0.7225 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 10, + "trueNegative": 10, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 10, + "trueNegative": 10, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 10 + }, + "event_statement": { + "intentsMeeting95Precision": 0, + "macroF1": 0.0, + "macroPrecision": 0.0, + "macroRecall": 0.0, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 4 + }, + "follow_up_reminder_boundary": { + "intentsMeeting95Precision": 0, + "macroF1": 0.0, + "macroPrecision": 0.0, + "macroRecall": 0.0, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 1, + "precision": 0.0, + "predictedPositive": 1, + "recall": 0.0, + "total": 20, + "trueNegative": 19, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 3, + "precision": 0.0, + "predictedPositive": 3, + "recall": 0.0, + "total": 20, + "trueNegative": 17, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 20 + }, + "follow_up_reminder_golden": { + "intentsMeeting95Precision": 1, + "macroF1": 0.2187, + "macroPrecision": 0.3333, + "macroRecall": 0.1706, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.8571, + "falseNegative": 5, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 15, + "recall": 0.75, + "total": 20, + "trueNegative": 0, + "truePositive": 15, + "wilsonPrecisionLower95": 0.7961 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.6667, + "falseNegative": 7, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 7, + "recall": 0.5, + "total": 20, + "trueNegative": 6, + "truePositive": 7, + "wilsonPrecisionLower95": 0.6457 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.4444, + "falseNegative": 10, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 4, + "recall": 0.2857, + "total": 20, + "trueNegative": 6, + "truePositive": 4, + "wilsonPrecisionLower95": 0.5101 + } + }, + "recordCount": 20 + }, + "information_question": { + "intentsMeeting95Precision": 0, + "macroF1": 0.1693, + "macroPrecision": 0.2222, + "macroRecall": 0.1389, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.6667, + "falseNegative": 2, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 2, + "recall": 0.5, + "total": 4, + "trueNegative": 0, + "truePositive": 2, + "wilsonPrecisionLower95": 0.3424 + }, + "replyableMessage": { + "f1": 0.8571, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 3, + "recall": 0.75, + "total": 4, + "trueNegative": 0, + "truePositive": 3, + "wilsonPrecisionLower95": 0.4385 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 4 + }, + "invitation_question": { + "intentsMeeting95Precision": 0, + "macroF1": 0.3175, + "macroPrecision": 0.3333, + "macroRecall": 0.3056, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.8571, + "falseNegative": 2, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 6, + "recall": 0.75, + "total": 8, + "trueNegative": 0, + "truePositive": 6, + "wilsonPrecisionLower95": 0.6097 + }, + "question": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 8, + "recall": 1.0, + "total": 8, + "trueNegative": 0, + "truePositive": 8, + "wilsonPrecisionLower95": 0.6756 + }, + "replyableMessage": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 8, + "recall": 1.0, + "total": 8, + "trueNegative": 0, + "truePositive": 8, + "wilsonPrecisionLower95": 0.6756 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 8 + }, + "negative_news": { + "intentsMeeting95Precision": 0, + "macroF1": 0.0, + "macroPrecision": 0.0, + "macroRecall": 0.0, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 1, + "precision": 0.0, + "predictedPositive": 1, + "recall": 0.0, + "total": 6, + "trueNegative": 5, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 6 + }, + "neutral_fact": { + "intentsMeeting95Precision": 0, + "macroF1": 0.0, + "macroPrecision": 0.0, + "macroRecall": 0.0, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 10, + "trueNegative": 10, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 10, + "trueNegative": 10, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 10, + "trueNegative": 10, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 10, + "trueNegative": 10, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 10, + "trueNegative": 10, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 10, + "trueNegative": 10, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 2, + "precision": 0.0, + "predictedPositive": 2, + "recall": 0.0, + "total": 10, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 10, + "trueNegative": 10, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 10, + "trueNegative": 10, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 10 + }, + "positive_feedback": { + "intentsMeeting95Precision": 0, + "macroF1": 0.1111, + "macroPrecision": 0.1111, + "macroRecall": 0.1111, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 6, + "recall": 1.0, + "total": 6, + "trueNegative": 0, + "truePositive": 6, + "wilsonPrecisionLower95": 0.6097 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 6 + }, + "quoted_question": { + "intentsMeeting95Precision": 0, + "macroF1": 0.0, + "macroPrecision": 0.0, + "macroRecall": 0.0, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 1, + "precision": 0.0, + "predictedPositive": 1, + "recall": 0.0, + "total": 4, + "trueNegative": 3, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + } + }, + "recordCount": 4 + }, + "schedule_negotiation_boundary": { + "intentsMeeting95Precision": 0, + "macroF1": 0.2541, + "macroPrecision": 0.284, + "macroRecall": 0.2639, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.5455, + "falseNegative": 5, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 3, + "recall": 0.375, + "total": 20, + "trueNegative": 12, + "truePositive": 3, + "wilsonPrecisionLower95": 0.4385 + }, + "question": { + "f1": 0.8, + "falseNegative": 0, + "falsePositive": 2, + "precision": 0.6667, + "predictedPositive": 6, + "recall": 1.0, + "total": 20, + "trueNegative": 14, + "truePositive": 4, + "wilsonPrecisionLower95": 0.3 + }, + "replyableMessage": { + "f1": 0.9412, + "falseNegative": 0, + "falsePositive": 1, + "precision": 0.8889, + "predictedPositive": 9, + "recall": 1.0, + "total": 20, + "trueNegative": 11, + "truePositive": 8, + "wilsonPrecisionLower95": 0.565 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 1, + "precision": 0.0, + "predictedPositive": 1, + "recall": 0.0, + "total": 20, + "trueNegative": 19, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 20 + }, + "schedule_negotiation_golden": { + "intentsMeeting95Precision": 2, + "macroF1": 0.2905, + "macroPrecision": 0.3275, + "macroRecall": 0.2722, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 1, + "precision": 0.0, + "predictedPositive": 1, + "recall": 0.0, + "total": 20, + "trueNegative": 19, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.973, + "falseNegative": 0, + "falsePositive": 1, + "precision": 0.9474, + "predictedPositive": 19, + "recall": 1.0, + "total": 20, + "trueNegative": 1, + "truePositive": 18, + "wilsonPrecisionLower95": 0.7536 + }, + "replyableMessage": { + "f1": 0.9744, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 19, + "recall": 0.95, + "total": 20, + "trueNegative": 0, + "truePositive": 19, + "wilsonPrecisionLower95": 0.8318 + }, + "scheduleNegotiation": { + "f1": 0.6667, + "falseNegative": 10, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 10, + "recall": 0.5, + "total": 20, + "trueNegative": 0, + "truePositive": 10, + "wilsonPrecisionLower95": 0.7225 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 20 + }, + "self_plan": { + "intentsMeeting95Precision": 0, + "macroF1": 0.1111, + "macroPrecision": 0.1111, + "macroRecall": 0.1111, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 2, + "recall": 1.0, + "total": 6, + "trueNegative": 4, + "truePositive": 2, + "wilsonPrecisionLower95": 0.3424 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 2, + "precision": 0.0, + "predictedPositive": 2, + "recall": 0.0, + "total": 6, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 6 + }, + "task_question": { + "intentsMeeting95Precision": 0, + "macroF1": 0.2963, + "macroPrecision": 0.3333, + "macroRecall": 0.2778, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 6, + "recall": 1.0, + "total": 6, + "trueNegative": 0, + "truePositive": 6, + "wilsonPrecisionLower95": 0.6097 + }, + "replyableMessage": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 6, + "recall": 1.0, + "total": 6, + "trueNegative": 0, + "truePositive": 6, + "wilsonPrecisionLower95": 0.6097 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.6667, + "falseNegative": 3, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 3, + "recall": 0.5, + "total": 6, + "trueNegative": 0, + "truePositive": 3, + "wilsonPrecisionLower95": 0.4385 + } + }, + "recordCount": 6 + }, + "task_statement": { + "intentsMeeting95Precision": 0, + "macroF1": 0.2519, + "macroPrecision": 0.3333, + "macroRecall": 0.2083, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.6667, + "falseNegative": 2, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 2, + "recall": 0.5, + "total": 8, + "trueNegative": 4, + "truePositive": 2, + "wilsonPrecisionLower95": 0.3424 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.9333, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 7, + "recall": 0.875, + "total": 8, + "trueNegative": 0, + "truePositive": 7, + "wilsonPrecisionLower95": 0.6457 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.6667, + "falseNegative": 4, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 4, + "recall": 0.5, + "total": 8, + "trueNegative": 0, + "truePositive": 4, + "wilsonPrecisionLower95": 0.5101 + } + }, + "recordCount": 8 + } + }, + "intentsMeeting95Precision": 1, + "macroF1": 0.7666, + "macroPrecision": 0.9377, + "macroRecall": 0.6819, + "minimumPrecision": 0.7917, + "perIntent": { + "blessing": { + "f1": 0.6207, + "falseNegative": 11, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 9, + "recall": 0.45, + "total": 252, + "trueNegative": 232, + "truePositive": 9, + "wilsonPrecisionLower95": 0.7009 + }, + "complaint": { + "f1": 0.8571, + "falseNegative": 3, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 9, + "recall": 0.75, + "total": 252, + "trueNegative": 240, + "truePositive": 9, + "wilsonPrecisionLower95": 0.7009 + }, + "confirmationDecision": { + "f1": 0.8636, + "falseNegative": 1, + "falsePositive": 5, + "precision": 0.7917, + "predictedPositive": 24, + "recall": 0.95, + "total": 252, + "trueNegative": 227, + "truePositive": 19, + "wilsonPrecisionLower95": 0.5953 + }, + "followUpReminder": { + "f1": 0.8444, + "falseNegative": 7, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 19, + "recall": 0.7308, + "total": 252, + "trueNegative": 226, + "truePositive": 19, + "wilsonPrecisionLower95": 0.8318 + }, + "invitation": { + "f1": 0.72, + "falseNegative": 7, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 9, + "recall": 0.5625, + "total": 252, + "trueNegative": 236, + "truePositive": 9, + "wilsonPrecisionLower95": 0.7009 + }, + "question": { + "f1": 0.9231, + "falseNegative": 4, + "falsePositive": 3, + "precision": 0.9333, + "predictedPositive": 45, + "recall": 0.913, + "total": 252, + "trueNegative": 203, + "truePositive": 42, + "wilsonPrecisionLower95": 0.8214 + }, + "replyableMessage": { + "f1": 0.9014, + "falseNegative": 12, + "falsePositive": 16, + "precision": 0.8889, + "predictedPositive": 144, + "recall": 0.9143, + "total": 252, + "trueNegative": 96, + "truePositive": 128, + "wilsonPrecisionLower95": 0.8271 + }, + "scheduleNegotiation": { + "f1": 0.6452, + "falseNegative": 10, + "falsePositive": 1, + "precision": 0.9091, + "predictedPositive": 11, + "recall": 0.5, + "total": 252, + "trueNegative": 231, + "truePositive": 10, + "wilsonPrecisionLower95": 0.6226 + }, + "task": { + "f1": 0.5238, + "falseNegative": 19, + "falsePositive": 1, + "precision": 0.9167, + "predictedPositive": 12, + "recall": 0.3667, + "total": 252, + "trueNegative": 221, + "truePositive": 11, + "wilsonPrecisionLower95": 0.6461 + } + }, + "recordCount": 252 + }, + "randomHoldout": { + "byLanguage": { + "en": { + "intentsMeeting95Precision": 5, + "macroF1": 0.6248, + "macroPrecision": 0.8629, + "macroRecall": 0.5355, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 340, + "trueNegative": 340, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.3333, + "falseNegative": 16, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 4, + "recall": 0.2, + "total": 340, + "trueNegative": 320, + "truePositive": 4, + "wilsonPrecisionLower95": 0.5101 + }, + "confirmationDecision": { + "f1": 0.9091, + "falseNegative": 0, + "falsePositive": 4, + "precision": 0.8333, + "predictedPositive": 24, + "recall": 1.0, + "total": 340, + "trueNegative": 316, + "truePositive": 20, + "wilsonPrecisionLower95": 0.6415 + }, + "followUpReminder": { + "f1": 0.9027, + "falseNegative": 9, + "falsePositive": 2, + "precision": 0.9623, + "predictedPositive": 53, + "recall": 0.85, + "total": 340, + "trueNegative": 278, + "truePositive": 51, + "wilsonPrecisionLower95": 0.8725 + }, + "invitation": { + "f1": 0.5714, + "falseNegative": 12, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 8, + "recall": 0.4, + "total": 340, + "trueNegative": 320, + "truePositive": 8, + "wilsonPrecisionLower95": 0.6756 + }, + "question": { + "f1": 0.6087, + "falseNegative": 45, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 35, + "recall": 0.4375, + "total": 340, + "trueNegative": 260, + "truePositive": 35, + "wilsonPrecisionLower95": 0.9011 + }, + "replyableMessage": { + "f1": 0.8859, + "falseNegative": 37, + "falsePositive": 5, + "precision": 0.9702, + "predictedPositive": 168, + "recall": 0.815, + "total": 340, + "trueNegative": 135, + "truePositive": 163, + "wilsonPrecisionLower95": 0.9322 + }, + "scheduleNegotiation": { + "f1": 0.8235, + "falseNegative": 6, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 14, + "recall": 0.7, + "total": 340, + "trueNegative": 320, + "truePositive": 14, + "wilsonPrecisionLower95": 0.7847 + }, + "task": { + "f1": 0.5882, + "falseNegative": 35, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 25, + "recall": 0.4167, + "total": 340, + "trueNegative": 280, + "truePositive": 25, + "wilsonPrecisionLower95": 0.8668 + } + }, + "recordCount": 340 + }, + "zh-Hans": { + "intentsMeeting95Precision": 3, + "macroF1": 0.497, + "macroPrecision": 0.7719, + "macroRecall": 0.4055, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 340, + "trueNegative": 340, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.3333, + "falseNegative": 16, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 4, + "recall": 0.2, + "total": 340, + "trueNegative": 320, + "truePositive": 4, + "wilsonPrecisionLower95": 0.5101 + }, + "confirmationDecision": { + "f1": 0.8235, + "falseNegative": 6, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 14, + "recall": 0.7, + "total": 340, + "trueNegative": 320, + "truePositive": 14, + "wilsonPrecisionLower95": 0.7847 + }, + "followUpReminder": { + "f1": 0.7879, + "falseNegative": 21, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 39, + "recall": 0.65, + "total": 340, + "trueNegative": 280, + "truePositive": 39, + "wilsonPrecisionLower95": 0.9103 + }, + "invitation": { + "f1": 0.6207, + "falseNegative": 11, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 9, + "recall": 0.45, + "total": 340, + "trueNegative": 320, + "truePositive": 9, + "wilsonPrecisionLower95": 0.7009 + }, + "question": { + "f1": 0.7786, + "falseNegative": 29, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 51, + "recall": 0.6375, + "total": 340, + "trueNegative": 260, + "truePositive": 51, + "wilsonPrecisionLower95": 0.93 + }, + "replyableMessage": { + "f1": 0.9203, + "falseNegative": 21, + "falsePositive": 10, + "precision": 0.9471, + "predictedPositive": 189, + "recall": 0.895, + "total": 340, + "trueNegative": 130, + "truePositive": 179, + "wilsonPrecisionLower95": 0.9054 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 20, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 340, + "trueNegative": 320, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.209, + "falseNegative": 53, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 7, + "recall": 0.1167, + "total": 340, + "trueNegative": 280, + "truePositive": 7, + "wilsonPrecisionLower95": 0.6457 + } + }, + "recordCount": 340 + } + }, + "bySource": { + "acknowledgment_boundary": { + "intentsMeeting95Precision": 0, + "macroF1": 0.0, + "macroPrecision": 0.0, + "macroRecall": 0.0, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 40 + }, + "complaint_support": { + "intentsMeeting95Precision": 1, + "macroF1": 0.1783, + "macroPrecision": 0.3333, + "macroRecall": 0.1389, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.3333, + "falseNegative": 32, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 8, + "recall": 0.2, + "total": 40, + "trueNegative": 0, + "truePositive": 8, + "wilsonPrecisionLower95": 0.6756 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.3673, + "falseNegative": 31, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 9, + "recall": 0.225, + "total": 40, + "trueNegative": 0, + "truePositive": 9, + "wilsonPrecisionLower95": 0.7009 + }, + "replyableMessage": { + "f1": 0.9041, + "falseNegative": 7, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 33, + "recall": 0.825, + "total": 40, + "trueNegative": 0, + "truePositive": 33, + "wilsonPrecisionLower95": 0.8957 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 40 + }, + "confirmation_decision": { + "intentsMeeting95Precision": 2, + "macroF1": 0.2026, + "macroPrecision": 0.2222, + "macroRecall": 0.1861, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.9189, + "falseNegative": 6, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 34, + "recall": 0.85, + "total": 40, + "trueNegative": 0, + "truePositive": 34, + "wilsonPrecisionLower95": 0.8985 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.9041, + "falseNegative": 7, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 33, + "recall": 0.825, + "total": 40, + "trueNegative": 0, + "truePositive": 33, + "wilsonPrecisionLower95": 0.8957 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 40 + }, + "conversation": { + "intentsMeeting95Precision": 1, + "macroF1": 0.1053, + "macroPrecision": 0.1111, + "macroRecall": 0.1, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.9474, + "falseNegative": 4, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 36, + "recall": 0.9, + "total": 40, + "trueNegative": 0, + "truePositive": 36, + "wilsonPrecisionLower95": 0.9036 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 40 + }, + "event_boundary": { + "intentsMeeting95Precision": 0, + "macroF1": 0.0, + "macroPrecision": 0.0, + "macroRecall": 0.0, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 2, + "precision": 0.0, + "predictedPositive": 2, + "recall": 0.0, + "total": 40, + "trueNegative": 38, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 40 + }, + "fixed_invitation": { + "intentsMeeting95Precision": 3, + "macroF1": 0.2797, + "macroPrecision": 0.3333, + "macroRecall": 0.2528, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.5965, + "falseNegative": 23, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 17, + "recall": 0.425, + "total": 40, + "trueNegative": 0, + "truePositive": 17, + "wilsonPrecisionLower95": 0.8157 + }, + "question": { + "f1": 0.9333, + "falseNegative": 5, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 35, + "recall": 0.875, + "total": 40, + "trueNegative": 0, + "truePositive": 35, + "wilsonPrecisionLower95": 0.9011 + }, + "replyableMessage": { + "f1": 0.9873, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 39, + "recall": 0.975, + "total": 40, + "trueNegative": 0, + "truePositive": 39, + "wilsonPrecisionLower95": 0.9103 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 40 + }, + "follow_up_instruction": { + "intentsMeeting95Precision": 3, + "macroF1": 0.2225, + "macroPrecision": 0.3333, + "macroRecall": 0.175, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.7302, + "falseNegative": 17, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 23, + "recall": 0.575, + "total": 40, + "trueNegative": 0, + "truePositive": 23, + "wilsonPrecisionLower95": 0.8569 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.8406, + "falseNegative": 11, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 29, + "recall": 0.725, + "total": 40, + "trueNegative": 0, + "truePositive": 29, + "wilsonPrecisionLower95": 0.883 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.4314, + "falseNegative": 29, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 11, + "recall": 0.275, + "total": 40, + "trueNegative": 0, + "truePositive": 11, + "wilsonPrecisionLower95": 0.7412 + } + }, + "recordCount": 40 + }, + "follow_up_personal_reminder": { + "intentsMeeting95Precision": 1, + "macroF1": 0.097, + "macroPrecision": 0.1111, + "macroRecall": 0.0861, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.8732, + "falseNegative": 9, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 31, + "recall": 0.775, + "total": 40, + "trueNegative": 0, + "truePositive": 31, + "wilsonPrecisionLower95": 0.8897 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 40 + }, + "information_question": { + "intentsMeeting95Precision": 2, + "macroF1": 0.1421, + "macroPrecision": 0.2222, + "macroRecall": 0.1083, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 2, + "precision": 0.0, + "predictedPositive": 2, + "recall": 0.0, + "total": 40, + "trueNegative": 38, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 2, + "precision": 0.0, + "predictedPositive": 2, + "recall": 0.0, + "total": 40, + "trueNegative": 38, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.4906, + "falseNegative": 27, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 13, + "recall": 0.325, + "total": 40, + "trueNegative": 0, + "truePositive": 13, + "wilsonPrecisionLower95": 0.7719 + }, + "replyableMessage": { + "f1": 0.7879, + "falseNegative": 14, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 26, + "recall": 0.65, + "total": 40, + "trueNegative": 0, + "truePositive": 26, + "wilsonPrecisionLower95": 0.8713 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 40 + }, + "negative_news": { + "intentsMeeting95Precision": 0, + "macroF1": 0.0, + "macroPrecision": 0.0, + "macroRecall": 0.0, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 3, + "precision": 0.0, + "predictedPositive": 3, + "recall": 0.0, + "total": 40, + "trueNegative": 37, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 40 + }, + "neutral_fact": { + "intentsMeeting95Precision": 0, + "macroF1": 0.0, + "macroPrecision": 0.0, + "macroRecall": 0.0, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 5, + "precision": 0.0, + "predictedPositive": 5, + "recall": 0.0, + "total": 40, + "trueNegative": 35, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 40 + }, + "positive_feedback": { + "intentsMeeting95Precision": 1, + "macroF1": 0.1068, + "macroPrecision": 0.1111, + "macroRecall": 0.1028, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.961, + "falseNegative": 3, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 37, + "recall": 0.925, + "total": 40, + "trueNegative": 0, + "truePositive": 37, + "wilsonPrecisionLower95": 0.9059 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 40 + }, + "quoted_question_boundary": { + "intentsMeeting95Precision": 0, + "macroF1": 0.0, + "macroPrecision": 0.0, + "macroRecall": 0.0, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 1, + "precision": 0.0, + "predictedPositive": 1, + "recall": 0.0, + "total": 40, + "trueNegative": 39, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 40 + }, + "schedule_negotiation": { + "intentsMeeting95Precision": 3, + "macroF1": 0.2621, + "macroPrecision": 0.3333, + "macroRecall": 0.2306, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.8406, + "falseNegative": 11, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 29, + "recall": 0.725, + "total": 40, + "trueNegative": 0, + "truePositive": 29, + "wilsonPrecisionLower95": 0.883 + }, + "replyableMessage": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 40, + "recall": 1.0, + "total": 40, + "trueNegative": 0, + "truePositive": 40, + "wilsonPrecisionLower95": 0.9124 + }, + "scheduleNegotiation": { + "f1": 0.5185, + "falseNegative": 26, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 14, + "recall": 0.35, + "total": 40, + "trueNegative": 0, + "truePositive": 14, + "wilsonPrecisionLower95": 0.7847 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 40 + }, + "task_next_action": { + "intentsMeeting95Precision": 2, + "macroF1": 0.2304, + "macroPrecision": 0.3333, + "macroRecall": 0.2056, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 1, + "precision": 0.0, + "predictedPositive": 1, + "recall": 0.0, + "total": 40, + "trueNegative": 39, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "followUpReminder": { + "f1": 0.9474, + "falseNegative": 4, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 36, + "recall": 0.9, + "total": 40, + "trueNegative": 0, + "truePositive": 36, + "wilsonPrecisionLower95": 0.9036 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.9041, + "falseNegative": 7, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 33, + "recall": 0.825, + "total": 40, + "trueNegative": 0, + "truePositive": 33, + "wilsonPrecisionLower95": 0.8957 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.2222, + "falseNegative": 35, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 5, + "recall": 0.125, + "total": 40, + "trueNegative": 0, + "truePositive": 5, + "wilsonPrecisionLower95": 0.5655 + } + }, + "recordCount": 40 + }, + "task_request": { + "intentsMeeting95Precision": 2, + "macroF1": 0.1688, + "macroPrecision": 0.2222, + "macroRecall": 0.1444, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 1, + "precision": 0.0, + "predictedPositive": 1, + "recall": 0.0, + "total": 40, + "trueNegative": 39, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.9474, + "falseNegative": 4, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 36, + "recall": 0.9, + "total": 40, + "trueNegative": 0, + "truePositive": 36, + "wilsonPrecisionLower95": 0.9036 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.5714, + "falseNegative": 24, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 16, + "recall": 0.4, + "total": 40, + "trueNegative": 0, + "truePositive": 16, + "wilsonPrecisionLower95": 0.8064 + } + }, + "recordCount": 40 + }, + "vague_future_boundary": { + "intentsMeeting95Precision": 0, + "macroF1": 0.0, + "macroPrecision": 0.0, + "macroRecall": 0.0, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 4, + "precision": 0.0, + "predictedPositive": 4, + "recall": 0.0, + "total": 40, + "trueNegative": 36, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 40 + } + }, + "intentsMeeting95Precision": 6, + "macroF1": 0.577, + "macroPrecision": 0.8701, + "macroRecall": 0.4705, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 680, + "trueNegative": 680, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.3333, + "falseNegative": 32, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 8, + "recall": 0.2, + "total": 680, + "trueNegative": 640, + "truePositive": 8, + "wilsonPrecisionLower95": 0.6756 + }, + "confirmationDecision": { + "f1": 0.8718, + "falseNegative": 6, + "falsePositive": 4, + "precision": 0.8947, + "predictedPositive": 38, + "recall": 0.85, + "total": 680, + "trueNegative": 636, + "truePositive": 34, + "wilsonPrecisionLower95": 0.7587 + }, + "followUpReminder": { + "f1": 0.8491, + "falseNegative": 30, + "falsePositive": 2, + "precision": 0.9783, + "predictedPositive": 92, + "recall": 0.75, + "total": 680, + "trueNegative": 558, + "truePositive": 90, + "wilsonPrecisionLower95": 0.9242 + }, + "invitation": { + "f1": 0.5965, + "falseNegative": 23, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 17, + "recall": 0.425, + "total": 680, + "trueNegative": 640, + "truePositive": 17, + "wilsonPrecisionLower95": 0.8157 + }, + "question": { + "f1": 0.6992, + "falseNegative": 74, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 86, + "recall": 0.5375, + "total": 680, + "trueNegative": 520, + "truePositive": 86, + "wilsonPrecisionLower95": 0.9572 + }, + "replyableMessage": { + "f1": 0.9036, + "falseNegative": 58, + "falsePositive": 15, + "precision": 0.958, + "predictedPositive": 357, + "recall": 0.855, + "total": 680, + "trueNegative": 265, + "truePositive": 342, + "wilsonPrecisionLower95": 0.9318 + }, + "scheduleNegotiation": { + "f1": 0.5185, + "falseNegative": 26, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 14, + "recall": 0.35, + "total": 680, + "trueNegative": 640, + "truePositive": 14, + "wilsonPrecisionLower95": 0.7847 + }, + "task": { + "f1": 0.4211, + "falseNegative": 88, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 32, + "recall": 0.2667, + "total": 680, + "trueNegative": 560, + "truePositive": 32, + "wilsonPrecisionLower95": 0.8928 + } + }, + "recordCount": 680 + }, + "researchOnlyComprehensive": { + "byLanguage": { + "en": { + "intentsMeeting95Precision": 1, + "macroF1": 0.2884, + "macroPrecision": 0.3871, + "macroRecall": 0.2731, + "minimumPrecision": 0.0006, + "perIntent": { + "blessing": { + "f1": 0.4667, + "falseNegative": 48, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 21, + "recall": 0.3043, + "total": 13176, + "trueNegative": 13107, + "truePositive": 21, + "wilsonPrecisionLower95": 0.8454 + }, + "complaint": { + "f1": 0.048, + "falseNegative": 421, + "falsePositive": 95, + "precision": 0.1204, + "predictedPositive": 108, + "recall": 0.03, + "total": 13176, + "trueNegative": 12647, + "truePositive": 13, + "wilsonPrecisionLower95": 0.0717 + }, + "confirmationDecision": { + "f1": 0.0012, + "falseNegative": 11, + "falsePositive": 1649, + "precision": 0.0006, + "predictedPositive": 1650, + "recall": 0.0833, + "total": 13176, + "trueNegative": 11515, + "truePositive": 1, + "wilsonPrecisionLower95": 0.0001 + }, + "followUpReminder": { + "f1": 0.1134, + "falseNegative": 114, + "falsePositive": 58, + "precision": 0.1594, + "predictedPositive": 69, + "recall": 0.088, + "total": 13176, + "trueNegative": 12993, + "truePositive": 11, + "wilsonPrecisionLower95": 0.0914 + }, + "invitation": { + "f1": 0.336, + "falseNegative": 40, + "falsePositive": 43, + "precision": 0.3281, + "predictedPositive": 64, + "recall": 0.3443, + "total": 13176, + "trueNegative": 13072, + "truePositive": 21, + "wilsonPrecisionLower95": 0.2257 + }, + "question": { + "f1": 0.4288, + "falseNegative": 1828, + "falsePositive": 378, + "precision": 0.6866, + "predictedPositive": 1206, + "recall": 0.3117, + "total": 13176, + "trueNegative": 10142, + "truePositive": 828, + "wilsonPrecisionLower95": 0.6598 + }, + "replyableMessage": { + "f1": 0.7642, + "falseNegative": 2967, + "falsePositive": 1917, + "precision": 0.805, + "predictedPositive": 9831, + "recall": 0.7273, + "total": 13176, + "trueNegative": 378, + "truePositive": 7914, + "wilsonPrecisionLower95": 0.7971 + }, + "scheduleNegotiation": { + "f1": 0.1333, + "falseNegative": 50, + "falsePositive": 171, + "precision": 0.0904, + "predictedPositive": 188, + "recall": 0.2537, + "total": 13176, + "trueNegative": 12938, + "truePositive": 17, + "wilsonPrecisionLower95": 0.0572 + }, + "task": { + "f1": 0.3039, + "falseNegative": 766, + "falsePositive": 851, + "precision": 0.2932, + "predictedPositive": 1204, + "recall": 0.3155, + "total": 13176, + "trueNegative": 11206, + "truePositive": 353, + "wilsonPrecisionLower95": 0.2682 + } + }, + "recordCount": 13176 + }, + "zh-Hans": { + "intentsMeeting95Precision": 0, + "macroF1": 0.1992, + "macroPrecision": 0.4046, + "macroRecall": 0.1613, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.2105, + "falseNegative": 15, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 2, + "recall": 0.1176, + "total": 28019, + "trueNegative": 28002, + "truePositive": 2, + "wilsonPrecisionLower95": 0.3424 + }, + "complaint": { + "f1": 0.4077, + "falseNegative": 154, + "falsePositive": 183, + "precision": 0.388, + "predictedPositive": 299, + "recall": 0.4296, + "total": 28019, + "trueNegative": 27566, + "truePositive": 116, + "wilsonPrecisionLower95": 0.3345 + }, + "confirmationDecision": { + "f1": 0.0594, + "falseNegative": 54, + "falsePositive": 41, + "precision": 0.0682, + "predictedPositive": 44, + "recall": 0.0526, + "total": 28019, + "trueNegative": 27921, + "truePositive": 3, + "wilsonPrecisionLower95": 0.0235 + }, + "followUpReminder": { + "f1": 0.1856, + "falseNegative": 234, + "falsePositive": 38, + "precision": 0.4493, + "predictedPositive": 69, + "recall": 0.117, + "total": 28019, + "trueNegative": 27716, + "truePositive": 31, + "wilsonPrecisionLower95": 0.3377 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 73, + "falsePositive": 9, + "precision": 0.0, + "predictedPositive": 9, + "recall": 0.0, + "total": 28019, + "trueNegative": 27937, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "question": { + "f1": 0.1419, + "falseNegative": 6619, + "falsePositive": 216, + "precision": 0.7234, + "predictedPositive": 781, + "recall": 0.0786, + "total": 28019, + "trueNegative": 20619, + "truePositive": 565, + "wilsonPrecisionLower95": 0.691 + }, + "replyableMessage": { + "f1": 0.7622, + "falseNegative": 9480, + "falsePositive": 1096, + "precision": 0.9393, + "predictedPositive": 18042, + "recall": 0.6413, + "total": 28019, + "trueNegative": 497, + "truePositive": 16946, + "wilsonPrecisionLower95": 0.9357 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 4, + "falsePositive": 5, + "precision": 0.0, + "predictedPositive": 5, + "recall": 0.0, + "total": 28019, + "trueNegative": 28010, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "task": { + "f1": 0.0251, + "falseNegative": 649, + "falsePositive": 127, + "precision": 0.073, + "predictedPositive": 137, + "recall": 0.0152, + "total": 28019, + "trueNegative": 27233, + "truePositive": 10, + "wilsonPrecisionLower95": 0.0401 + } + }, + "recordCount": 28019 + } + }, + "bySource": { + "AmazonScience/MASSIVE en-US test": { + "intentsMeeting95Precision": 2, + "macroF1": 0.1663, + "macroPrecision": 0.2646, + "macroRecall": 0.1307, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 2967, + "trueNegative": 2967, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 17, + "precision": 0.0, + "predictedPositive": 17, + "recall": 0.0, + "total": 2967, + "trueNegative": 2950, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 441, + "precision": 0.0, + "predictedPositive": 441, + "recall": 0.0, + "total": 2967, + "trueNegative": 2526, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "followUpReminder": { + "f1": 0.1447, + "falseNegative": 86, + "falsePositive": 44, + "precision": 0.2, + "predictedPositive": 55, + "recall": 0.1134, + "total": 2967, + "trueNegative": 2826, + "truePositive": 11, + "wilsonPrecisionLower95": 0.1155 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 7, + "precision": 0.0, + "predictedPositive": 7, + "recall": 0.0, + "total": 2967, + "trueNegative": 2960, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "question": { + "f1": 0.3532, + "falseNegative": 1266, + "falsePositive": 16, + "precision": 0.9563, + "predictedPositive": 366, + "recall": 0.2166, + "total": 2967, + "trueNegative": 1335, + "truePositive": 350, + "wilsonPrecisionLower95": 0.9302 + }, + "replyableMessage": { + "f1": 0.7951, + "falseNegative": 1009, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 1958, + "recall": 0.6599, + "total": 2967, + "trueNegative": 0, + "truePositive": 1958, + "wilsonPrecisionLower95": 0.998 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 69, + "precision": 0.0, + "predictedPositive": 69, + "recall": 0.0, + "total": 2967, + "trueNegative": 2898, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "task": { + "f1": 0.204, + "falseNegative": 440, + "falsePositive": 348, + "precision": 0.2249, + "predictedPositive": 449, + "recall": 0.1867, + "total": 2967, + "trueNegative": 2078, + "truePositive": 101, + "wilsonPrecisionLower95": 0.1887 + } + }, + "recordCount": 2967 + }, + "AmazonScience/MASSIVE zh-CN test": { + "intentsMeeting95Precision": 1, + "macroF1": 0.1386, + "macroPrecision": 0.3031, + "macroRecall": 0.1008, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 2850, + "trueNegative": 2850, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 2, + "precision": 0.0, + "predictedPositive": 2, + "recall": 0.0, + "total": 2850, + "trueNegative": 2848, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 5, + "precision": 0.0, + "predictedPositive": 5, + "recall": 0.0, + "total": 2850, + "trueNegative": 2845, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "followUpReminder": { + "f1": 0.2537, + "falseNegative": 140, + "falsePositive": 13, + "precision": 0.6667, + "predictedPositive": 39, + "recall": 0.1566, + "total": 2850, + "trueNegative": 2671, + "truePositive": 26, + "wilsonPrecisionLower95": 0.5098 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 7, + "precision": 0.0, + "predictedPositive": 7, + "recall": 0.0, + "total": 2850, + "trueNegative": 2843, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "question": { + "f1": 0.2054, + "falseNegative": 1381, + "falsePositive": 19, + "precision": 0.905, + "predictedPositive": 200, + "recall": 0.1159, + "total": 2850, + "trueNegative": 1269, + "truePositive": 181, + "wilsonPrecisionLower95": 0.8564 + }, + "replyableMessage": { + "f1": 0.7684, + "falseNegative": 1072, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 1778, + "recall": 0.6239, + "total": 2850, + "trueNegative": 0, + "truePositive": 1778, + "wilsonPrecisionLower95": 0.9978 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 4, + "precision": 0.0, + "predictedPositive": 4, + "recall": 0.0, + "total": 2850, + "trueNegative": 2846, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "task": { + "f1": 0.0196, + "falseNegative": 473, + "falsePositive": 27, + "precision": 0.1562, + "predictedPositive": 32, + "recall": 0.0105, + "total": 2850, + "trueNegative": 2345, + "truePositive": 5, + "wilsonPrecisionLower95": 0.0686 + } + }, + "recordCount": 2850 + }, + "CFPB via hpe-ai/customer-complaints": { + "intentsMeeting95Precision": 1, + "macroF1": 0.0968, + "macroPrecision": 0.2685, + "macroRecall": 0.0714, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 250, + "trueNegative": 250, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0159, + "falseNegative": 248, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 2, + "recall": 0.008, + "total": 250, + "trueNegative": 0, + "truePositive": 2, + "wilsonPrecisionLower95": 0.3424 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 4, + "precision": 0.0, + "predictedPositive": 4, + "recall": 0.0, + "total": 250, + "trueNegative": 246, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 250, + "trueNegative": 250, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 250, + "trueNegative": 250, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 17, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 250, + "trueNegative": 233, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.7013, + "falseNegative": 115, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 135, + "recall": 0.54, + "total": 250, + "trueNegative": 0, + "truePositive": 135, + "wilsonPrecisionLower95": 0.9723 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 250, + "trueNegative": 250, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.1538, + "falseNegative": 48, + "falsePositive": 7, + "precision": 0.4167, + "predictedPositive": 12, + "recall": 0.0943, + "total": 250, + "trueNegative": 190, + "truePositive": 5, + "wilsonPrecisionLower95": 0.1933 + } + }, + "recordCount": 250 + }, + "CPED test split": { + "intentsMeeting95Precision": 0, + "macroF1": 0.1434, + "macroPrecision": 0.3223, + "macroRecall": 0.1091, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.2105, + "falseNegative": 15, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 2, + "recall": 0.1176, + "total": 25049, + "trueNegative": 25032, + "truePositive": 2, + "wilsonPrecisionLower95": 0.3424 + }, + "complaint": { + "f1": 0.018, + "falseNegative": 147, + "falsePositive": 181, + "precision": 0.0163, + "predictedPositive": 184, + "recall": 0.02, + "total": 25049, + "trueNegative": 24718, + "truePositive": 3, + "wilsonPrecisionLower95": 0.0056 + }, + "confirmationDecision": { + "f1": 0.0625, + "falseNegative": 54, + "falsePositive": 36, + "precision": 0.0769, + "predictedPositive": 39, + "recall": 0.0526, + "total": 25049, + "trueNegative": 24956, + "truePositive": 3, + "wilsonPrecisionLower95": 0.0265 + }, + "followUpReminder": { + "f1": 0.0775, + "falseNegative": 94, + "falsePositive": 25, + "precision": 0.1667, + "predictedPositive": 30, + "recall": 0.0505, + "total": 25049, + "trueNegative": 24925, + "truePositive": 5, + "wilsonPrecisionLower95": 0.0734 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 73, + "falsePositive": 2, + "precision": 0.0, + "predictedPositive": 2, + "recall": 0.0, + "total": 25049, + "trueNegative": 24974, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "question": { + "f1": 0.1245, + "falseNegative": 5202, + "falsePositive": 197, + "precision": 0.6609, + "predictedPositive": 581, + "recall": 0.0687, + "total": 25049, + "trueNegative": 19266, + "truePositive": 384, + "wilsonPrecisionLower95": 0.6215 + }, + "replyableMessage": { + "f1": 0.7609, + "falseNegative": 8378, + "falsePositive": 1096, + "precision": 0.9322, + "predictedPositive": 16174, + "recall": 0.6428, + "total": 25049, + "trueNegative": 497, + "truePositive": 15078, + "wilsonPrecisionLower95": 0.9283 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 4, + "falsePositive": 1, + "precision": 0.0, + "predictedPositive": 1, + "recall": 0.0, + "total": 25049, + "trueNegative": 25044, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "task": { + "f1": 0.0366, + "falseNegative": 163, + "falsePositive": 100, + "precision": 0.0476, + "predictedPositive": 105, + "recall": 0.0298, + "total": 25049, + "trueNegative": 24781, + "truePositive": 5, + "wilsonPrecisionLower95": 0.0205 + } + }, + "recordCount": 25049 + }, + "Google Research GoEmotions test": { + "intentsMeeting95Precision": 2, + "macroF1": 0.2218, + "macroPrecision": 0.3073, + "macroRecall": 0.1928, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.4651, + "falseNegative": 46, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 20, + "recall": 0.303, + "total": 5418, + "trueNegative": 5352, + "truePositive": 20, + "wilsonPrecisionLower95": 0.8389 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 20, + "falsePositive": 27, + "precision": 0.0, + "predictedPositive": 27, + "recall": 0.0, + "total": 5418, + "trueNegative": 5371, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 742, + "precision": 0.0, + "predictedPositive": 742, + "recall": 0.0, + "total": 5418, + "trueNegative": 4676, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 6, + "precision": 0.0, + "predictedPositive": 6, + "recall": 0.0, + "total": 5418, + "trueNegative": 5412, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 8, + "precision": 0.0, + "predictedPositive": 8, + "recall": 0.0, + "total": 5418, + "trueNegative": 5410, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "question": { + "f1": 0.3991, + "falseNegative": 363, + "falsePositive": 146, + "precision": 0.5365, + "predictedPositive": 315, + "recall": 0.3177, + "total": 5418, + "trueNegative": 4740, + "truePositive": 169, + "wilsonPrecisionLower95": 0.4813 + }, + "replyableMessage": { + "f1": 0.8467, + "falseNegative": 1440, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 3978, + "recall": 0.7342, + "total": 5418, + "trueNegative": 0, + "truePositive": 3978, + "wilsonPrecisionLower95": 0.999 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 23, + "precision": 0.0, + "predictedPositive": 23, + "recall": 0.0, + "total": 5418, + "trueNegative": 5395, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "task": { + "f1": 0.2857, + "falseNegative": 31, + "falsePositive": 64, + "precision": 0.2289, + "predictedPositive": 83, + "recall": 0.38, + "total": 5418, + "trueNegative": 5304, + "truePositive": 19, + "wilsonPrecisionLower95": 0.1517 + } + }, + "recordCount": 5418 + }, + "Meituan-Dianping/ASAP test": { + "intentsMeeting95Precision": 2, + "macroF1": 0.203, + "macroPrecision": 0.2222, + "macroRecall": 0.188, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 120, + "trueNegative": 120, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.97, + "falseNegative": 7, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 113, + "recall": 0.9417, + "total": 120, + "trueNegative": 0, + "truePositive": 113, + "wilsonPrecisionLower95": 0.9671 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 120, + "trueNegative": 120, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 120, + "trueNegative": 120, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 120, + "trueNegative": 120, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 36, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 120, + "trueNegative": 84, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.8571, + "falseNegative": 30, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 90, + "recall": 0.75, + "total": 120, + "trueNegative": 0, + "truePositive": 90, + "wilsonPrecisionLower95": 0.9591 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 120, + "trueNegative": 120, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 13, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 120, + "trueNegative": 107, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 120 + }, + "Sulak2020 GitHub issues multi-repo dataset": { + "intentsMeeting95Precision": 1, + "macroF1": 0.1573, + "macroPrecision": 0.2376, + "macroRecall": 0.1229, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 2, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 1004, + "trueNegative": 1002, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0995, + "falseNegative": 153, + "falsePositive": 46, + "precision": 0.193, + "predictedPositive": 57, + "recall": 0.0671, + "total": 1004, + "trueNegative": 794, + "truePositive": 11, + "wilsonPrecisionLower95": 0.1113 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 2, + "falsePositive": 84, + "precision": 0.0, + "predictedPositive": 84, + "recall": 0.0, + "total": 1004, + "trueNegative": 918, + "truePositive": 0, + "wilsonPrecisionLower95": -0.0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 2, + "falsePositive": 1, + "precision": 0.0, + "predictedPositive": 1, + "recall": 0.0, + "total": 1004, + "trueNegative": 1001, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 2, + "precision": 0.0, + "predictedPositive": 2, + "recall": 0.0, + "total": 1004, + "trueNegative": 1002, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "question": { + "f1": 0.4474, + "falseNegative": 39, + "falsePositive": 3, + "precision": 0.85, + "predictedPositive": 20, + "recall": 0.3036, + "total": 1004, + "trueNegative": 945, + "truePositive": 17, + "wilsonPrecisionLower95": 0.6396 + }, + "replyableMessage": { + "f1": 0.8326, + "falseNegative": 288, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 716, + "recall": 0.7131, + "total": 1004, + "trueNegative": 0, + "truePositive": 716, + "wilsonPrecisionLower95": 0.9947 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 25, + "falsePositive": 4, + "precision": 0.0, + "predictedPositive": 4, + "recall": 0.0, + "total": 1004, + "trueNegative": 975, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "task": { + "f1": 0.036, + "falseNegative": 88, + "falsePositive": 19, + "precision": 0.0952, + "predictedPositive": 21, + "recall": 0.0222, + "total": 1004, + "trueNegative": 895, + "truePositive": 2, + "wilsonPrecisionLower95": 0.0265 + } + }, + "recordCount": 1004 + }, + "Verified Enron Intent Dataset": { + "intentsMeeting95Precision": 0, + "macroF1": 0.3596, + "macroPrecision": 0.3263, + "macroRecall": 0.4461, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 1, + "recall": 1.0, + "total": 3537, + "trueNegative": 3536, + "truePositive": 1, + "wilsonPrecisionLower95": 0.2065 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 5, + "precision": 0.0, + "predictedPositive": 5, + "recall": 0.0, + "total": 3537, + "trueNegative": 3532, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "confirmationDecision": { + "f1": 0.0051, + "falseNegative": 9, + "falsePositive": 378, + "precision": 0.0026, + "predictedPositive": 379, + "recall": 0.1, + "total": 3537, + "trueNegative": 3149, + "truePositive": 1, + "wilsonPrecisionLower95": 0.0005 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 26, + "falsePositive": 7, + "precision": 0.0, + "predictedPositive": 7, + "recall": 0.0, + "total": 3537, + "trueNegative": 3504, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "invitation": { + "f1": 0.3889, + "falseNegative": 40, + "falsePositive": 26, + "precision": 0.4468, + "predictedPositive": 47, + "recall": 0.3443, + "total": 3537, + "trueNegative": 3450, + "truePositive": 21, + "wilsonPrecisionLower95": 0.3141 + }, + "question": { + "f1": 0.6213, + "falseNegative": 143, + "falsePositive": 213, + "precision": 0.5782, + "predictedPositive": 505, + "recall": 0.6713, + "total": 3537, + "trueNegative": 2889, + "truePositive": 292, + "wilsonPrecisionLower95": 0.5347 + }, + "replyableMessage": { + "f1": 0.5259, + "falseNegative": 115, + "falsePositive": 1917, + "precision": 0.3702, + "predictedPositive": 3044, + "recall": 0.9074, + "total": 3537, + "trueNegative": 378, + "truePositive": 1127, + "wilsonPrecisionLower95": 0.3533 + }, + "scheduleNegotiation": { + "f1": 0.2537, + "falseNegative": 25, + "falsePositive": 75, + "precision": 0.1848, + "predictedPositive": 92, + "recall": 0.4048, + "total": 3537, + "trueNegative": 3420, + "truePositive": 17, + "wilsonPrecisionLower95": 0.1187 + }, + "task": { + "f1": 0.4414, + "falseNegative": 159, + "falsePositive": 413, + "precision": 0.3537, + "predictedPositive": 639, + "recall": 0.587, + "total": 3537, + "trueNegative": 2739, + "truePositive": 226, + "wilsonPrecisionLower95": 0.3176 + } + }, + "recordCount": 3537 + } + }, + "intentsMeeting95Precision": 1, + "macroF1": 0.2645, + "macroPrecision": 0.4292, + "macroRecall": 0.225, + "minimumPrecision": 0.0024, + "perIntent": { + "blessing": { + "f1": 0.422, + "falseNegative": 63, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 23, + "recall": 0.2674, + "total": 41195, + "trueNegative": 41109, + "truePositive": 23, + "wilsonPrecisionLower95": 0.8569 + }, + "complaint": { + "f1": 0.2322, + "falseNegative": 575, + "falsePositive": 278, + "precision": 0.317, + "predictedPositive": 407, + "recall": 0.1832, + "total": 41195, + "trueNegative": 40213, + "truePositive": 129, + "wilsonPrecisionLower95": 0.2736 + }, + "confirmationDecision": { + "f1": 0.0045, + "falseNegative": 65, + "falsePositive": 1690, + "precision": 0.0024, + "predictedPositive": 1694, + "recall": 0.058, + "total": 41195, + "trueNegative": 39436, + "truePositive": 4, + "wilsonPrecisionLower95": 0.0009 + }, + "followUpReminder": { + "f1": 0.1591, + "falseNegative": 348, + "falsePositive": 96, + "precision": 0.3043, + "predictedPositive": 138, + "recall": 0.1077, + "total": 41195, + "trueNegative": 40709, + "truePositive": 42, + "wilsonPrecisionLower95": 0.2337 + }, + "invitation": { + "f1": 0.2029, + "falseNegative": 113, + "falsePositive": 52, + "precision": 0.2877, + "predictedPositive": 73, + "recall": 0.1567, + "total": 41195, + "trueNegative": 41009, + "truePositive": 21, + "wilsonPrecisionLower95": 0.1965 + }, + "question": { + "f1": 0.2356, + "falseNegative": 8447, + "falsePositive": 594, + "precision": 0.7011, + "predictedPositive": 1987, + "recall": 0.1416, + "total": 41195, + "trueNegative": 30761, + "truePositive": 1393, + "wilsonPrecisionLower95": 0.6806 + }, + "replyableMessage": { + "f1": 0.7628, + "falseNegative": 12447, + "falsePositive": 3013, + "precision": 0.8919, + "predictedPositive": 27873, + "recall": 0.6664, + "total": 41195, + "trueNegative": 875, + "truePositive": 24860, + "wilsonPrecisionLower95": 0.8882 + }, + "scheduleNegotiation": { + "f1": 0.1288, + "falseNegative": 54, + "falsePositive": 176, + "precision": 0.0881, + "predictedPositive": 193, + "recall": 0.2394, + "total": 41195, + "trueNegative": 40948, + "truePositive": 17, + "wilsonPrecisionLower95": 0.0557 + }, + "task": { + "f1": 0.2328, + "falseNegative": 1415, + "falsePositive": 978, + "precision": 0.2707, + "predictedPositive": 1341, + "recall": 0.2042, + "total": 41195, + "trueNegative": 38439, + "truePositive": 363, + "wilsonPrecisionLower95": 0.2476 + } + }, + "recordCount": 41195 + }, + "targetedRelease": { + "byLanguage": { + "en": { + "intentsMeeting95Precision": 4, + "macroF1": 0.4252, + "macroPrecision": 0.5437, + "macroRecall": 0.3679, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 320, + "trueNegative": 320, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.6957, + "falseNegative": 56, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 64, + "recall": 0.5333, + "total": 320, + "trueNegative": 200, + "truePositive": 64, + "wilsonPrecisionLower95": 0.9434 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 8, + "precision": 0.0, + "predictedPositive": 8, + "recall": 0.0, + "total": 320, + "trueNegative": 312, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "followUpReminder": { + "f1": 0.961, + "falseNegative": 3, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 37, + "recall": 0.925, + "total": 320, + "trueNegative": 280, + "truePositive": 37, + "wilsonPrecisionLower95": 0.9059 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 320, + "trueNegative": 320, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.4762, + "falseNegative": 55, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 25, + "recall": 0.3125, + "total": 320, + "trueNegative": 240, + "truePositive": 25, + "wilsonPrecisionLower95": 0.8668 + }, + "replyableMessage": { + "f1": 0.8819, + "falseNegative": 32, + "falsePositive": 13, + "precision": 0.9282, + "predictedPositive": 181, + "recall": 0.84, + "total": 320, + "trueNegative": 107, + "truePositive": 168, + "wilsonPrecisionLower95": 0.881 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 320, + "trueNegative": 320, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.8116, + "falseNegative": 24, + "falsePositive": 2, + "precision": 0.9655, + "predictedPositive": 58, + "recall": 0.7, + "total": 320, + "trueNegative": 238, + "truePositive": 56, + "wilsonPrecisionLower95": 0.8827 + } + }, + "recordCount": 320 + }, + "zh-Hans": { + "intentsMeeting95Precision": 4, + "macroF1": 0.3643, + "macroPrecision": 0.5501, + "macroRecall": 0.3071, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 320, + "trueNegative": 320, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.4516, + "falseNegative": 85, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 35, + "recall": 0.2917, + "total": 320, + "trueNegative": 200, + "truePositive": 35, + "wilsonPrecisionLower95": 0.9011 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 320, + "trueNegative": 320, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.2609, + "falseNegative": 34, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 6, + "recall": 0.15, + "total": 320, + "trueNegative": 280, + "truePositive": 6, + "wilsonPrecisionLower95": 0.6097 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 320, + "trueNegative": 320, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.8811, + "falseNegative": 17, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 63, + "recall": 0.7875, + "total": 320, + "trueNegative": 240, + "truePositive": 63, + "wilsonPrecisionLower95": 0.9425 + }, + "replyableMessage": { + "f1": 0.9552, + "falseNegative": 8, + "falsePositive": 10, + "precision": 0.9505, + "predictedPositive": 202, + "recall": 0.96, + "total": 320, + "trueNegative": 110, + "truePositive": 192, + "wilsonPrecisionLower95": 0.9113 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 320, + "trueNegative": 320, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.7302, + "falseNegative": 34, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 46, + "recall": 0.575, + "total": 320, + "trueNegative": 240, + "truePositive": 46, + "wilsonPrecisionLower95": 0.9229 + } + }, + "recordCount": 320 + } + }, + "bySource": { + "release_complaint_brief": { + "intentsMeeting95Precision": 2, + "macroF1": 0.1893, + "macroPrecision": 0.2222, + "macroRecall": 0.1708, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.7097, + "falseNegative": 36, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 44, + "recall": 0.55, + "total": 80, + "trueNegative": 0, + "truePositive": 44, + "wilsonPrecisionLower95": 0.9197 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.9937, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 79, + "recall": 0.9875, + "total": 80, + "trueNegative": 0, + "truePositive": 79, + "wilsonPrecisionLower95": 0.9536 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 80 + }, + "release_complaint_question": { + "intentsMeeting95Precision": 3, + "macroF1": 0.2152, + "macroPrecision": 0.3333, + "macroRecall": 0.1667, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.5586, + "falseNegative": 49, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 31, + "recall": 0.3875, + "total": 80, + "trueNegative": 0, + "truePositive": 31, + "wilsonPrecisionLower95": 0.8897 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.5047, + "falseNegative": 53, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 27, + "recall": 0.3375, + "total": 80, + "trueNegative": 0, + "truePositive": 27, + "wilsonPrecisionLower95": 0.8754 + }, + "replyableMessage": { + "f1": 0.8732, + "falseNegative": 18, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 62, + "recall": 0.775, + "total": 80, + "trueNegative": 0, + "truePositive": 62, + "wilsonPrecisionLower95": 0.9417 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 80 + }, + "release_complaint_unspoken": { + "intentsMeeting95Precision": 2, + "macroF1": 0.1565, + "macroPrecision": 0.2222, + "macroRecall": 0.1333, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.4615, + "falseNegative": 56, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 24, + "recall": 0.3, + "total": 80, + "trueNegative": 0, + "truePositive": 24, + "wilsonPrecisionLower95": 0.862 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.9474, + "falseNegative": 8, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 72, + "recall": 0.9, + "total": 80, + "trueNegative": 0, + "truePositive": 72, + "wilsonPrecisionLower95": 0.9493 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 80 + }, + "release_negative_boundary": { + "intentsMeeting95Precision": 0, + "macroF1": 0.0, + "macroPrecision": 0.0, + "macroRecall": 0.0, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 80 + }, + "release_personal_boundary": { + "intentsMeeting95Precision": 1, + "macroF1": 0.0777, + "macroPrecision": 0.1111, + "macroRecall": 0.0597, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.6992, + "falseNegative": 37, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 43, + "recall": 0.5375, + "total": 80, + "trueNegative": 0, + "truePositive": 43, + "wilsonPrecisionLower95": 0.918 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 4, + "precision": 0.0, + "predictedPositive": 4, + "recall": 0.0, + "total": 80, + "trueNegative": 76, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 80 + }, + "release_resolved_boundary": { + "intentsMeeting95Precision": 0, + "macroF1": 0.0, + "macroPrecision": 0.0, + "macroRecall": 0.0, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 19, + "precision": 0.0, + "predictedPositive": 19, + "recall": 0.0, + "total": 80, + "trueNegative": 61, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 2, + "precision": 0.0, + "predictedPositive": 2, + "recall": 0.0, + "total": 80, + "trueNegative": 78, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + } + }, + "recordCount": 80 + }, + "release_task_owner": { + "intentsMeeting95Precision": 2, + "macroF1": 0.2, + "macroPrecision": 0.2222, + "macroRecall": 0.1847, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 8, + "precision": 0.0, + "predictedPositive": 8, + "recall": 0.0, + "total": 80, + "trueNegative": 72, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.9937, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 79, + "recall": 0.9875, + "total": 80, + "trueNegative": 0, + "truePositive": 79, + "wilsonPrecisionLower95": 0.9536 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.806, + "falseNegative": 26, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 54, + "recall": 0.675, + "total": 80, + "trueNegative": 0, + "truePositive": 54, + "wilsonPrecisionLower95": 0.9336 + } + }, + "recordCount": 80 + }, + "release_task_polite_request": { + "intentsMeeting95Precision": 3, + "macroF1": 0.2816, + "macroPrecision": 0.3333, + "macroRecall": 0.2458, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.8652, + "falseNegative": 19, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 61, + "recall": 0.7625, + "total": 80, + "trueNegative": 0, + "truePositive": 61, + "wilsonPrecisionLower95": 0.9408 + }, + "replyableMessage": { + "f1": 0.9189, + "falseNegative": 12, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 68, + "recall": 0.85, + "total": 80, + "trueNegative": 0, + "truePositive": 68, + "wilsonPrecisionLower95": 0.9465 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.75, + "falseNegative": 32, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 48, + "recall": 0.6, + "total": 80, + "trueNegative": 0, + "truePositive": 48, + "wilsonPrecisionLower95": 0.9259 + } + }, + "recordCount": 80 + } + }, + "intentsMeeting95Precision": 4, + "macroF1": 0.4095, + "macroPrecision": 0.5467, + "macroRecall": 0.3375, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 640, + "trueNegative": 640, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.5841, + "falseNegative": 141, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 99, + "recall": 0.4125, + "total": 640, + "trueNegative": 400, + "truePositive": 99, + "wilsonPrecisionLower95": 0.9626 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 8, + "precision": 0.0, + "predictedPositive": 8, + "recall": 0.0, + "total": 640, + "trueNegative": 632, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "followUpReminder": { + "f1": 0.6992, + "falseNegative": 37, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 43, + "recall": 0.5375, + "total": 640, + "trueNegative": 560, + "truePositive": 43, + "wilsonPrecisionLower95": 0.918 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 640, + "trueNegative": 640, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.7097, + "falseNegative": 72, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 88, + "recall": 0.55, + "total": 640, + "trueNegative": 480, + "truePositive": 88, + "wilsonPrecisionLower95": 0.9582 + }, + "replyableMessage": { + "f1": 0.9195, + "falseNegative": 40, + "falsePositive": 23, + "precision": 0.9399, + "predictedPositive": 383, + "recall": 0.9, + "total": 640, + "trueNegative": 217, + "truePositive": 360, + "wilsonPrecisionLower95": 0.9115 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 640, + "trueNegative": 640, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.7727, + "falseNegative": 58, + "falsePositive": 2, + "precision": 0.9808, + "predictedPositive": 104, + "recall": 0.6375, + "total": 640, + "trueNegative": 478, + "truePositive": 102, + "wilsonPrecisionLower95": 0.9326 + } + }, + "recordCount": 640 + } + }, + "generatedAtUnix": 1787825777, + "openTrainingRecords": 3845, + "overlapChecks": { + "ModelTraining/ClipboardSemantics/comprehensive-online-holdout-corpus.jsonl": 0, + "ModelTraining/ClipboardSemantics/random-holdout-corpus.jsonl": 0, + "ModelTraining/ClipboardSemantics/targeted-release-holdout-corpus.jsonl": 0 + }, + "platformLimitation": "Linux surrogate research only. Apple Create ML/Core ML artifacts must be retrained and tested on macOS before deployment.", + "releaseGatePassed": false, + "roundCount": 20, + "schemaVersion": 1, + "seed": 20260827, + "selectedConfiguration": { + "alpha": 2e-05, + "augmentation": "none", + "char_max": 5, + "char_min": 2, + "external_weight": 0.5700000000000001, + "hard_example_weight": 1.0, + "l1_ratio": 0.0, + "max_char_features": 45000, + "max_word_features": 22000, + "min_df": 2, + "pseudo_weight": 0.16499999999999998, + "round": 1, + "word_max": 2 + }, + "selectedRound": 1, + "selectedRoundsByIntent": { + "blessing": 19, + "complaint": 1, + "confirmationDecision": 3, + "followUpReminder": 1, + "invitation": 1, + "question": 1, + "replyableMessage": 1, + "scheduleNegotiation": 7, + "task": 15 + }, + "selectedThresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 180, + "recall": 1.0, + "total": 1485, + "trueNegative": 1305, + "truePositive": 180, + "wilsonPrecisionLower95": 0.9791 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9526, + "falseNegative": 9, + "falsePositive": 8, + "precision": 0.9553, + "predictedPositive": 179, + "recall": 0.95, + "total": 1485, + "trueNegative": 1297, + "truePositive": 171, + "wilsonPrecisionLower95": 0.9143 + }, + "threshold": 0.54 + } + }, + "metrics": { + "f1": 0.9764, + "falseNegative": 9, + "falsePositive": 8, + "precision": 0.9777, + "predictedPositive": 359, + "recall": 0.975, + "total": 2970, + "trueNegative": 2602, + "truePositive": 351, + "wilsonPrecisionLower95": 0.9567 + }, + "threshold": 0.54 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9727, + "falseNegative": 1, + "falsePositive": 4, + "precision": 0.957, + "predictedPositive": 93, + "recall": 0.9889, + "total": 1485, + "trueNegative": 1391, + "truePositive": 89, + "wilsonPrecisionLower95": 0.8946 + }, + "threshold": 0.585 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7815, + "falseNegative": 31, + "falsePositive": 2, + "precision": 0.9672, + "predictedPositive": 61, + "recall": 0.6556, + "total": 1485, + "trueNegative": 1393, + "truePositive": 59, + "wilsonPrecisionLower95": 0.8881 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.8701, + "falseNegative": 36, + "falsePositive": 7, + "precision": 0.9536, + "predictedPositive": 151, + "recall": 0.8, + "total": 2970, + "trueNegative": 2783, + "truePositive": 144, + "wilsonPrecisionLower95": 0.9074 + }, + "threshold": 0.555 + }, + "followUpReminder": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8554, + "falseNegative": 41, + "falsePositive": 6, + "precision": 0.9586, + "predictedPositive": 145, + "recall": 0.7722, + "total": 1485, + "trueNegative": 1299, + "truePositive": 139, + "wilsonPrecisionLower95": 0.9127 + }, + "threshold": 0.83 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8354, + "falseNegative": 48, + "falsePositive": 4, + "precision": 0.9706, + "predictedPositive": 136, + "recall": 0.7333, + "total": 1485, + "trueNegative": 1301, + "truePositive": 132, + "wilsonPrecisionLower95": 0.9268 + }, + "threshold": 0.505 + } + }, + "metrics": { + "f1": 0.7994, + "falseNegative": 113, + "falsePositive": 11, + "precision": 0.9574, + "predictedPositive": 258, + "recall": 0.6861, + "total": 2970, + "trueNegative": 2599, + "truePositive": 247, + "wilsonPrecisionLower95": 0.9253 + }, + "threshold": 0.815 + }, + "invitation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8302, + "falseNegative": 24, + "falsePositive": 3, + "precision": 0.9565, + "predictedPositive": 69, + "recall": 0.7333, + "total": 1485, + "trueNegative": 1392, + "truePositive": 66, + "wilsonPrecisionLower95": 0.8798 + }, + "threshold": 0.935 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6765, + "falseNegative": 44, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 46, + "recall": 0.5111, + "total": 1485, + "trueNegative": 1395, + "truePositive": 46, + "wilsonPrecisionLower95": 0.9229 + }, + "threshold": 0.505 + } + }, + "metrics": { + "f1": 0.5476, + "falseNegative": 111, + "falsePositive": 3, + "precision": 0.9583, + "predictedPositive": 72, + "recall": 0.3833, + "total": 2970, + "trueNegative": 2787, + "truePositive": 69, + "wilsonPrecisionLower95": 0.8845 + }, + "threshold": 0.935 + }, + "question": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8445, + "falseNegative": 76, + "falsePositive": 12, + "precision": 0.9522, + "predictedPositive": 251, + "recall": 0.7587, + "total": 1485, + "trueNegative": 1158, + "truePositive": 239, + "wilsonPrecisionLower95": 0.9183 + }, + "threshold": 0.725 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8536, + "falseNegative": 76, + "falsePositive": 6, + "precision": 0.9755, + "predictedPositive": 245, + "recall": 0.7587, + "total": 1485, + "trueNegative": 1164, + "truePositive": 239, + "wilsonPrecisionLower95": 0.9476 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.8729, + "falseNegative": 122, + "falsePositive": 26, + "precision": 0.9513, + "predictedPositive": 534, + "recall": 0.8063, + "total": 2970, + "trueNegative": 2314, + "truePositive": 508, + "wilsonPrecisionLower95": 0.9296 + }, + "threshold": 0.515 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.948, + "falseNegative": 52, + "falsePositive": 46, + "precision": 0.951, + "predictedPositive": 939, + "recall": 0.945, + "total": 1485, + "trueNegative": 494, + "truePositive": 893, + "wilsonPrecisionLower95": 0.9353 + }, + "threshold": 0.7 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9424, + "falseNegative": 62, + "falsePositive": 46, + "precision": 0.9505, + "predictedPositive": 929, + "recall": 0.9344, + "total": 1485, + "trueNegative": 494, + "truePositive": 883, + "wilsonPrecisionLower95": 0.9346 + }, + "threshold": 0.695 + } + }, + "metrics": { + "f1": 0.9454, + "falseNegative": 115, + "falsePositive": 90, + "precision": 0.9517, + "predictedPositive": 1865, + "recall": 0.9392, + "total": 2970, + "trueNegative": 990, + "truePositive": 1775, + "wilsonPrecisionLower95": 0.9411 + }, + "threshold": 0.7 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 45, + "recall": 1.0, + "total": 1485, + "trueNegative": 1440, + "truePositive": 45, + "wilsonPrecisionLower95": 0.9213 + }, + "threshold": 0.66 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 45, + "recall": 1.0, + "total": 1485, + "trueNegative": 1440, + "truePositive": 45, + "wilsonPrecisionLower95": 0.9213 + }, + "threshold": 0.55 + } + }, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 90, + "recall": 1.0, + "total": 2970, + "trueNegative": 2880, + "truePositive": 90, + "wilsonPrecisionLower95": 0.9591 + }, + "threshold": 0.66 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8593, + "falseNegative": 79, + "falsePositive": 13, + "precision": 0.9558, + "predictedPositive": 294, + "recall": 0.7806, + "total": 1485, + "trueNegative": 1112, + "truePositive": 281, + "wilsonPrecisionLower95": 0.9258 + }, + "threshold": 0.98 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8166, + "falseNegative": 104, + "falsePositive": 11, + "precision": 0.9588, + "predictedPositive": 267, + "recall": 0.7111, + "total": 1485, + "trueNegative": 1114, + "truePositive": 256, + "wilsonPrecisionLower95": 0.9277 + }, + "threshold": 0.96 + } + }, + "metrics": { + "f1": 0.8434, + "falseNegative": 176, + "falsePositive": 26, + "precision": 0.9544, + "predictedPositive": 570, + "recall": 0.7556, + "total": 2970, + "trueNegative": 2224, + "truePositive": 544, + "wilsonPrecisionLower95": 0.934 + }, + "threshold": 0.965 + } + }, + "selectionObjective": 0.8669, + "trainingPolicy": "No user clipboard data. Official partial labels plus deterministic label-preserving augmentation and three-round high-confidence self-training consensus. Frozen holdouts never enter training or threshold calibration." +} diff --git a/ModelTraining/ClipboardSemantics/IterativeResearch/PerIntentFineTune/macos-replay.json b/ModelTraining/ClipboardSemantics/IterativeResearch/PerIntentFineTune/macos-replay.json new file mode 100644 index 0000000..6391d06 --- /dev/null +++ b/ModelTraining/ClipboardSemantics/IterativeResearch/PerIntentFineTune/macos-replay.json @@ -0,0 +1,629 @@ +{ + "automaticPromotionAllowed": false, + "baseCorpus": "ModelTraining/ClipboardSemantics/clipboard_semantic_corpus.jsonl", + "configuration": { + "alpha": 2e-05, + "augmentation": "none", + "char_max": 5, + "char_min": 2, + "external_weight": 0.5700000000000001, + "hard_example_weight": 1.0, + "l1_ratio": 0.0, + "max_char_features": 45000, + "max_word_features": 22000, + "min_df": 2, + "pseudo_weight": 0.16499999999999998, + "round": 1, + "word_max": 2 + }, + "configurationsByIntent": { + "blessing": { + "alpha": 1.2e-06, + "augmentation": "numeric", + "char_max": 8, + "char_min": 3, + "external_weight": 0.93, + "hard_example_weight": 2.7, + "l1_ratio": 0.25, + "max_char_features": 95000, + "max_word_features": 45000, + "min_df": 1, + "pseudo_weight": 0.43499999999999994, + "round": 19, + "word_max": 3 + }, + "complaint": { + "alpha": 2e-05, + "augmentation": "none", + "char_max": 5, + "char_min": 2, + "external_weight": 0.5700000000000001, + "hard_example_weight": 1.0, + "l1_ratio": 0.0, + "max_char_features": 45000, + "max_word_features": 22000, + "min_df": 2, + "pseudo_weight": 0.16499999999999998, + "round": 1, + "word_max": 2 + }, + "confirmationDecision": { + "alpha": 1e-05, + "augmentation": "none", + "char_max": 6, + "char_min": 2, + "external_weight": 0.6100000000000001, + "hard_example_weight": 1.2, + "l1_ratio": 0.05, + "max_char_features": 55000, + "max_word_features": 25000, + "min_df": 2, + "pseudo_weight": 0.195, + "round": 3, + "word_max": 2 + }, + "followUpReminder": { + "alpha": 2e-05, + "augmentation": "none", + "char_max": 5, + "char_min": 2, + "external_weight": 0.5700000000000001, + "hard_example_weight": 1.0, + "l1_ratio": 0.0, + "max_char_features": 45000, + "max_word_features": 22000, + "min_df": 2, + "pseudo_weight": 0.16499999999999998, + "round": 1, + "word_max": 2 + }, + "invitation": { + "alpha": 2e-05, + "augmentation": "none", + "char_max": 5, + "char_min": 2, + "external_weight": 0.5700000000000001, + "hard_example_weight": 1.0, + "l1_ratio": 0.0, + "max_char_features": 45000, + "max_word_features": 22000, + "min_df": 2, + "pseudo_weight": 0.16499999999999998, + "round": 1, + "word_max": 2 + }, + "question": { + "alpha": 2e-05, + "augmentation": "none", + "char_max": 5, + "char_min": 2, + "external_weight": 0.5700000000000001, + "hard_example_weight": 1.0, + "l1_ratio": 0.0, + "max_char_features": 45000, + "max_word_features": 22000, + "min_df": 2, + "pseudo_weight": 0.16499999999999998, + "round": 1, + "word_max": 2 + }, + "replyableMessage": { + "alpha": 2e-05, + "augmentation": "none", + "char_max": 5, + "char_min": 2, + "external_weight": 0.5700000000000001, + "hard_example_weight": 1.0, + "l1_ratio": 0.0, + "max_char_features": 45000, + "max_word_features": 22000, + "min_df": 2, + "pseudo_weight": 0.16499999999999998, + "round": 1, + "word_max": 2 + }, + "scheduleNegotiation": { + "alpha": 6e-06, + "augmentation": "punctuation", + "char_max": 6, + "char_min": 3, + "external_weight": 0.6900000000000001, + "hard_example_weight": 1.5, + "l1_ratio": 0.1, + "max_char_features": 65000, + "max_word_features": 32000, + "min_df": 2, + "pseudo_weight": 0.255, + "round": 7, + "word_max": 3 + }, + "task": { + "alpha": 3.5e-06, + "augmentation": "prefix", + "char_max": 6, + "char_min": 2, + "external_weight": 0.8500000000000001, + "hard_example_weight": 2.2, + "l1_ratio": 0.2, + "max_char_features": 85000, + "max_word_features": 42000, + "min_df": 1, + "pseudo_weight": 0.375, + "round": 15, + "word_max": 3 + } + }, + "openCorpus": "ModelTraining/ClipboardSemantics/open-training-corpus.jsonl", + "reason": "Surrogate feature weights are not deployable NLModel assets and cannot bypass the existing macOS acceptance policy.", + "requiredCommands": [ + "python3 Scripts/clipboard_semantics/generate_open_training_corpus.py", + "xcrun swift Scripts/clipboard_semantics/train_models.swift --algorithms maxEnt --corpus ModelTraining/ClipboardSemantics/combined-training-corpus.jsonl", + "xcrun swift Scripts/clipboard_semantics/evaluate_random_holdout.swift --corpus ModelTraining/ClipboardSemantics/random-holdout-corpus.jsonl" + ], + "schemaVersion": 1, + "seed": 20260827, + "selectedRound": 1, + "selectedRoundsByIntent": { + "blessing": 19, + "complaint": 1, + "confirmationDecision": 3, + "followUpReminder": 1, + "invitation": 1, + "question": 1, + "replyableMessage": 1, + "scheduleNegotiation": 7, + "task": 15 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 180, + "recall": 1.0, + "total": 1485, + "trueNegative": 1305, + "truePositive": 180, + "wilsonPrecisionLower95": 0.9791 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9526, + "falseNegative": 9, + "falsePositive": 8, + "precision": 0.9553, + "predictedPositive": 179, + "recall": 0.95, + "total": 1485, + "trueNegative": 1297, + "truePositive": 171, + "wilsonPrecisionLower95": 0.9143 + }, + "threshold": 0.54 + } + }, + "metrics": { + "f1": 0.9764, + "falseNegative": 9, + "falsePositive": 8, + "precision": 0.9777, + "predictedPositive": 359, + "recall": 0.975, + "total": 2970, + "trueNegative": 2602, + "truePositive": 351, + "wilsonPrecisionLower95": 0.9567 + }, + "threshold": 0.54 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9727, + "falseNegative": 1, + "falsePositive": 4, + "precision": 0.957, + "predictedPositive": 93, + "recall": 0.9889, + "total": 1485, + "trueNegative": 1391, + "truePositive": 89, + "wilsonPrecisionLower95": 0.8946 + }, + "threshold": 0.585 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7815, + "falseNegative": 31, + "falsePositive": 2, + "precision": 0.9672, + "predictedPositive": 61, + "recall": 0.6556, + "total": 1485, + "trueNegative": 1393, + "truePositive": 59, + "wilsonPrecisionLower95": 0.8881 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.8701, + "falseNegative": 36, + "falsePositive": 7, + "precision": 0.9536, + "predictedPositive": 151, + "recall": 0.8, + "total": 2970, + "trueNegative": 2783, + "truePositive": 144, + "wilsonPrecisionLower95": 0.9074 + }, + "threshold": 0.555 + }, + "followUpReminder": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8554, + "falseNegative": 41, + "falsePositive": 6, + "precision": 0.9586, + "predictedPositive": 145, + "recall": 0.7722, + "total": 1485, + "trueNegative": 1299, + "truePositive": 139, + "wilsonPrecisionLower95": 0.9127 + }, + "threshold": 0.83 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8354, + "falseNegative": 48, + "falsePositive": 4, + "precision": 0.9706, + "predictedPositive": 136, + "recall": 0.7333, + "total": 1485, + "trueNegative": 1301, + "truePositive": 132, + "wilsonPrecisionLower95": 0.9268 + }, + "threshold": 0.505 + } + }, + "metrics": { + "f1": 0.7994, + "falseNegative": 113, + "falsePositive": 11, + "precision": 0.9574, + "predictedPositive": 258, + "recall": 0.6861, + "total": 2970, + "trueNegative": 2599, + "truePositive": 247, + "wilsonPrecisionLower95": 0.9253 + }, + "threshold": 0.815 + }, + "invitation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8302, + "falseNegative": 24, + "falsePositive": 3, + "precision": 0.9565, + "predictedPositive": 69, + "recall": 0.7333, + "total": 1485, + "trueNegative": 1392, + "truePositive": 66, + "wilsonPrecisionLower95": 0.8798 + }, + "threshold": 0.935 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6765, + "falseNegative": 44, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 46, + "recall": 0.5111, + "total": 1485, + "trueNegative": 1395, + "truePositive": 46, + "wilsonPrecisionLower95": 0.9229 + }, + "threshold": 0.505 + } + }, + "metrics": { + "f1": 0.5476, + "falseNegative": 111, + "falsePositive": 3, + "precision": 0.9583, + "predictedPositive": 72, + "recall": 0.3833, + "total": 2970, + "trueNegative": 2787, + "truePositive": 69, + "wilsonPrecisionLower95": 0.8845 + }, + "threshold": 0.935 + }, + "question": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8445, + "falseNegative": 76, + "falsePositive": 12, + "precision": 0.9522, + "predictedPositive": 251, + "recall": 0.7587, + "total": 1485, + "trueNegative": 1158, + "truePositive": 239, + "wilsonPrecisionLower95": 0.9183 + }, + "threshold": 0.725 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8536, + "falseNegative": 76, + "falsePositive": 6, + "precision": 0.9755, + "predictedPositive": 245, + "recall": 0.7587, + "total": 1485, + "trueNegative": 1164, + "truePositive": 239, + "wilsonPrecisionLower95": 0.9476 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.8729, + "falseNegative": 122, + "falsePositive": 26, + "precision": 0.9513, + "predictedPositive": 534, + "recall": 0.8063, + "total": 2970, + "trueNegative": 2314, + "truePositive": 508, + "wilsonPrecisionLower95": 0.9296 + }, + "threshold": 0.515 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.948, + "falseNegative": 52, + "falsePositive": 46, + "precision": 0.951, + "predictedPositive": 939, + "recall": 0.945, + "total": 1485, + "trueNegative": 494, + "truePositive": 893, + "wilsonPrecisionLower95": 0.9353 + }, + "threshold": 0.7 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9424, + "falseNegative": 62, + "falsePositive": 46, + "precision": 0.9505, + "predictedPositive": 929, + "recall": 0.9344, + "total": 1485, + "trueNegative": 494, + "truePositive": 883, + "wilsonPrecisionLower95": 0.9346 + }, + "threshold": 0.695 + } + }, + "metrics": { + "f1": 0.9454, + "falseNegative": 115, + "falsePositive": 90, + "precision": 0.9517, + "predictedPositive": 1865, + "recall": 0.9392, + "total": 2970, + "trueNegative": 990, + "truePositive": 1775, + "wilsonPrecisionLower95": 0.9411 + }, + "threshold": 0.7 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 45, + "recall": 1.0, + "total": 1485, + "trueNegative": 1440, + "truePositive": 45, + "wilsonPrecisionLower95": 0.9213 + }, + "threshold": 0.66 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 45, + "recall": 1.0, + "total": 1485, + "trueNegative": 1440, + "truePositive": 45, + "wilsonPrecisionLower95": 0.9213 + }, + "threshold": 0.55 + } + }, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 90, + "recall": 1.0, + "total": 2970, + "trueNegative": 2880, + "truePositive": 90, + "wilsonPrecisionLower95": 0.9591 + }, + "threshold": 0.66 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8593, + "falseNegative": 79, + "falsePositive": 13, + "precision": 0.9558, + "predictedPositive": 294, + "recall": 0.7806, + "total": 1485, + "trueNegative": 1112, + "truePositive": 281, + "wilsonPrecisionLower95": 0.9258 + }, + "threshold": 0.98 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8166, + "falseNegative": 104, + "falsePositive": 11, + "precision": 0.9588, + "predictedPositive": 267, + "recall": 0.7111, + "total": 1485, + "trueNegative": 1114, + "truePositive": 256, + "wilsonPrecisionLower95": 0.9277 + }, + "threshold": 0.96 + } + }, + "metrics": { + "f1": 0.8434, + "falseNegative": 176, + "falsePositive": 26, + "precision": 0.9544, + "predictedPositive": 570, + "recall": 0.7556, + "total": 2970, + "trueNegative": 2224, + "truePositive": 544, + "wilsonPrecisionLower95": 0.934 + }, + "threshold": 0.965 + } + } +} diff --git a/ModelTraining/ClipboardSemantics/IterativeResearch/PerIntentFineTune/rounds.json b/ModelTraining/ClipboardSemantics/IterativeResearch/PerIntentFineTune/rounds.json new file mode 100644 index 0000000..f64dde3 --- /dev/null +++ b/ModelTraining/ClipboardSemantics/IterativeResearch/PerIntentFineTune/rounds.json @@ -0,0 +1,14831 @@ +[ + { + "configuration": { + "alpha": 2e-05, + "augmentation": "none", + "char_max": 5, + "char_min": 2, + "external_weight": 0.5700000000000001, + "hard_example_weight": 1.0, + "l1_ratio": 0.0, + "max_char_features": 45000, + "max_word_features": 22000, + "min_df": 2, + "pseudo_weight": 0.16499999999999998, + "round": 1, + "word_max": 2 + }, + "featureCount": 56451, + "pseudoLabelCounts": {}, + "round": 1, + "selectionObjective": 0.8669, + "syntheticRegression": { + "intentsMeeting95Precision": 6, + "macroF1": 0.8088, + "macroPrecision": 0.9661, + "macroRecall": 0.7541, + "minimumPrecision": 0.8589, + "perIntent": { + "blessing": { + "f1": 0.125, + "falseNegative": 84, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 6, + "recall": 0.0667, + "total": 2970, + "trueNegative": 2880, + "truePositive": 6, + "wilsonPrecisionLower95": 0.6097 + }, + "complaint": { + "f1": 0.9773, + "falseNegative": 16, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 344, + "recall": 0.9556, + "total": 2970, + "trueNegative": 2610, + "truePositive": 344, + "wilsonPrecisionLower95": 0.989 + }, + "confirmationDecision": { + "f1": 0.8163, + "falseNegative": 40, + "falsePositive": 23, + "precision": 0.8589, + "predictedPositive": 163, + "recall": 0.7778, + "total": 2970, + "trueNegative": 2767, + "truePositive": 140, + "wilsonPrecisionLower95": 0.7972 + }, + "followUpReminder": { + "f1": 0.8705, + "falseNegative": 81, + "falsePositive": 2, + "precision": 0.9929, + "predictedPositive": 281, + "recall": 0.775, + "total": 2970, + "trueNegative": 2608, + "truePositive": 279, + "wilsonPrecisionLower95": 0.9744 + }, + "invitation": { + "f1": 0.8545, + "falseNegative": 42, + "falsePositive": 5, + "precision": 0.965, + "predictedPositive": 143, + "recall": 0.7667, + "total": 2970, + "trueNegative": 2785, + "truePositive": 138, + "wilsonPrecisionLower95": 0.9208 + }, + "question": { + "f1": 0.9092, + "falseNegative": 104, + "falsePositive": 1, + "precision": 0.9981, + "predictedPositive": 527, + "recall": 0.8349, + "total": 2970, + "trueNegative": 2339, + "truePositive": 526, + "wilsonPrecisionLower95": 0.9893 + }, + "replyableMessage": { + "f1": 0.9465, + "falseNegative": 156, + "falsePositive": 40, + "precision": 0.9775, + "predictedPositive": 1774, + "recall": 0.9175, + "total": 2970, + "trueNegative": 1040, + "truePositive": 1734, + "wilsonPrecisionLower95": 0.9694 + }, + "scheduleNegotiation": { + "f1": 0.9727, + "falseNegative": 1, + "falsePositive": 4, + "precision": 0.957, + "predictedPositive": 93, + "recall": 0.9889, + "total": 2970, + "trueNegative": 2876, + "truePositive": 89, + "wilsonPrecisionLower95": 0.8946 + }, + "task": { + "f1": 0.8073, + "falseNegative": 213, + "falsePositive": 29, + "precision": 0.9459, + "predictedPositive": 536, + "recall": 0.7042, + "total": 2970, + "trueNegative": 2221, + "truePositive": 507, + "wilsonPrecisionLower95": 0.9234 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9318, + "falseNegative": 4, + "falsePositive": 2, + "precision": 0.9535, + "predictedPositive": 43, + "recall": 0.9111, + "total": 1485, + "trueNegative": 1438, + "truePositive": 41, + "wilsonPrecisionLower95": 0.8454 + }, + "threshold": 0.915 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7568, + "falseNegative": 17, + "falsePositive": 1, + "precision": 0.9655, + "predictedPositive": 29, + "recall": 0.6222, + "total": 1485, + "trueNegative": 1439, + "truePositive": 28, + "wilsonPrecisionLower95": 0.8282 + }, + "threshold": 0.9 + } + }, + "metrics": { + "f1": 0.8375, + "falseNegative": 23, + "falsePositive": 3, + "precision": 0.9571, + "predictedPositive": 70, + "recall": 0.7444, + "total": 2970, + "trueNegative": 2877, + "truePositive": 67, + "wilsonPrecisionLower95": 0.8814 + }, + "threshold": 0.915 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 180, + "recall": 1.0, + "total": 1485, + "trueNegative": 1305, + "truePositive": 180, + "wilsonPrecisionLower95": 0.9791 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9526, + "falseNegative": 9, + "falsePositive": 8, + "precision": 0.9553, + "predictedPositive": 179, + "recall": 0.95, + "total": 1485, + "trueNegative": 1297, + "truePositive": 171, + "wilsonPrecisionLower95": 0.9143 + }, + "threshold": 0.54 + } + }, + "metrics": { + "f1": 0.9764, + "falseNegative": 9, + "falsePositive": 8, + "precision": 0.9777, + "predictedPositive": 359, + "recall": 0.975, + "total": 2970, + "trueNegative": 2602, + "truePositive": 351, + "wilsonPrecisionLower95": 0.9567 + }, + "threshold": 0.54 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8848, + "falseNegative": 17, + "falsePositive": 2, + "precision": 0.9733, + "predictedPositive": 75, + "recall": 0.8111, + "total": 1485, + "trueNegative": 1393, + "truePositive": 73, + "wilsonPrecisionLower95": 0.9079 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7947, + "falseNegative": 30, + "falsePositive": 1, + "precision": 0.9836, + "predictedPositive": 61, + "recall": 0.6667, + "total": 1485, + "trueNegative": 1394, + "truePositive": 60, + "wilsonPrecisionLower95": 0.9128 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.8418, + "falseNegative": 47, + "falsePositive": 3, + "precision": 0.9779, + "predictedPositive": 136, + "recall": 0.7389, + "total": 2970, + "trueNegative": 2787, + "truePositive": 133, + "wilsonPrecisionLower95": 0.9372 + }, + "threshold": 0.5 + }, + "followUpReminder": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8554, + "falseNegative": 41, + "falsePositive": 6, + "precision": 0.9586, + "predictedPositive": 145, + "recall": 0.7722, + "total": 1485, + "trueNegative": 1299, + "truePositive": 139, + "wilsonPrecisionLower95": 0.9127 + }, + "threshold": 0.83 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8354, + "falseNegative": 48, + "falsePositive": 4, + "precision": 0.9706, + "predictedPositive": 136, + "recall": 0.7333, + "total": 1485, + "trueNegative": 1301, + "truePositive": 132, + "wilsonPrecisionLower95": 0.9268 + }, + "threshold": 0.505 + } + }, + "metrics": { + "f1": 0.7994, + "falseNegative": 113, + "falsePositive": 11, + "precision": 0.9574, + "predictedPositive": 258, + "recall": 0.6861, + "total": 2970, + "trueNegative": 2599, + "truePositive": 247, + "wilsonPrecisionLower95": 0.9253 + }, + "threshold": 0.815 + }, + "invitation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8302, + "falseNegative": 24, + "falsePositive": 3, + "precision": 0.9565, + "predictedPositive": 69, + "recall": 0.7333, + "total": 1485, + "trueNegative": 1392, + "truePositive": 66, + "wilsonPrecisionLower95": 0.8798 + }, + "threshold": 0.935 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6765, + "falseNegative": 44, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 46, + "recall": 0.5111, + "total": 1485, + "trueNegative": 1395, + "truePositive": 46, + "wilsonPrecisionLower95": 0.9229 + }, + "threshold": 0.505 + } + }, + "metrics": { + "f1": 0.5476, + "falseNegative": 111, + "falsePositive": 3, + "precision": 0.9583, + "predictedPositive": 72, + "recall": 0.3833, + "total": 2970, + "trueNegative": 2787, + "truePositive": 69, + "wilsonPrecisionLower95": 0.8845 + }, + "threshold": 0.935 + }, + "question": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8445, + "falseNegative": 76, + "falsePositive": 12, + "precision": 0.9522, + "predictedPositive": 251, + "recall": 0.7587, + "total": 1485, + "trueNegative": 1158, + "truePositive": 239, + "wilsonPrecisionLower95": 0.9183 + }, + "threshold": 0.725 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8536, + "falseNegative": 76, + "falsePositive": 6, + "precision": 0.9755, + "predictedPositive": 245, + "recall": 0.7587, + "total": 1485, + "trueNegative": 1164, + "truePositive": 239, + "wilsonPrecisionLower95": 0.9476 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.8729, + "falseNegative": 122, + "falsePositive": 26, + "precision": 0.9513, + "predictedPositive": 534, + "recall": 0.8063, + "total": 2970, + "trueNegative": 2314, + "truePositive": 508, + "wilsonPrecisionLower95": 0.9296 + }, + "threshold": 0.515 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.948, + "falseNegative": 52, + "falsePositive": 46, + "precision": 0.951, + "predictedPositive": 939, + "recall": 0.945, + "total": 1485, + "trueNegative": 494, + "truePositive": 893, + "wilsonPrecisionLower95": 0.9353 + }, + "threshold": 0.7 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9424, + "falseNegative": 62, + "falsePositive": 46, + "precision": 0.9505, + "predictedPositive": 929, + "recall": 0.9344, + "total": 1485, + "trueNegative": 494, + "truePositive": 883, + "wilsonPrecisionLower95": 0.9346 + }, + "threshold": 0.695 + } + }, + "metrics": { + "f1": 0.9454, + "falseNegative": 115, + "falsePositive": 90, + "precision": 0.9517, + "predictedPositive": 1865, + "recall": 0.9392, + "total": 2970, + "trueNegative": 990, + "truePositive": 1775, + "wilsonPrecisionLower95": 0.9411 + }, + "threshold": 0.7 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 45, + "recall": 1.0, + "total": 1485, + "trueNegative": 1440, + "truePositive": 45, + "wilsonPrecisionLower95": 0.9213 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.967, + "falseNegative": 1, + "falsePositive": 2, + "precision": 0.9565, + "predictedPositive": 46, + "recall": 0.9778, + "total": 1485, + "trueNegative": 1438, + "truePositive": 44, + "wilsonPrecisionLower95": 0.8547 + }, + "threshold": 0.93 + } + }, + "metrics": { + "f1": 0.978, + "falseNegative": 1, + "falsePositive": 3, + "precision": 0.9674, + "predictedPositive": 92, + "recall": 0.9889, + "total": 2970, + "trueNegative": 2877, + "truePositive": 89, + "wilsonPrecisionLower95": 0.9085 + }, + "threshold": 0.91 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8403, + "falseNegative": 89, + "falsePositive": 14, + "precision": 0.9509, + "predictedPositive": 285, + "recall": 0.7528, + "total": 1485, + "trueNegative": 1111, + "truePositive": 271, + "wilsonPrecisionLower95": 0.9192 + }, + "threshold": 0.835 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8278, + "falseNegative": 98, + "falsePositive": 11, + "precision": 0.9597, + "predictedPositive": 273, + "recall": 0.7278, + "total": 1485, + "trueNegative": 1114, + "truePositive": 262, + "wilsonPrecisionLower95": 0.9293 + }, + "threshold": 0.745 + } + }, + "metrics": { + "f1": 0.8637, + "falseNegative": 153, + "falsePositive": 26, + "precision": 0.9562, + "predictedPositive": 593, + "recall": 0.7875, + "total": 2970, + "trueNegative": 2224, + "truePositive": 567, + "wilsonPrecisionLower95": 0.9365 + }, + "threshold": 0.755 + } + }, + "trainingRecords": 15345, + "validation": { + "intentsMeeting95Precision": 9, + "macroF1": 0.822, + "macroPrecision": 0.9713, + "macroRecall": 0.7487, + "minimumPrecision": 0.9507, + "perIntent": { + "blessing": { + "f1": 0.3636, + "falseNegative": 70, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 20, + "recall": 0.2222, + "total": 2970, + "trueNegative": 2880, + "truePositive": 20, + "wilsonPrecisionLower95": 0.8389 + }, + "complaint": { + "f1": 0.9764, + "falseNegative": 9, + "falsePositive": 8, + "precision": 0.9777, + "predictedPositive": 359, + "recall": 0.975, + "total": 2970, + "trueNegative": 2602, + "truePositive": 351, + "wilsonPrecisionLower95": 0.9567 + }, + "confirmationDecision": { + "f1": 0.8418, + "falseNegative": 47, + "falsePositive": 3, + "precision": 0.9779, + "predictedPositive": 136, + "recall": 0.7389, + "total": 2970, + "trueNegative": 2787, + "truePositive": 133, + "wilsonPrecisionLower95": 0.9372 + }, + "followUpReminder": { + "f1": 0.8456, + "falseNegative": 89, + "falsePositive": 10, + "precision": 0.9644, + "predictedPositive": 281, + "recall": 0.7528, + "total": 2970, + "trueNegative": 2600, + "truePositive": 271, + "wilsonPrecisionLower95": 0.9357 + }, + "invitation": { + "f1": 0.7593, + "falseNegative": 68, + "falsePositive": 3, + "precision": 0.9739, + "predictedPositive": 115, + "recall": 0.6222, + "total": 2970, + "trueNegative": 2787, + "truePositive": 112, + "wilsonPrecisionLower95": 0.9261 + }, + "question": { + "f1": 0.849, + "falseNegative": 152, + "falsePositive": 18, + "precision": 0.9637, + "predictedPositive": 496, + "recall": 0.7587, + "total": 2970, + "trueNegative": 2322, + "truePositive": 478, + "wilsonPrecisionLower95": 0.9434 + }, + "replyableMessage": { + "f1": 0.9452, + "falseNegative": 114, + "falsePositive": 92, + "precision": 0.9507, + "predictedPositive": 1868, + "recall": 0.9397, + "total": 2970, + "trueNegative": 988, + "truePositive": 1776, + "wilsonPrecisionLower95": 0.94 + }, + "scheduleNegotiation": { + "f1": 0.9834, + "falseNegative": 1, + "falsePositive": 2, + "precision": 0.978, + "predictedPositive": 91, + "recall": 0.9889, + "total": 2970, + "trueNegative": 2878, + "truePositive": 89, + "wilsonPrecisionLower95": 0.9234 + }, + "task": { + "f1": 0.8341, + "falseNegative": 187, + "falsePositive": 25, + "precision": 0.9552, + "predictedPositive": 558, + "recall": 0.7403, + "total": 2970, + "trueNegative": 2225, + "truePositive": 533, + "wilsonPrecisionLower95": 0.9347 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 1.5e-05, + "augmentation": "none", + "char_max": 5, + "char_min": 3, + "external_weight": 0.5900000000000001, + "hard_example_weight": 1.0, + "l1_ratio": 0.0, + "max_char_features": 50000, + "max_word_features": 25000, + "min_df": 2, + "pseudo_weight": 0.18, + "round": 2, + "word_max": 2 + }, + "featureCount": 61451, + "pseudoLabelCounts": {}, + "round": 2, + "selectionObjective": 0.8228, + "syntheticRegression": { + "intentsMeeting95Precision": 7, + "macroF1": 0.751, + "macroPrecision": 0.9667, + "macroRecall": 0.6602, + "minimumPrecision": 0.8606, + "perIntent": { + "blessing": { + "f1": 0.2178, + "falseNegative": 79, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 11, + "recall": 0.1222, + "total": 2970, + "trueNegative": 2880, + "truePositive": 11, + "wilsonPrecisionLower95": 0.7412 + }, + "complaint": { + "f1": 0.9595, + "falseNegative": 28, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 332, + "recall": 0.9222, + "total": 2970, + "trueNegative": 2610, + "truePositive": 332, + "wilsonPrecisionLower95": 0.9886 + }, + "confirmationDecision": { + "f1": 0.8232, + "falseNegative": 38, + "falsePositive": 23, + "precision": 0.8606, + "predictedPositive": 165, + "recall": 0.7889, + "total": 2970, + "trueNegative": 2767, + "truePositive": 142, + "wilsonPrecisionLower95": 0.7995 + }, + "followUpReminder": { + "f1": 0.7565, + "falseNegative": 141, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 219, + "recall": 0.6083, + "total": 2970, + "trueNegative": 2610, + "truePositive": 219, + "wilsonPrecisionLower95": 0.9828 + }, + "invitation": { + "f1": 0.6394, + "falseNegative": 94, + "falsePositive": 3, + "precision": 0.9663, + "predictedPositive": 89, + "recall": 0.4778, + "total": 2970, + "trueNegative": 2787, + "truePositive": 86, + "wilsonPrecisionLower95": 0.9055 + }, + "question": { + "f1": 0.86, + "falseNegative": 151, + "falsePositive": 5, + "precision": 0.9897, + "predictedPositive": 484, + "recall": 0.7603, + "total": 2970, + "trueNegative": 2335, + "truePositive": 479, + "wilsonPrecisionLower95": 0.976 + }, + "replyableMessage": { + "f1": 0.9038, + "falseNegative": 293, + "falsePositive": 47, + "precision": 0.9714, + "predictedPositive": 1644, + "recall": 0.845, + "total": 2970, + "trueNegative": 1033, + "truePositive": 1597, + "wilsonPrecisionLower95": 0.9622 + }, + "scheduleNegotiation": { + "f1": 0.9425, + "falseNegative": 8, + "falsePositive": 2, + "precision": 0.9762, + "predictedPositive": 84, + "recall": 0.9111, + "total": 2970, + "trueNegative": 2878, + "truePositive": 82, + "wilsonPrecisionLower95": 0.9173 + }, + "task": { + "f1": 0.6564, + "falseNegative": 356, + "falsePositive": 25, + "precision": 0.9357, + "predictedPositive": 389, + "recall": 0.5056, + "total": 2970, + "trueNegative": 2225, + "truePositive": 364, + "wilsonPrecisionLower95": 0.9069 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9048, + "falseNegative": 7, + "falsePositive": 1, + "precision": 0.9744, + "predictedPositive": 39, + "recall": 0.8444, + "total": 1485, + "trueNegative": 1439, + "truePositive": 38, + "wilsonPrecisionLower95": 0.8682 + }, + "threshold": 0.59 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 180, + "recall": 1.0, + "total": 1485, + "trueNegative": 1305, + "truePositive": 180, + "wilsonPrecisionLower95": 0.9791 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9226, + "falseNegative": 19, + "falsePositive": 8, + "precision": 0.9527, + "predictedPositive": 169, + "recall": 0.8944, + "total": 1485, + "trueNegative": 1297, + "truePositive": 161, + "wilsonPrecisionLower95": 0.9094 + }, + "threshold": 0.555 + } + }, + "metrics": { + "f1": 0.9649, + "falseNegative": 16, + "falsePositive": 9, + "precision": 0.9745, + "predictedPositive": 353, + "recall": 0.9556, + "total": 2970, + "trueNegative": 2601, + "truePositive": 344, + "wilsonPrecisionLower95": 0.9523 + }, + "threshold": 0.5 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.954, + "falseNegative": 7, + "falsePositive": 1, + "precision": 0.9881, + "predictedPositive": 84, + "recall": 0.9222, + "total": 1485, + "trueNegative": 1394, + "truePositive": 83, + "wilsonPrecisionLower95": 0.9356 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7183, + "falseNegative": 39, + "falsePositive": 1, + "precision": 0.9808, + "predictedPositive": 52, + "recall": 0.5667, + "total": 1485, + "trueNegative": 1394, + "truePositive": 51, + "wilsonPrecisionLower95": 0.8988 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.8481, + "falseNegative": 46, + "falsePositive": 2, + "precision": 0.9853, + "predictedPositive": 136, + "recall": 0.7444, + "total": 2970, + "trueNegative": 2788, + "truePositive": 134, + "wilsonPrecisionLower95": 0.948 + }, + "threshold": 0.5 + }, + "followUpReminder": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8328, + "falseNegative": 48, + "falsePositive": 5, + "precision": 0.9635, + "predictedPositive": 137, + "recall": 0.7333, + "total": 1485, + "trueNegative": 1300, + "truePositive": 132, + "wilsonPrecisionLower95": 0.9174 + }, + "threshold": 0.91 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6879, + "falseNegative": 83, + "falsePositive": 5, + "precision": 0.951, + "predictedPositive": 102, + "recall": 0.5389, + "total": 1485, + "trueNegative": 1300, + "truePositive": 97, + "wilsonPrecisionLower95": 0.8903 + }, + "threshold": 0.765 + } + }, + "metrics": { + "f1": 0.6243, + "falseNegative": 193, + "falsePositive": 8, + "precision": 0.9543, + "predictedPositive": 175, + "recall": 0.4639, + "total": 2970, + "trueNegative": 2602, + "truePositive": 167, + "wilsonPrecisionLower95": 0.9124 + }, + "threshold": 0.905 + }, + "invitation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9059, + "falseNegative": 13, + "falsePositive": 3, + "precision": 0.9625, + "predictedPositive": 80, + "recall": 0.8556, + "total": 1485, + "trueNegative": 1392, + "truePositive": 77, + "wilsonPrecisionLower95": 0.8955 + }, + "threshold": 0.86 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.6113, + "falseNegative": 99, + "falsePositive": 4, + "precision": 0.9529, + "predictedPositive": 85, + "recall": 0.45, + "total": 2970, + "trueNegative": 2786, + "truePositive": 81, + "wilsonPrecisionLower95": 0.8852 + }, + "threshold": 0.855 + }, + "question": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8505, + "falseNegative": 76, + "falsePositive": 8, + "precision": 0.9676, + "predictedPositive": 247, + "recall": 0.7587, + "total": 1485, + "trueNegative": 1162, + "truePositive": 239, + "wilsonPrecisionLower95": 0.9374 + }, + "threshold": 0.705 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7308, + "falseNegative": 129, + "falsePositive": 8, + "precision": 0.9588, + "predictedPositive": 194, + "recall": 0.5905, + "total": 1485, + "trueNegative": 1162, + "truePositive": 186, + "wilsonPrecisionLower95": 0.9208 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.7813, + "falseNegative": 212, + "falsePositive": 22, + "precision": 0.95, + "predictedPositive": 440, + "recall": 0.6635, + "total": 2970, + "trueNegative": 2318, + "truePositive": 418, + "wilsonPrecisionLower95": 0.9255 + }, + "threshold": 0.57 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9629, + "falseNegative": 23, + "falsePositive": 48, + "precision": 0.9505, + "predictedPositive": 970, + "recall": 0.9757, + "total": 1485, + "trueNegative": 492, + "truePositive": 922, + "wilsonPrecisionLower95": 0.935 + }, + "threshold": 0.62 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8526, + "falseNegative": 216, + "falsePositive": 36, + "precision": 0.9529, + "predictedPositive": 765, + "recall": 0.7714, + "total": 1485, + "trueNegative": 504, + "truePositive": 729, + "wilsonPrecisionLower95": 0.9355 + }, + "threshold": 0.84 + } + }, + "metrics": { + "f1": 0.9224, + "falseNegative": 197, + "falsePositive": 88, + "precision": 0.9506, + "predictedPositive": 1781, + "recall": 0.8958, + "total": 2970, + "trueNegative": 992, + "truePositive": 1693, + "wilsonPrecisionLower95": 0.9395 + }, + "threshold": 0.775 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9783, + "falseNegative": 0, + "falsePositive": 2, + "precision": 0.9574, + "predictedPositive": 47, + "recall": 1.0, + "total": 1485, + "trueNegative": 1438, + "truePositive": 45, + "wilsonPrecisionLower95": 0.8575 + }, + "threshold": 0.765 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9778, + "falseNegative": 1, + "falsePositive": 1, + "precision": 0.9778, + "predictedPositive": 45, + "recall": 0.9778, + "total": 1485, + "trueNegative": 1439, + "truePositive": 44, + "wilsonPrecisionLower95": 0.8843 + }, + "threshold": 0.855 + } + }, + "metrics": { + "f1": 0.978, + "falseNegative": 1, + "falsePositive": 3, + "precision": 0.9674, + "predictedPositive": 92, + "recall": 0.9889, + "total": 2970, + "trueNegative": 2877, + "truePositive": 89, + "wilsonPrecisionLower95": 0.9085 + }, + "threshold": 0.855 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8541, + "falseNegative": 82, + "falsePositive": 13, + "precision": 0.9553, + "predictedPositive": 291, + "recall": 0.7722, + "total": 1485, + "trueNegative": 1112, + "truePositive": 278, + "wilsonPrecisionLower95": 0.9251 + }, + "threshold": 0.84 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.5418, + "falseNegative": 224, + "falsePositive": 6, + "precision": 0.9577, + "predictedPositive": 142, + "recall": 0.3778, + "total": 1485, + "trueNegative": 1119, + "truePositive": 136, + "wilsonPrecisionLower95": 0.9109 + }, + "threshold": 0.91 + } + }, + "metrics": { + "f1": 0.7246, + "falseNegative": 299, + "falsePositive": 21, + "precision": 0.9525, + "predictedPositive": 442, + "recall": 0.5847, + "total": 2970, + "trueNegative": 2229, + "truePositive": 421, + "wilsonPrecisionLower95": 0.9285 + }, + "threshold": 0.88 + } + }, + "trainingRecords": 15345, + "validation": { + "intentsMeeting95Precision": 9, + "macroF1": 0.7578, + "macroPrecision": 0.9691, + "macroRecall": 0.668, + "minimumPrecision": 0.9516, + "perIntent": { + "blessing": { + "f1": 0.2524, + "falseNegative": 77, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 13, + "recall": 0.1444, + "total": 2970, + "trueNegative": 2880, + "truePositive": 13, + "wilsonPrecisionLower95": 0.7719 + }, + "complaint": { + "f1": 0.9619, + "falseNegative": 19, + "falsePositive": 8, + "precision": 0.9771, + "predictedPositive": 349, + "recall": 0.9472, + "total": 2970, + "trueNegative": 2602, + "truePositive": 341, + "wilsonPrecisionLower95": 0.9554 + }, + "confirmationDecision": { + "f1": 0.8481, + "falseNegative": 46, + "falsePositive": 2, + "precision": 0.9853, + "predictedPositive": 136, + "recall": 0.7444, + "total": 2970, + "trueNegative": 2788, + "truePositive": 134, + "wilsonPrecisionLower95": 0.948 + }, + "followUpReminder": { + "f1": 0.7646, + "falseNegative": 131, + "falsePositive": 10, + "precision": 0.9582, + "predictedPositive": 239, + "recall": 0.6361, + "total": 2970, + "trueNegative": 2600, + "truePositive": 229, + "wilsonPrecisionLower95": 0.9247 + }, + "invitation": { + "f1": 0.5923, + "falseNegative": 103, + "falsePositive": 3, + "precision": 0.9625, + "predictedPositive": 80, + "recall": 0.4278, + "total": 2970, + "trueNegative": 2787, + "truePositive": 77, + "wilsonPrecisionLower95": 0.8955 + }, + "question": { + "f1": 0.7937, + "falseNegative": 205, + "falsePositive": 16, + "precision": 0.9637, + "predictedPositive": 441, + "recall": 0.6746, + "total": 2970, + "trueNegative": 2324, + "truePositive": 425, + "wilsonPrecisionLower95": 0.9419 + }, + "replyableMessage": { + "f1": 0.9109, + "falseNegative": 239, + "falsePositive": 84, + "precision": 0.9516, + "predictedPositive": 1735, + "recall": 0.8735, + "total": 2970, + "trueNegative": 996, + "truePositive": 1651, + "wilsonPrecisionLower95": 0.9404 + }, + "scheduleNegotiation": { + "f1": 0.978, + "falseNegative": 1, + "falsePositive": 3, + "precision": 0.9674, + "predictedPositive": 92, + "recall": 0.9889, + "total": 2970, + "trueNegative": 2877, + "truePositive": 89, + "wilsonPrecisionLower95": 0.9085 + }, + "task": { + "f1": 0.7181, + "falseNegative": 306, + "falsePositive": 19, + "precision": 0.9561, + "predictedPositive": 433, + "recall": 0.575, + "total": 2970, + "trueNegative": 2231, + "truePositive": 414, + "wilsonPrecisionLower95": 0.9325 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 1e-05, + "augmentation": "none", + "char_max": 6, + "char_min": 2, + "external_weight": 0.6100000000000001, + "hard_example_weight": 1.2, + "l1_ratio": 0.05, + "max_char_features": 55000, + "max_word_features": 25000, + "min_df": 2, + "pseudo_weight": 0.195, + "round": 3, + "word_max": 2 + }, + "featureCount": 66451, + "pseudoLabelCounts": {}, + "round": 3, + "selectionObjective": 0.8348, + "syntheticRegression": { + "intentsMeeting95Precision": 4, + "macroF1": 0.7343, + "macroPrecision": 0.9403, + "macroRecall": 0.6672, + "minimumPrecision": 0.809, + "perIntent": { + "blessing": { + "f1": 0.0645, + "falseNegative": 87, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 3, + "recall": 0.0333, + "total": 2970, + "trueNegative": 2880, + "truePositive": 3, + "wilsonPrecisionLower95": 0.4385 + }, + "complaint": { + "f1": 0.9816, + "falseNegative": 13, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 347, + "recall": 0.9639, + "total": 2970, + "trueNegative": 2610, + "truePositive": 347, + "wilsonPrecisionLower95": 0.9891 + }, + "confirmationDecision": { + "f1": 0.8045, + "falseNegative": 36, + "falsePositive": 34, + "precision": 0.809, + "predictedPositive": 178, + "recall": 0.8, + "total": 2970, + "trueNegative": 2756, + "truePositive": 144, + "wilsonPrecisionLower95": 0.745 + }, + "followUpReminder": { + "f1": 0.7967, + "falseNegative": 121, + "falsePositive": 1, + "precision": 0.9958, + "predictedPositive": 240, + "recall": 0.6639, + "total": 2970, + "trueNegative": 2609, + "truePositive": 239, + "wilsonPrecisionLower95": 0.9768 + }, + "invitation": { + "f1": 0.5985, + "falseNegative": 101, + "falsePositive": 5, + "precision": 0.9405, + "predictedPositive": 84, + "recall": 0.4389, + "total": 2970, + "trueNegative": 2785, + "truePositive": 79, + "wilsonPrecisionLower95": 0.8681 + }, + "question": { + "f1": 0.8643, + "falseNegative": 149, + "falsePositive": 2, + "precision": 0.9959, + "predictedPositive": 483, + "recall": 0.7635, + "total": 2970, + "trueNegative": 2338, + "truePositive": 481, + "wilsonPrecisionLower95": 0.985 + }, + "replyableMessage": { + "f1": 0.931, + "falseNegative": 189, + "falsePositive": 63, + "precision": 0.9643, + "predictedPositive": 1764, + "recall": 0.9, + "total": 2970, + "trueNegative": 1017, + "truePositive": 1701, + "wilsonPrecisionLower95": 0.9546 + }, + "scheduleNegotiation": { + "f1": 0.8222, + "falseNegative": 16, + "falsePositive": 16, + "precision": 0.8222, + "predictedPositive": 90, + "recall": 0.8222, + "total": 2970, + "trueNegative": 2864, + "truePositive": 74, + "wilsonPrecisionLower95": 0.7306 + }, + "task": { + "f1": 0.7452, + "falseNegative": 274, + "falsePositive": 31, + "precision": 0.935, + "predictedPositive": 477, + "recall": 0.6194, + "total": 2970, + "trueNegative": 2219, + "truePositive": 446, + "wilsonPrecisionLower95": 0.9092 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 180, + "recall": 1.0, + "total": 1485, + "trueNegative": 1305, + "truePositive": 180, + "wilsonPrecisionLower95": 0.9791 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9371, + "falseNegative": 16, + "falsePositive": 6, + "precision": 0.9647, + "predictedPositive": 170, + "recall": 0.9111, + "total": 1485, + "trueNegative": 1299, + "truePositive": 164, + "wilsonPrecisionLower95": 0.9251 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.969, + "falseNegative": 16, + "falsePositive": 6, + "precision": 0.9829, + "predictedPositive": 350, + "recall": 0.9556, + "total": 2970, + "trueNegative": 2604, + "truePositive": 344, + "wilsonPrecisionLower95": 0.9631 + }, + "threshold": 0.5 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9727, + "falseNegative": 1, + "falsePositive": 4, + "precision": 0.957, + "predictedPositive": 93, + "recall": 0.9889, + "total": 1485, + "trueNegative": 1391, + "truePositive": 89, + "wilsonPrecisionLower95": 0.8946 + }, + "threshold": 0.585 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7815, + "falseNegative": 31, + "falsePositive": 2, + "precision": 0.9672, + "predictedPositive": 61, + "recall": 0.6556, + "total": 1485, + "trueNegative": 1393, + "truePositive": 59, + "wilsonPrecisionLower95": 0.8881 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.8701, + "falseNegative": 36, + "falsePositive": 7, + "precision": 0.9536, + "predictedPositive": 151, + "recall": 0.8, + "total": 2970, + "trueNegative": 2783, + "truePositive": 144, + "wilsonPrecisionLower95": 0.9074 + }, + "threshold": 0.555 + }, + "followUpReminder": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8454, + "falseNegative": 46, + "falsePositive": 3, + "precision": 0.9781, + "predictedPositive": 137, + "recall": 0.7444, + "total": 1485, + "trueNegative": 1302, + "truePositive": 134, + "wilsonPrecisionLower95": 0.9376 + }, + "threshold": 0.97 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7525, + "falseNegative": 69, + "falsePositive": 4, + "precision": 0.9652, + "predictedPositive": 115, + "recall": 0.6167, + "total": 1485, + "trueNegative": 1301, + "truePositive": 111, + "wilsonPrecisionLower95": 0.914 + }, + "threshold": 0.785 + } + }, + "metrics": { + "f1": 0.703, + "falseNegative": 160, + "falsePositive": 9, + "precision": 0.9569, + "predictedPositive": 209, + "recall": 0.5556, + "total": 2970, + "trueNegative": 2601, + "truePositive": 200, + "wilsonPrecisionLower95": 0.9202 + }, + "threshold": 0.955 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7974, + "falseNegative": 29, + "falsePositive": 2, + "precision": 0.9683, + "predictedPositive": 63, + "recall": 0.6778, + "total": 1485, + "trueNegative": 1393, + "truePositive": 61, + "wilsonPrecisionLower95": 0.8914 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8259, + "falseNegative": 85, + "falsePositive": 12, + "precision": 0.9504, + "predictedPositive": 242, + "recall": 0.7302, + "total": 1485, + "trueNegative": 1158, + "truePositive": 230, + "wilsonPrecisionLower95": 0.9153 + }, + "threshold": 0.865 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8194, + "falseNegative": 95, + "falsePositive": 2, + "precision": 0.991, + "predictedPositive": 222, + "recall": 0.6984, + "total": 1485, + "trueNegative": 1168, + "truePositive": 220, + "wilsonPrecisionLower95": 0.9678 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.7959, + "falseNegative": 199, + "falsePositive": 22, + "precision": 0.9514, + "predictedPositive": 453, + "recall": 0.6841, + "total": 2970, + "trueNegative": 2318, + "truePositive": 431, + "wilsonPrecisionLower95": 0.9276 + }, + "threshold": 0.74 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9496, + "falseNegative": 50, + "falsePositive": 45, + "precision": 0.9521, + "predictedPositive": 940, + "recall": 0.9471, + "total": 1485, + "trueNegative": 495, + "truePositive": 895, + "wilsonPrecisionLower95": 0.9365 + }, + "threshold": 0.705 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9209, + "falseNegative": 101, + "falsePositive": 44, + "precision": 0.9505, + "predictedPositive": 888, + "recall": 0.8931, + "total": 1485, + "trueNegative": 496, + "truePositive": 844, + "wilsonPrecisionLower95": 0.9341 + }, + "threshold": 0.78 + } + }, + "metrics": { + "f1": 0.9318, + "falseNegative": 163, + "falsePositive": 90, + "precision": 0.9505, + "predictedPositive": 1817, + "recall": 0.9138, + "total": 2970, + "trueNegative": 990, + "truePositive": 1727, + "wilsonPrecisionLower95": 0.9395 + }, + "threshold": 0.77 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 45, + "recall": 1.0, + "total": 1485, + "trueNegative": 1440, + "truePositive": 45, + "wilsonPrecisionLower95": 0.9213 + }, + "threshold": 0.745 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9783, + "falseNegative": 0, + "falsePositive": 2, + "precision": 0.9574, + "predictedPositive": 47, + "recall": 1.0, + "total": 1485, + "trueNegative": 1438, + "truePositive": 45, + "wilsonPrecisionLower95": 0.8575 + }, + "threshold": 0.93 + } + }, + "metrics": { + "f1": 0.9778, + "falseNegative": 2, + "falsePositive": 2, + "precision": 0.9778, + "predictedPositive": 90, + "recall": 0.9778, + "total": 2970, + "trueNegative": 2878, + "truePositive": 88, + "wilsonPrecisionLower95": 0.9226 + }, + "threshold": 0.93 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8488, + "falseNegative": 85, + "falsePositive": 13, + "precision": 0.9549, + "predictedPositive": 288, + "recall": 0.7639, + "total": 1485, + "trueNegative": 1112, + "truePositive": 275, + "wilsonPrecisionLower95": 0.9243 + }, + "threshold": 0.865 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8006, + "falseNegative": 111, + "falsePositive": 13, + "precision": 0.9504, + "predictedPositive": 262, + "recall": 0.6917, + "total": 1485, + "trueNegative": 1112, + "truePositive": 249, + "wilsonPrecisionLower95": 0.917 + }, + "threshold": 0.8 + } + }, + "metrics": { + "f1": 0.8301, + "falseNegative": 190, + "falsePositive": 27, + "precision": 0.9515, + "predictedPositive": 557, + "recall": 0.7361, + "total": 2970, + "trueNegative": 2223, + "truePositive": 530, + "wilsonPrecisionLower95": 0.9304 + }, + "threshold": 0.82 + } + }, + "trainingRecords": 15345, + "validation": { + "intentsMeeting95Precision": 8, + "macroF1": 0.7723, + "macroPrecision": 0.9713, + "macroRecall": 0.7011, + "minimumPrecision": 0.9513, + "perIntent": { + "blessing": { + "f1": 0.1443, + "falseNegative": 83, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 7, + "recall": 0.0778, + "total": 2970, + "trueNegative": 2880, + "truePositive": 7, + "wilsonPrecisionLower95": 0.6457 + }, + "complaint": { + "f1": 0.969, + "falseNegative": 16, + "falsePositive": 6, + "precision": 0.9829, + "predictedPositive": 350, + "recall": 0.9556, + "total": 2970, + "trueNegative": 2604, + "truePositive": 344, + "wilsonPrecisionLower95": 0.9631 + }, + "confirmationDecision": { + "f1": 0.8862, + "falseNegative": 32, + "falsePositive": 6, + "precision": 0.961, + "predictedPositive": 154, + "recall": 0.8222, + "total": 2970, + "trueNegative": 2784, + "truePositive": 148, + "wilsonPrecisionLower95": 0.9176 + }, + "followUpReminder": { + "f1": 0.8007, + "falseNegative": 115, + "falsePositive": 7, + "precision": 0.9722, + "predictedPositive": 252, + "recall": 0.6806, + "total": 2970, + "trueNegative": 2603, + "truePositive": 245, + "wilsonPrecisionLower95": 0.9438 + }, + "invitation": { + "f1": 0.5781, + "falseNegative": 106, + "falsePositive": 2, + "precision": 0.9737, + "predictedPositive": 76, + "recall": 0.4111, + "total": 2970, + "trueNegative": 2788, + "truePositive": 74, + "wilsonPrecisionLower95": 0.909 + }, + "question": { + "f1": 0.8227, + "falseNegative": 180, + "falsePositive": 14, + "precision": 0.9698, + "predictedPositive": 464, + "recall": 0.7143, + "total": 2970, + "trueNegative": 2326, + "truePositive": 450, + "wilsonPrecisionLower95": 0.95 + }, + "replyableMessage": { + "f1": 0.9354, + "falseNegative": 151, + "falsePositive": 89, + "precision": 0.9513, + "predictedPositive": 1828, + "recall": 0.9201, + "total": 2970, + "trueNegative": 991, + "truePositive": 1739, + "wilsonPrecisionLower95": 0.9405 + }, + "scheduleNegotiation": { + "f1": 0.989, + "falseNegative": 0, + "falsePositive": 2, + "precision": 0.9783, + "predictedPositive": 92, + "recall": 1.0, + "total": 2970, + "trueNegative": 2878, + "truePositive": 90, + "wilsonPrecisionLower95": 0.9242 + }, + "task": { + "f1": 0.8252, + "falseNegative": 196, + "falsePositive": 26, + "precision": 0.9527, + "predictedPositive": 550, + "recall": 0.7278, + "total": 2970, + "trueNegative": 2224, + "truePositive": 524, + "wilsonPrecisionLower95": 0.9316 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 8e-06, + "augmentation": "none", + "char_max": 6, + "char_min": 3, + "external_weight": 0.63, + "hard_example_weight": 1.2, + "l1_ratio": 0.05, + "max_char_features": 60000, + "max_word_features": 30000, + "min_df": 2, + "pseudo_weight": 0.21, + "round": 4, + "word_max": 3 + }, + "featureCount": 79458, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 0 + }, + "complaint": { + "negative": 97, + "positive": 0 + }, + "confirmationDecision": { + "negative": 51, + "positive": 2 + }, + "followUpReminder": { + "negative": 250, + "positive": 0 + }, + "invitation": { + "negative": 250, + "positive": 0 + }, + "question": { + "negative": 36, + "positive": 1 + }, + "replyableMessage": { + "negative": 3, + "positive": 130 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 0 + }, + "task": { + "negative": 85, + "positive": 0 + } + }, + "round": 4, + "selectionObjective": 0.796, + "syntheticRegression": { + "intentsMeeting95Precision": 6, + "macroF1": 0.6978, + "macroPrecision": 0.9611, + "macroRecall": 0.5994, + "minimumPrecision": 0.8074, + "perIntent": { + "blessing": { + "f1": 0.0645, + "falseNegative": 87, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 3, + "recall": 0.0333, + "total": 2970, + "trueNegative": 2880, + "truePositive": 3, + "wilsonPrecisionLower95": 0.4385 + }, + "complaint": { + "f1": 0.967, + "falseNegative": 23, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 337, + "recall": 0.9361, + "total": 2970, + "trueNegative": 2610, + "truePositive": 337, + "wilsonPrecisionLower95": 0.9887 + }, + "confirmationDecision": { + "f1": 0.6921, + "falseNegative": 71, + "falsePositive": 26, + "precision": 0.8074, + "predictedPositive": 135, + "recall": 0.6056, + "total": 2970, + "trueNegative": 2764, + "truePositive": 109, + "wilsonPrecisionLower95": 0.7328 + }, + "followUpReminder": { + "f1": 0.7734, + "falseNegative": 133, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 227, + "recall": 0.6306, + "total": 2970, + "trueNegative": 2610, + "truePositive": 227, + "wilsonPrecisionLower95": 0.9834 + }, + "invitation": { + "f1": 0.5837, + "falseNegative": 105, + "falsePositive": 2, + "precision": 0.974, + "predictedPositive": 77, + "recall": 0.4167, + "total": 2970, + "trueNegative": 2788, + "truePositive": 75, + "wilsonPrecisionLower95": 0.9102 + }, + "question": { + "f1": 0.7613, + "falseNegative": 241, + "falsePositive": 3, + "precision": 0.9923, + "predictedPositive": 392, + "recall": 0.6175, + "total": 2970, + "trueNegative": 2337, + "truePositive": 389, + "wilsonPrecisionLower95": 0.9777 + }, + "replyableMessage": { + "f1": 0.8816, + "falseNegative": 353, + "falsePositive": 60, + "precision": 0.9624, + "predictedPositive": 1597, + "recall": 0.8132, + "total": 2970, + "trueNegative": 1020, + "truePositive": 1537, + "wilsonPrecisionLower95": 0.9519 + }, + "scheduleNegotiation": { + "f1": 0.9222, + "falseNegative": 13, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 77, + "recall": 0.8556, + "total": 2970, + "trueNegative": 2880, + "truePositive": 77, + "wilsonPrecisionLower95": 0.9525 + }, + "task": { + "f1": 0.6346, + "falseNegative": 370, + "falsePositive": 33, + "precision": 0.9138, + "predictedPositive": 383, + "recall": 0.4861, + "total": 2970, + "trueNegative": 2217, + "truePositive": 350, + "wilsonPrecisionLower95": 0.8815 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9972, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 179, + "recall": 0.9944, + "total": 1485, + "trueNegative": 1305, + "truePositive": 179, + "wilsonPrecisionLower95": 0.979 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9253, + "falseNegative": 19, + "falsePositive": 7, + "precision": 0.9583, + "predictedPositive": 168, + "recall": 0.8944, + "total": 1485, + "trueNegative": 1298, + "truePositive": 161, + "wilsonPrecisionLower95": 0.9165 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.9618, + "falseNegative": 20, + "falsePositive": 7, + "precision": 0.9798, + "predictedPositive": 347, + "recall": 0.9444, + "total": 2970, + "trueNegative": 2603, + "truePositive": 340, + "wilsonPrecisionLower95": 0.959 + }, + "threshold": 0.5 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8765, + "falseNegative": 19, + "falsePositive": 1, + "precision": 0.9861, + "predictedPositive": 72, + "recall": 0.7889, + "total": 1485, + "trueNegative": 1394, + "truePositive": 71, + "wilsonPrecisionLower95": 0.9254 + }, + "threshold": 0.945 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6569, + "falseNegative": 45, + "falsePositive": 2, + "precision": 0.9574, + "predictedPositive": 47, + "recall": 0.5, + "total": 1485, + "trueNegative": 1393, + "truePositive": 45, + "wilsonPrecisionLower95": 0.8575 + }, + "threshold": 0.64 + } + }, + "metrics": { + "f1": 0.7108, + "falseNegative": 78, + "falsePositive": 5, + "precision": 0.9533, + "predictedPositive": 107, + "recall": 0.5667, + "total": 2970, + "trueNegative": 2785, + "truePositive": 102, + "wilsonPrecisionLower95": 0.8952 + }, + "threshold": 0.91 + }, + "followUpReminder": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8, + "falseNegative": 56, + "falsePositive": 6, + "precision": 0.9538, + "predictedPositive": 130, + "recall": 0.6889, + "total": 1485, + "trueNegative": 1299, + "truePositive": 124, + "wilsonPrecisionLower95": 0.903 + }, + "threshold": 0.98 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6165, + "falseNegative": 98, + "falsePositive": 4, + "precision": 0.9535, + "predictedPositive": 86, + "recall": 0.4556, + "total": 1485, + "trueNegative": 1301, + "truePositive": 82, + "wilsonPrecisionLower95": 0.8864 + }, + "threshold": 0.825 + } + }, + "metrics": { + "f1": 0.5912, + "falseNegative": 206, + "falsePositive": 7, + "precision": 0.9565, + "predictedPositive": 161, + "recall": 0.4278, + "total": 2970, + "trueNegative": 2603, + "truePositive": 154, + "wilsonPrecisionLower95": 0.913 + }, + "threshold": 0.98 + }, + "invitation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8302, + "falseNegative": 24, + "falsePositive": 3, + "precision": 0.9565, + "predictedPositive": 69, + "recall": 0.7333, + "total": 1485, + "trueNegative": 1392, + "truePositive": 66, + "wilsonPrecisionLower95": 0.8798 + }, + "threshold": 0.98 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.536, + "falseNegative": 113, + "falsePositive": 3, + "precision": 0.9571, + "predictedPositive": 70, + "recall": 0.3722, + "total": 2970, + "trueNegative": 2787, + "truePositive": 67, + "wilsonPrecisionLower95": 0.8814 + }, + "threshold": 0.995 + }, + "question": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.7813, + "falseNegative": 106, + "falsePositive": 11, + "precision": 0.95, + "predictedPositive": 220, + "recall": 0.6635, + "total": 1485, + "trueNegative": 1159, + "truePositive": 209, + "wilsonPrecisionLower95": 0.9127 + }, + "threshold": 0.865 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6844, + "falseNegative": 148, + "falsePositive": 6, + "precision": 0.9653, + "predictedPositive": 173, + "recall": 0.5302, + "total": 1485, + "trueNegative": 1164, + "truePositive": 167, + "wilsonPrecisionLower95": 0.9264 + }, + "threshold": 0.63 + } + }, + "metrics": { + "f1": 0.75, + "falseNegative": 240, + "falsePositive": 20, + "precision": 0.9512, + "predictedPositive": 410, + "recall": 0.619, + "total": 2970, + "trueNegative": 2320, + "truePositive": 390, + "wilsonPrecisionLower95": 0.9259 + }, + "threshold": 0.74 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9441, + "falseNegative": 59, + "falsePositive": 46, + "precision": 0.9506, + "predictedPositive": 932, + "recall": 0.9376, + "total": 1485, + "trueNegative": 494, + "truePositive": 886, + "wilsonPrecisionLower95": 0.9348 + }, + "threshold": 0.76 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8342, + "falseNegative": 243, + "falsePositive": 36, + "precision": 0.9512, + "predictedPositive": 738, + "recall": 0.7429, + "total": 1485, + "trueNegative": 504, + "truePositive": 702, + "wilsonPrecisionLower95": 0.9332 + }, + "threshold": 0.885 + } + }, + "metrics": { + "f1": 0.9056, + "falseNegative": 255, + "falsePositive": 86, + "precision": 0.95, + "predictedPositive": 1721, + "recall": 0.8651, + "total": 2970, + "trueNegative": 994, + "truePositive": 1635, + "wilsonPrecisionLower95": 0.9387 + }, + "threshold": 0.855 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 45, + "recall": 1.0, + "total": 1485, + "trueNegative": 1440, + "truePositive": 45, + "wilsonPrecisionLower95": 0.9213 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9318, + "falseNegative": 4, + "falsePositive": 2, + "precision": 0.9535, + "predictedPositive": 43, + "recall": 0.9111, + "total": 1485, + "trueNegative": 1438, + "truePositive": 41, + "wilsonPrecisionLower95": 0.8454 + }, + "threshold": 0.945 + } + }, + "metrics": { + "f1": 0.9727, + "falseNegative": 1, + "falsePositive": 4, + "precision": 0.957, + "predictedPositive": 93, + "recall": 0.9889, + "total": 2970, + "trueNegative": 2876, + "truePositive": 89, + "wilsonPrecisionLower95": 0.8946 + }, + "threshold": 0.62 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.851, + "falseNegative": 83, + "falsePositive": 14, + "precision": 0.9519, + "predictedPositive": 291, + "recall": 0.7694, + "total": 1485, + "trueNegative": 1111, + "truePositive": 277, + "wilsonPrecisionLower95": 0.9209 + }, + "threshold": 0.87 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.4886, + "falseNegative": 242, + "falsePositive": 5, + "precision": 0.9593, + "predictedPositive": 123, + "recall": 0.3278, + "total": 1485, + "trueNegative": 1120, + "truePositive": 118, + "wilsonPrecisionLower95": 0.9084 + }, + "threshold": 0.935 + } + }, + "metrics": { + "f1": 0.6978, + "falseNegative": 324, + "falsePositive": 19, + "precision": 0.9542, + "predictedPositive": 415, + "recall": 0.55, + "total": 2970, + "trueNegative": 2231, + "truePositive": 396, + "wilsonPrecisionLower95": 0.9296 + }, + "threshold": 0.915 + } + }, + "trainingRecords": 15345, + "validation": { + "intentsMeeting95Precision": 9, + "macroF1": 0.7192, + "macroPrecision": 0.9671, + "macroRecall": 0.62, + "minimumPrecision": 0.9509, + "perIntent": { + "blessing": { + "f1": 0.2, + "falseNegative": 80, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 10, + "recall": 0.1111, + "total": 2970, + "trueNegative": 2880, + "truePositive": 10, + "wilsonPrecisionLower95": 0.7225 + }, + "complaint": { + "f1": 0.9618, + "falseNegative": 20, + "falsePositive": 7, + "precision": 0.9798, + "predictedPositive": 347, + "recall": 0.9444, + "total": 2970, + "trueNegative": 2603, + "truePositive": 340, + "wilsonPrecisionLower95": 0.959 + }, + "confirmationDecision": { + "f1": 0.7759, + "falseNegative": 64, + "falsePositive": 3, + "precision": 0.9748, + "predictedPositive": 119, + "recall": 0.6444, + "total": 2970, + "trueNegative": 2787, + "truePositive": 116, + "wilsonPrecisionLower95": 0.9285 + }, + "followUpReminder": { + "f1": 0.7153, + "falseNegative": 154, + "falsePositive": 10, + "precision": 0.9537, + "predictedPositive": 216, + "recall": 0.5722, + "total": 2970, + "trueNegative": 2600, + "truePositive": 206, + "wilsonPrecisionLower95": 0.9169 + }, + "invitation": { + "f1": 0.5301, + "falseNegative": 114, + "falsePositive": 3, + "precision": 0.9565, + "predictedPositive": 69, + "recall": 0.3667, + "total": 2970, + "trueNegative": 2787, + "truePositive": 66, + "wilsonPrecisionLower95": 0.8798 + }, + "question": { + "f1": 0.7351, + "falseNegative": 254, + "falsePositive": 17, + "precision": 0.9567, + "predictedPositive": 393, + "recall": 0.5968, + "total": 2970, + "trueNegative": 2323, + "truePositive": 376, + "wilsonPrecisionLower95": 0.9318 + }, + "replyableMessage": { + "f1": 0.8921, + "falseNegative": 302, + "falsePositive": 82, + "precision": 0.9509, + "predictedPositive": 1670, + "recall": 0.8402, + "total": 2970, + "trueNegative": 998, + "truePositive": 1588, + "wilsonPrecisionLower95": 0.9395 + }, + "scheduleNegotiation": { + "f1": 0.9663, + "falseNegative": 4, + "falsePositive": 2, + "precision": 0.9773, + "predictedPositive": 88, + "recall": 0.9556, + "total": 2970, + "trueNegative": 2878, + "truePositive": 86, + "wilsonPrecisionLower95": 0.9209 + }, + "task": { + "f1": 0.6966, + "falseNegative": 325, + "falsePositive": 19, + "precision": 0.9541, + "predictedPositive": 414, + "recall": 0.5486, + "total": 2970, + "trueNegative": 2231, + "truePositive": 395, + "wilsonPrecisionLower95": 0.9294 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 1.2e-05, + "augmentation": "none", + "char_max": 5, + "char_min": 2, + "external_weight": 0.65, + "hard_example_weight": 1.4, + "l1_ratio": 0.1, + "max_char_features": 60000, + "max_word_features": 32000, + "min_df": 1, + "pseudo_weight": 0.22499999999999998, + "round": 5, + "word_max": 3 + }, + "featureCount": 92000, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 0 + }, + "complaint": { + "negative": 88, + "positive": 0 + }, + "confirmationDecision": { + "negative": 96, + "positive": 3 + }, + "followUpReminder": { + "negative": 250, + "positive": 0 + }, + "invitation": { + "negative": 250, + "positive": 0 + }, + "question": { + "negative": 40, + "positive": 1 + }, + "replyableMessage": { + "negative": 3, + "positive": 150 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 0 + }, + "task": { + "negative": 97, + "positive": 0 + } + }, + "round": 5, + "selectionObjective": 0.7771, + "syntheticRegression": { + "intentsMeeting95Precision": 5, + "macroF1": 0.6709, + "macroPrecision": 0.9586, + "macroRecall": 0.5803, + "minimumPrecision": 0.8588, + "perIntent": { + "blessing": { + "f1": 0.022, + "falseNegative": 89, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 1, + "recall": 0.0111, + "total": 2970, + "trueNegative": 2880, + "truePositive": 1, + "wilsonPrecisionLower95": 0.2065 + }, + "complaint": { + "f1": 0.967, + "falseNegative": 23, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 337, + "recall": 0.9361, + "total": 2970, + "trueNegative": 2610, + "truePositive": 337, + "wilsonPrecisionLower95": 0.9887 + }, + "confirmationDecision": { + "f1": 0.8343, + "falseNegative": 34, + "falsePositive": 24, + "precision": 0.8588, + "predictedPositive": 170, + "recall": 0.8111, + "total": 2970, + "trueNegative": 2766, + "truePositive": 146, + "wilsonPrecisionLower95": 0.7985 + }, + "followUpReminder": { + "f1": 0.7173, + "falseNegative": 157, + "falsePositive": 3, + "precision": 0.9854, + "predictedPositive": 206, + "recall": 0.5639, + "total": 2970, + "trueNegative": 2607, + "truePositive": 203, + "wilsonPrecisionLower95": 0.9581 + }, + "invitation": { + "f1": 0.3612, + "falseNegative": 139, + "falsePositive": 6, + "precision": 0.8723, + "predictedPositive": 47, + "recall": 0.2278, + "total": 2970, + "trueNegative": 2784, + "truePositive": 41, + "wilsonPrecisionLower95": 0.7483 + }, + "question": { + "f1": 0.8336, + "falseNegative": 179, + "falsePositive": 1, + "precision": 0.9978, + "predictedPositive": 452, + "recall": 0.7159, + "total": 2970, + "trueNegative": 2339, + "truePositive": 451, + "wilsonPrecisionLower95": 0.9876 + }, + "replyableMessage": { + "f1": 0.912, + "falseNegative": 257, + "falsePositive": 58, + "precision": 0.9657, + "predictedPositive": 1691, + "recall": 0.864, + "total": 2970, + "trueNegative": 1022, + "truePositive": 1633, + "wilsonPrecisionLower95": 0.9559 + }, + "scheduleNegotiation": { + "f1": 0.6364, + "falseNegative": 48, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 42, + "recall": 0.4667, + "total": 2970, + "trueNegative": 2880, + "truePositive": 42, + "wilsonPrecisionLower95": 0.9162 + }, + "task": { + "f1": 0.7542, + "falseNegative": 269, + "falsePositive": 25, + "precision": 0.9475, + "predictedPositive": 476, + "recall": 0.6264, + "total": 2970, + "trueNegative": 2225, + "truePositive": 451, + "wilsonPrecisionLower95": 0.9236 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 180, + "recall": 1.0, + "total": 1485, + "trueNegative": 1305, + "truePositive": 180, + "wilsonPrecisionLower95": 0.9791 + }, + "threshold": 0.69 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9288, + "falseNegative": 17, + "falsePositive": 8, + "precision": 0.9532, + "predictedPositive": 171, + "recall": 0.9056, + "total": 1485, + "trueNegative": 1297, + "truePositive": 163, + "wilsonPrecisionLower95": 0.9104 + }, + "threshold": 0.51 + } + }, + "metrics": { + "f1": 0.9569, + "falseNegative": 16, + "falsePositive": 15, + "precision": 0.9582, + "predictedPositive": 359, + "recall": 0.9556, + "total": 2970, + "trueNegative": 2595, + "truePositive": 344, + "wilsonPrecisionLower95": 0.9322 + }, + "threshold": 0.5 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8589, + "falseNegative": 20, + "falsePositive": 3, + "precision": 0.9589, + "predictedPositive": 73, + "recall": 0.7778, + "total": 1485, + "trueNegative": 1392, + "truePositive": 70, + "wilsonPrecisionLower95": 0.886 + }, + "threshold": 0.57 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6763, + "falseNegative": 43, + "falsePositive": 2, + "precision": 0.9592, + "predictedPositive": 49, + "recall": 0.5222, + "total": 1485, + "trueNegative": 1393, + "truePositive": 47, + "wilsonPrecisionLower95": 0.8629 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.76, + "falseNegative": 66, + "falsePositive": 6, + "precision": 0.95, + "predictedPositive": 120, + "recall": 0.6333, + "total": 2970, + "trueNegative": 2784, + "truePositive": 114, + "wilsonPrecisionLower95": 0.8952 + }, + "threshold": 0.56 + }, + "followUpReminder": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8, + "falseNegative": 56, + "falsePositive": 6, + "precision": 0.9538, + "predictedPositive": 130, + "recall": 0.6889, + "total": 1485, + "trueNegative": 1299, + "truePositive": 124, + "wilsonPrecisionLower95": 0.903 + }, + "threshold": 0.96 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7708, + "falseNegative": 64, + "falsePositive": 5, + "precision": 0.9587, + "predictedPositive": 121, + "recall": 0.6444, + "total": 1485, + "trueNegative": 1300, + "truePositive": 116, + "wilsonPrecisionLower95": 0.9069 + }, + "threshold": 0.58 + } + }, + "metrics": { + "f1": 0.6892, + "falseNegative": 166, + "falsePositive": 9, + "precision": 0.9557, + "predictedPositive": 203, + "recall": 0.5389, + "total": 2970, + "trueNegative": 2601, + "truePositive": 194, + "wilsonPrecisionLower95": 0.9179 + }, + "threshold": 0.915 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.5203, + "falseNegative": 58, + "falsePositive": 1, + "precision": 0.9697, + "predictedPositive": 33, + "recall": 0.3556, + "total": 1485, + "trueNegative": 1394, + "truePositive": 32, + "wilsonPrecisionLower95": 0.8468 + }, + "threshold": 0.88 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8225, + "falseNegative": 88, + "falsePositive": 10, + "precision": 0.9578, + "predictedPositive": 237, + "recall": 0.7206, + "total": 1485, + "trueNegative": 1160, + "truePositive": 227, + "wilsonPrecisionLower95": 0.9241 + }, + "threshold": 0.84 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8388, + "falseNegative": 86, + "falsePositive": 2, + "precision": 0.9913, + "predictedPositive": 231, + "recall": 0.727, + "total": 1485, + "trueNegative": 1168, + "truePositive": 229, + "wilsonPrecisionLower95": 0.969 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.8591, + "falseNegative": 136, + "falsePositive": 26, + "precision": 0.95, + "predictedPositive": 520, + "recall": 0.7841, + "total": 2970, + "trueNegative": 2314, + "truePositive": 494, + "wilsonPrecisionLower95": 0.9277 + }, + "threshold": 0.505 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9541, + "falseNegative": 40, + "falsePositive": 47, + "precision": 0.9506, + "predictedPositive": 952, + "recall": 0.9577, + "total": 1485, + "trueNegative": 493, + "truePositive": 905, + "wilsonPrecisionLower95": 0.935 + }, + "threshold": 0.69 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8807, + "falseNegative": 170, + "falsePositive": 40, + "precision": 0.9509, + "predictedPositive": 815, + "recall": 0.8201, + "total": 1485, + "trueNegative": 500, + "truePositive": 775, + "wilsonPrecisionLower95": 0.9339 + }, + "threshold": 0.8 + } + }, + "metrics": { + "f1": 0.914, + "falseNegative": 227, + "falsePositive": 86, + "precision": 0.9508, + "predictedPositive": 1749, + "recall": 0.8799, + "total": 2970, + "trueNegative": 994, + "truePositive": 1663, + "wilsonPrecisionLower95": 0.9397 + }, + "threshold": 0.78 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9888, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 44, + "recall": 0.9778, + "total": 1485, + "trueNegative": 1440, + "truePositive": 44, + "wilsonPrecisionLower95": 0.9197 + }, + "threshold": 0.635 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.8129, + "falseNegative": 27, + "falsePositive": 2, + "precision": 0.9692, + "predictedPositive": 65, + "recall": 0.7, + "total": 2970, + "trueNegative": 2878, + "truePositive": 63, + "wilsonPrecisionLower95": 0.8946 + }, + "threshold": 0.995 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8185, + "falseNegative": 103, + "falsePositive": 11, + "precision": 0.959, + "predictedPositive": 268, + "recall": 0.7139, + "total": 1485, + "trueNegative": 1114, + "truePositive": 257, + "wilsonPrecisionLower95": 0.928 + }, + "threshold": 0.83 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7702, + "falseNegative": 127, + "falsePositive": 12, + "precision": 0.951, + "predictedPositive": 245, + "recall": 0.6472, + "total": 1485, + "trueNegative": 1113, + "truePositive": 233, + "wilsonPrecisionLower95": 0.9164 + }, + "threshold": 0.765 + } + }, + "metrics": { + "f1": 0.7816, + "falseNegative": 244, + "falsePositive": 22, + "precision": 0.9558, + "predictedPositive": 498, + "recall": 0.6611, + "total": 2970, + "trueNegative": 2228, + "truePositive": 476, + "wilsonPrecisionLower95": 0.934 + }, + "threshold": 0.805 + } + }, + "trainingRecords": 15345, + "validation": { + "intentsMeeting95Precision": 8, + "macroF1": 0.6867, + "macroPrecision": 0.9721, + "macroRecall": 0.5915, + "minimumPrecision": 0.9508, + "perIntent": { + "blessing": { + "f1": 0.0851, + "falseNegative": 86, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 4, + "recall": 0.0444, + "total": 2970, + "trueNegative": 2880, + "truePositive": 4, + "wilsonPrecisionLower95": 0.5101 + }, + "complaint": { + "f1": 0.9648, + "falseNegative": 17, + "falsePositive": 8, + "precision": 0.9772, + "predictedPositive": 351, + "recall": 0.9528, + "total": 2970, + "trueNegative": 2602, + "truePositive": 343, + "wilsonPrecisionLower95": 0.9557 + }, + "confirmationDecision": { + "f1": 0.7748, + "falseNegative": 63, + "falsePositive": 5, + "precision": 0.959, + "predictedPositive": 122, + "recall": 0.65, + "total": 2970, + "trueNegative": 2785, + "truePositive": 117, + "wilsonPrecisionLower95": 0.9076 + }, + "followUpReminder": { + "f1": 0.7856, + "falseNegative": 120, + "falsePositive": 11, + "precision": 0.9562, + "predictedPositive": 251, + "recall": 0.6667, + "total": 2970, + "trueNegative": 2599, + "truePositive": 240, + "wilsonPrecisionLower95": 0.9232 + }, + "invitation": { + "f1": 0.3694, + "falseNegative": 139, + "falsePositive": 1, + "precision": 0.9762, + "predictedPositive": 42, + "recall": 0.2278, + "total": 2970, + "trueNegative": 2789, + "truePositive": 41, + "wilsonPrecisionLower95": 0.8768 + }, + "question": { + "f1": 0.8306, + "falseNegative": 174, + "falsePositive": 12, + "precision": 0.9744, + "predictedPositive": 468, + "recall": 0.7238, + "total": 2970, + "trueNegative": 2328, + "truePositive": 456, + "wilsonPrecisionLower95": 0.9557 + }, + "replyableMessage": { + "f1": 0.9188, + "falseNegative": 210, + "falsePositive": 87, + "precision": 0.9508, + "predictedPositive": 1767, + "recall": 0.8889, + "total": 2970, + "trueNegative": 993, + "truePositive": 1680, + "wilsonPrecisionLower95": 0.9397 + }, + "scheduleNegotiation": { + "f1": 0.6567, + "falseNegative": 46, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 44, + "recall": 0.4889, + "total": 2970, + "trueNegative": 2880, + "truePositive": 44, + "wilsonPrecisionLower95": 0.9197 + }, + "task": { + "f1": 0.7948, + "falseNegative": 230, + "falsePositive": 23, + "precision": 0.9552, + "predictedPositive": 513, + "recall": 0.6806, + "total": 2970, + "trueNegative": 2227, + "truePositive": 490, + "wilsonPrecisionLower95": 0.9336 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 8e-06, + "augmentation": "punctuation", + "char_max": 6, + "char_min": 2, + "external_weight": 0.67, + "hard_example_weight": 1.5, + "l1_ratio": 0.1, + "max_char_features": 65000, + "max_word_features": 28000, + "min_df": 2, + "pseudo_weight": 0.24, + "round": 6, + "word_max": 2 + }, + "featureCount": 78354, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 0 + }, + "complaint": { + "negative": 146, + "positive": 0 + }, + "confirmationDecision": { + "negative": 88, + "positive": 2 + }, + "followUpReminder": { + "negative": 250, + "positive": 0 + }, + "invitation": { + "negative": 250, + "positive": 0 + }, + "question": { + "negative": 36, + "positive": 5 + }, + "replyableMessage": { + "negative": 2, + "positive": 130 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 0 + }, + "task": { + "negative": 83, + "positive": 0 + } + }, + "round": 6, + "selectionObjective": 0.811, + "syntheticRegression": { + "intentsMeeting95Precision": 5, + "macroF1": 0.7064, + "macroPrecision": 0.9443, + "macroRecall": 0.623, + "minimumPrecision": 0.7975, + "perIntent": { + "blessing": { + "f1": 0.0435, + "falseNegative": 88, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 2, + "recall": 0.0222, + "total": 2970, + "trueNegative": 2880, + "truePositive": 2, + "wilsonPrecisionLower95": 0.3424 + }, + "complaint": { + "f1": 0.967, + "falseNegative": 23, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 337, + "recall": 0.9361, + "total": 2970, + "trueNegative": 2610, + "truePositive": 337, + "wilsonPrecisionLower95": 0.9887 + }, + "confirmationDecision": { + "f1": 0.7215, + "falseNegative": 66, + "falsePositive": 22, + "precision": 0.8382, + "predictedPositive": 136, + "recall": 0.6333, + "total": 2970, + "trueNegative": 2768, + "truePositive": 114, + "wilsonPrecisionLower95": 0.7672 + }, + "followUpReminder": { + "f1": 0.6847, + "falseNegative": 170, + "falsePositive": 5, + "precision": 0.9744, + "predictedPositive": 195, + "recall": 0.5278, + "total": 2970, + "trueNegative": 2605, + "truePositive": 190, + "wilsonPrecisionLower95": 0.9414 + }, + "invitation": { + "f1": 0.4865, + "falseNegative": 117, + "falsePositive": 16, + "precision": 0.7975, + "predictedPositive": 79, + "recall": 0.35, + "total": 2970, + "trueNegative": 2774, + "truePositive": 63, + "wilsonPrecisionLower95": 0.696 + }, + "question": { + "f1": 0.8556, + "falseNegative": 156, + "falsePositive": 4, + "precision": 0.9916, + "predictedPositive": 478, + "recall": 0.7524, + "total": 2970, + "trueNegative": 2336, + "truePositive": 474, + "wilsonPrecisionLower95": 0.9787 + }, + "replyableMessage": { + "f1": 0.8888, + "falseNegative": 335, + "falsePositive": 54, + "precision": 0.9664, + "predictedPositive": 1609, + "recall": 0.8228, + "total": 2970, + "trueNegative": 1026, + "truePositive": 1555, + "wilsonPrecisionLower95": 0.9565 + }, + "scheduleNegotiation": { + "f1": 0.9831, + "falseNegative": 3, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 87, + "recall": 0.9667, + "total": 2970, + "trueNegative": 2880, + "truePositive": 87, + "wilsonPrecisionLower95": 0.9577 + }, + "task": { + "f1": 0.7265, + "falseNegative": 291, + "falsePositive": 32, + "precision": 0.9306, + "predictedPositive": 461, + "recall": 0.5958, + "total": 2970, + "trueNegative": 2218, + "truePositive": 429, + "wilsonPrecisionLower95": 0.9037 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9834, + "falseNegative": 2, + "falsePositive": 4, + "precision": 0.978, + "predictedPositive": 182, + "recall": 0.9889, + "total": 1485, + "trueNegative": 1301, + "truePositive": 178, + "wilsonPrecisionLower95": 0.9449 + }, + "threshold": 0.555 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9494, + "falseNegative": 11, + "falsePositive": 7, + "precision": 0.9602, + "predictedPositive": 176, + "recall": 0.9389, + "total": 1485, + "trueNegative": 1298, + "truePositive": 169, + "wilsonPrecisionLower95": 0.9202 + }, + "threshold": 0.535 + } + }, + "metrics": { + "f1": 0.9652, + "falseNegative": 13, + "falsePositive": 12, + "precision": 0.9666, + "predictedPositive": 359, + "recall": 0.9639, + "total": 2970, + "trueNegative": 2598, + "truePositive": 347, + "wilsonPrecisionLower95": 0.9425 + }, + "threshold": 0.535 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8, + "falseNegative": 28, + "falsePositive": 3, + "precision": 0.9538, + "predictedPositive": 65, + "recall": 0.6889, + "total": 1485, + "trueNegative": 1392, + "truePositive": 62, + "wilsonPrecisionLower95": 0.8729 + }, + "threshold": 0.695 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.731, + "falseNegative": 37, + "falsePositive": 2, + "precision": 0.9636, + "predictedPositive": 55, + "recall": 0.5889, + "total": 1485, + "trueNegative": 1393, + "truePositive": 53, + "wilsonPrecisionLower95": 0.8768 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.7354, + "falseNegative": 73, + "falsePositive": 4, + "precision": 0.964, + "predictedPositive": 111, + "recall": 0.5944, + "total": 2970, + "trueNegative": 2786, + "truePositive": 107, + "wilsonPrecisionLower95": 0.911 + }, + "threshold": 0.695 + }, + "followUpReminder": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8632, + "falseNegative": 38, + "falsePositive": 7, + "precision": 0.953, + "predictedPositive": 149, + "recall": 0.7889, + "total": 1485, + "trueNegative": 1298, + "truePositive": 142, + "wilsonPrecisionLower95": 0.9062 + }, + "threshold": 0.9 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.4938, + "falseNegative": 120, + "falsePositive": 3, + "precision": 0.9524, + "predictedPositive": 63, + "recall": 0.3333, + "total": 1485, + "trueNegative": 1302, + "truePositive": 60, + "wilsonPrecisionLower95": 0.8691 + }, + "threshold": 0.845 + } + }, + "metrics": { + "f1": 0.7018, + "falseNegative": 160, + "falsePositive": 10, + "precision": 0.9524, + "predictedPositive": 210, + "recall": 0.5556, + "total": 2970, + "trueNegative": 2600, + "truePositive": 200, + "wilsonPrecisionLower95": 0.9146 + }, + "threshold": 0.885 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7222, + "falseNegative": 38, + "falsePositive": 2, + "precision": 0.963, + "predictedPositive": 54, + "recall": 0.5778, + "total": 1485, + "trueNegative": 1393, + "truePositive": 52, + "wilsonPrecisionLower95": 0.8746 + }, + "threshold": 0.665 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8167, + "falseNegative": 90, + "falsePositive": 11, + "precision": 0.9534, + "predictedPositive": 236, + "recall": 0.7143, + "total": 1485, + "trueNegative": 1159, + "truePositive": 225, + "wilsonPrecisionLower95": 0.9185 + }, + "threshold": 0.865 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8205, + "falseNegative": 91, + "falsePositive": 7, + "precision": 0.9697, + "predictedPositive": 231, + "recall": 0.7111, + "total": 1485, + "trueNegative": 1163, + "truePositive": 224, + "wilsonPrecisionLower95": 0.9388 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.804, + "falseNegative": 191, + "falsePositive": 23, + "precision": 0.9502, + "predictedPositive": 462, + "recall": 0.6968, + "total": 2970, + "trueNegative": 2317, + "truePositive": 439, + "wilsonPrecisionLower95": 0.9264 + }, + "threshold": 0.68 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9268, + "falseNegative": 90, + "falsePositive": 45, + "precision": 0.95, + "predictedPositive": 900, + "recall": 0.9048, + "total": 1485, + "trueNegative": 495, + "truePositive": 855, + "wilsonPrecisionLower95": 0.9338 + }, + "threshold": 0.86 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8789, + "falseNegative": 176, + "falsePositive": 36, + "precision": 0.9553, + "predictedPositive": 805, + "recall": 0.8138, + "total": 1485, + "trueNegative": 504, + "truePositive": 769, + "wilsonPrecisionLower95": 0.9387 + }, + "threshold": 0.85 + } + }, + "metrics": { + "f1": 0.9046, + "falseNegative": 259, + "falsePositive": 85, + "precision": 0.9505, + "predictedPositive": 1716, + "recall": 0.863, + "total": 2970, + "trueNegative": 995, + "truePositive": 1631, + "wilsonPrecisionLower95": 0.9392 + }, + "threshold": 0.85 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9773, + "falseNegative": 2, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 43, + "recall": 0.9556, + "total": 1485, + "trueNegative": 1440, + "truePositive": 43, + "wilsonPrecisionLower95": 0.918 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9888, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 44, + "recall": 0.9778, + "total": 1485, + "trueNegative": 1440, + "truePositive": 44, + "wilsonPrecisionLower95": 0.9197 + }, + "threshold": 0.97 + } + }, + "metrics": { + "f1": 0.9497, + "falseNegative": 5, + "falsePositive": 4, + "precision": 0.9551, + "predictedPositive": 89, + "recall": 0.9444, + "total": 2970, + "trueNegative": 2876, + "truePositive": 85, + "wilsonPrecisionLower95": 0.8901 + }, + "threshold": 0.91 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.827, + "falseNegative": 97, + "falsePositive": 13, + "precision": 0.9529, + "predictedPositive": 276, + "recall": 0.7306, + "total": 1485, + "trueNegative": 1112, + "truePositive": 263, + "wilsonPrecisionLower95": 0.9211 + }, + "threshold": 0.89 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7902, + "falseNegative": 117, + "falsePositive": 12, + "precision": 0.9529, + "predictedPositive": 255, + "recall": 0.675, + "total": 1485, + "trueNegative": 1113, + "truePositive": 243, + "wilsonPrecisionLower95": 0.9196 + }, + "threshold": 0.785 + } + }, + "metrics": { + "f1": 0.7783, + "falseNegative": 246, + "falsePositive": 24, + "precision": 0.9518, + "predictedPositive": 498, + "recall": 0.6583, + "total": 2970, + "trueNegative": 2226, + "truePositive": 474, + "wilsonPrecisionLower95": 0.9293 + }, + "threshold": 0.85 + } + }, + "trainingRecords": 17767, + "validation": { + "intentsMeeting95Precision": 8, + "macroF1": 0.7395, + "macroPrecision": 0.9687, + "macroRecall": 0.6518, + "minimumPrecision": 0.9525, + "perIntent": { + "blessing": { + "f1": 0.1633, + "falseNegative": 82, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 8, + "recall": 0.0889, + "total": 2970, + "trueNegative": 2880, + "truePositive": 8, + "wilsonPrecisionLower95": 0.6756 + }, + "complaint": { + "f1": 0.9666, + "falseNegative": 13, + "falsePositive": 11, + "precision": 0.9693, + "predictedPositive": 358, + "recall": 0.9639, + "total": 2970, + "trueNegative": 2599, + "truePositive": 347, + "wilsonPrecisionLower95": 0.9458 + }, + "confirmationDecision": { + "f1": 0.7667, + "falseNegative": 65, + "falsePositive": 5, + "precision": 0.9583, + "predictedPositive": 120, + "recall": 0.6389, + "total": 2970, + "trueNegative": 2785, + "truePositive": 115, + "wilsonPrecisionLower95": 0.9062 + }, + "followUpReminder": { + "f1": 0.7063, + "falseNegative": 158, + "falsePositive": 10, + "precision": 0.9528, + "predictedPositive": 212, + "recall": 0.5611, + "total": 2970, + "trueNegative": 2600, + "truePositive": 202, + "wilsonPrecisionLower95": 0.9154 + }, + "invitation": { + "f1": 0.5382, + "falseNegative": 113, + "falsePositive": 2, + "precision": 0.971, + "predictedPositive": 69, + "recall": 0.3722, + "total": 2970, + "trueNegative": 2788, + "truePositive": 67, + "wilsonPrecisionLower95": 0.9003 + }, + "question": { + "f1": 0.8186, + "falseNegative": 181, + "falsePositive": 18, + "precision": 0.9615, + "predictedPositive": 467, + "recall": 0.7127, + "total": 2970, + "trueNegative": 2322, + "truePositive": 449, + "wilsonPrecisionLower95": 0.9399 + }, + "replyableMessage": { + "f1": 0.9035, + "falseNegative": 266, + "falsePositive": 81, + "precision": 0.9525, + "predictedPositive": 1705, + "recall": 0.8593, + "total": 2970, + "trueNegative": 999, + "truePositive": 1624, + "wilsonPrecisionLower95": 0.9413 + }, + "scheduleNegotiation": { + "f1": 0.9831, + "falseNegative": 3, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 87, + "recall": 0.9667, + "total": 2970, + "trueNegative": 2880, + "truePositive": 87, + "wilsonPrecisionLower95": 0.9577 + }, + "task": { + "f1": 0.809, + "falseNegative": 214, + "falsePositive": 25, + "precision": 0.9529, + "predictedPositive": 531, + "recall": 0.7028, + "total": 2970, + "trueNegative": 2225, + "truePositive": 506, + "wilsonPrecisionLower95": 0.9314 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 6e-06, + "augmentation": "punctuation", + "char_max": 6, + "char_min": 3, + "external_weight": 0.6900000000000001, + "hard_example_weight": 1.5, + "l1_ratio": 0.1, + "max_char_features": 65000, + "max_word_features": 32000, + "min_df": 2, + "pseudo_weight": 0.255, + "round": 7, + "word_max": 3 + }, + "featureCount": 87507, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 0 + }, + "complaint": { + "negative": 141, + "positive": 0 + }, + "confirmationDecision": { + "negative": 95, + "positive": 2 + }, + "followUpReminder": { + "negative": 250, + "positive": 0 + }, + "invitation": { + "negative": 250, + "positive": 0 + }, + "question": { + "negative": 36, + "positive": 5 + }, + "replyableMessage": { + "negative": 2, + "positive": 129 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 0 + }, + "task": { + "negative": 87, + "positive": 0 + } + }, + "round": 7, + "selectionObjective": 0.7501, + "syntheticRegression": { + "intentsMeeting95Precision": 5, + "macroF1": 0.587, + "macroPrecision": 0.9374, + "macroRecall": 0.4917, + "minimumPrecision": 0.7548, + "perIntent": { + "blessing": { + "f1": 0.0851, + "falseNegative": 86, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 4, + "recall": 0.0444, + "total": 2970, + "trueNegative": 2880, + "truePositive": 4, + "wilsonPrecisionLower95": 0.5101 + }, + "complaint": { + "f1": 0.9304, + "falseNegative": 39, + "falsePositive": 9, + "precision": 0.9727, + "predictedPositive": 330, + "recall": 0.8917, + "total": 2970, + "trueNegative": 2601, + "truePositive": 321, + "wilsonPrecisionLower95": 0.949 + }, + "confirmationDecision": { + "f1": 0.6985, + "falseNegative": 63, + "falsePositive": 38, + "precision": 0.7548, + "predictedPositive": 155, + "recall": 0.65, + "total": 2970, + "trueNegative": 2752, + "truePositive": 117, + "wilsonPrecisionLower95": 0.6815 + }, + "followUpReminder": { + "f1": 0.3974, + "falseNegative": 269, + "falsePositive": 7, + "precision": 0.9286, + "predictedPositive": 98, + "recall": 0.2528, + "total": 2970, + "trueNegative": 2603, + "truePositive": 91, + "wilsonPrecisionLower95": 0.8598 + }, + "invitation": { + "f1": 0.2609, + "falseNegative": 153, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 27, + "recall": 0.15, + "total": 2970, + "trueNegative": 2790, + "truePositive": 27, + "wilsonPrecisionLower95": 0.8754 + }, + "question": { + "f1": 0.697, + "falseNegative": 285, + "falsePositive": 15, + "precision": 0.9583, + "predictedPositive": 360, + "recall": 0.5476, + "total": 2970, + "trueNegative": 2325, + "truePositive": 345, + "wilsonPrecisionLower95": 0.9324 + }, + "replyableMessage": { + "f1": 0.8083, + "falseNegative": 572, + "falsePositive": 53, + "precision": 0.9613, + "predictedPositive": 1371, + "recall": 0.6974, + "total": 2970, + "trueNegative": 1027, + "truePositive": 1318, + "wilsonPrecisionLower95": 0.9498 + }, + "scheduleNegotiation": { + "f1": 0.8929, + "falseNegative": 15, + "falsePositive": 3, + "precision": 0.9615, + "predictedPositive": 78, + "recall": 0.8333, + "total": 2970, + "trueNegative": 2877, + "truePositive": 75, + "wilsonPrecisionLower95": 0.8929 + }, + "task": { + "f1": 0.5124, + "falseNegative": 462, + "falsePositive": 29, + "precision": 0.899, + "predictedPositive": 287, + "recall": 0.3583, + "total": 2970, + "trueNegative": 2221, + "truePositive": 258, + "wilsonPrecisionLower95": 0.8586 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9809, + "falseNegative": 0, + "falsePositive": 7, + "precision": 0.9626, + "predictedPositive": 187, + "recall": 1.0, + "total": 1485, + "trueNegative": 1298, + "truePositive": 180, + "wilsonPrecisionLower95": 0.9248 + }, + "threshold": 0.74 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8974, + "falseNegative": 27, + "falsePositive": 8, + "precision": 0.9503, + "predictedPositive": 161, + "recall": 0.85, + "total": 1485, + "trueNegative": 1297, + "truePositive": 153, + "wilsonPrecisionLower95": 0.905 + }, + "threshold": 0.665 + } + }, + "metrics": { + "f1": 0.9529, + "falseNegative": 16, + "falsePositive": 18, + "precision": 0.9503, + "predictedPositive": 362, + "recall": 0.9556, + "total": 2970, + "trueNegative": 2592, + "truePositive": 344, + "wilsonPrecisionLower95": 0.9228 + }, + "threshold": 0.51 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.7763, + "falseNegative": 31, + "falsePositive": 3, + "precision": 0.9516, + "predictedPositive": 62, + "recall": 0.6556, + "total": 1485, + "trueNegative": 1392, + "truePositive": 59, + "wilsonPrecisionLower95": 0.8671 + }, + "threshold": 0.895 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7568, + "falseNegative": 34, + "falsePositive": 2, + "precision": 0.9655, + "predictedPositive": 58, + "recall": 0.6222, + "total": 1485, + "trueNegative": 1393, + "truePositive": 56, + "wilsonPrecisionLower95": 0.8827 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.6569, + "falseNegative": 90, + "falsePositive": 4, + "precision": 0.9574, + "predictedPositive": 94, + "recall": 0.5, + "total": 2970, + "trueNegative": 2786, + "truePositive": 90, + "wilsonPrecisionLower95": 0.8956 + }, + "threshold": 0.895 + }, + "followUpReminder": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8421, + "falseNegative": 44, + "falsePositive": 7, + "precision": 0.951, + "predictedPositive": 143, + "recall": 0.7556, + "total": 1485, + "trueNegative": 1298, + "truePositive": 136, + "wilsonPrecisionLower95": 0.9024 + }, + "threshold": 0.985 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.5985, + "falseNegative": 205, + "falsePositive": 3, + "precision": 0.981, + "predictedPositive": 158, + "recall": 0.4306, + "total": 2970, + "trueNegative": 2607, + "truePositive": 155, + "wilsonPrecisionLower95": 0.9457 + }, + "threshold": 0.995 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.7143, + "falseNegative": 135, + "falsePositive": 9, + "precision": 0.9524, + "predictedPositive": 189, + "recall": 0.5714, + "total": 1485, + "trueNegative": 1161, + "truePositive": 180, + "wilsonPrecisionLower95": 0.912 + }, + "threshold": 0.965 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7168, + "falseNegative": 134, + "falsePositive": 9, + "precision": 0.9526, + "predictedPositive": 190, + "recall": 0.5746, + "total": 1485, + "trueNegative": 1161, + "truePositive": 181, + "wilsonPrecisionLower95": 0.9124 + }, + "threshold": 0.52 + } + }, + "metrics": { + "f1": 0.7149, + "falseNegative": 269, + "falsePositive": 19, + "precision": 0.95, + "predictedPositive": 380, + "recall": 0.573, + "total": 2970, + "trueNegative": 2321, + "truePositive": 361, + "wilsonPrecisionLower95": 0.9232 + }, + "threshold": 0.795 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8846, + "falseNegative": 163, + "falsePositive": 41, + "precision": 0.9502, + "predictedPositive": 823, + "recall": 0.8275, + "total": 1485, + "trueNegative": 499, + "truePositive": 782, + "wilsonPrecisionLower95": 0.9331 + }, + "threshold": 0.89 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7476, + "falseNegative": 363, + "falsePositive": 30, + "precision": 0.951, + "predictedPositive": 612, + "recall": 0.6159, + "total": 1485, + "trueNegative": 510, + "truePositive": 582, + "wilsonPrecisionLower95": 0.9309 + }, + "threshold": 0.92 + } + }, + "metrics": { + "f1": 0.828, + "falseNegative": 506, + "falsePositive": 69, + "precision": 0.9525, + "predictedPositive": 1453, + "recall": 0.7323, + "total": 2970, + "trueNegative": 1011, + "truePositive": 1384, + "wilsonPrecisionLower95": 0.9403 + }, + "threshold": 0.91 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 45, + "recall": 1.0, + "total": 1485, + "trueNegative": 1440, + "truePositive": 45, + "wilsonPrecisionLower95": 0.9213 + }, + "threshold": 0.66 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 45, + "recall": 1.0, + "total": 1485, + "trueNegative": 1440, + "truePositive": 45, + "wilsonPrecisionLower95": 0.9213 + }, + "threshold": 0.55 + } + }, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 90, + "recall": 1.0, + "total": 2970, + "trueNegative": 2880, + "truePositive": 90, + "wilsonPrecisionLower95": 0.9591 + }, + "threshold": 0.66 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8615, + "falseNegative": 77, + "falsePositive": 14, + "precision": 0.9529, + "predictedPositive": 297, + "recall": 0.7861, + "total": 1485, + "trueNegative": 1111, + "truePositive": 283, + "wilsonPrecisionLower95": 0.9224 + }, + "threshold": 0.805 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.6868, + "falseNegative": 333, + "falsePositive": 20, + "precision": 0.9509, + "predictedPositive": 407, + "recall": 0.5375, + "total": 2970, + "trueNegative": 2230, + "truePositive": 387, + "wilsonPrecisionLower95": 0.9253 + }, + "threshold": 0.925 + } + }, + "trainingRecords": 17708, + "validation": { + "intentsMeeting95Precision": 8, + "macroF1": 0.6491, + "macroPrecision": 0.967, + "macroRecall": 0.5465, + "minimumPrecision": 0.9315, + "perIntent": { + "blessing": { + "f1": 0.2353, + "falseNegative": 78, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 12, + "recall": 0.1333, + "total": 2970, + "trueNegative": 2880, + "truePositive": 12, + "wilsonPrecisionLower95": 0.7575 + }, + "complaint": { + "f1": 0.9407, + "falseNegative": 27, + "falsePositive": 15, + "precision": 0.9569, + "predictedPositive": 348, + "recall": 0.925, + "total": 2970, + "trueNegative": 2595, + "truePositive": 333, + "wilsonPrecisionLower95": 0.9301 + }, + "confirmationDecision": { + "f1": 0.7667, + "falseNegative": 65, + "falsePositive": 5, + "precision": 0.9583, + "predictedPositive": 120, + "recall": 0.6389, + "total": 2970, + "trueNegative": 2785, + "truePositive": 115, + "wilsonPrecisionLower95": 0.9062 + }, + "followUpReminder": { + "f1": 0.5375, + "falseNegative": 224, + "falsePositive": 10, + "precision": 0.9315, + "predictedPositive": 146, + "recall": 0.3778, + "total": 2970, + "trueNegative": 2600, + "truePositive": 136, + "wilsonPrecisionLower95": 0.8785 + }, + "invitation": { + "f1": 0.2692, + "falseNegative": 152, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 28, + "recall": 0.1556, + "total": 2970, + "trueNegative": 2790, + "truePositive": 28, + "wilsonPrecisionLower95": 0.8794 + }, + "question": { + "f1": 0.7156, + "falseNegative": 269, + "falsePositive": 18, + "precision": 0.9525, + "predictedPositive": 379, + "recall": 0.573, + "total": 2970, + "trueNegative": 2322, + "truePositive": 361, + "wilsonPrecisionLower95": 0.9262 + }, + "replyableMessage": { + "f1": 0.8205, + "falseNegative": 526, + "falsePositive": 71, + "precision": 0.9505, + "predictedPositive": 1435, + "recall": 0.7217, + "total": 2970, + "trueNegative": 1009, + "truePositive": 1364, + "wilsonPrecisionLower95": 0.9381 + }, + "scheduleNegotiation": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 90, + "recall": 1.0, + "total": 2970, + "trueNegative": 2880, + "truePositive": 90, + "wilsonPrecisionLower95": 0.9591 + }, + "task": { + "f1": 0.5565, + "falseNegative": 437, + "falsePositive": 14, + "precision": 0.9529, + "predictedPositive": 297, + "recall": 0.3931, + "total": 2970, + "trueNegative": 2236, + "truePositive": 283, + "wilsonPrecisionLower95": 0.9224 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 5e-06, + "augmentation": "punctuation", + "char_max": 7, + "char_min": 2, + "external_weight": 0.7100000000000001, + "hard_example_weight": 1.7, + "l1_ratio": 0.15, + "max_char_features": 70000, + "max_word_features": 28000, + "min_df": 2, + "pseudo_weight": 0.27, + "round": 8, + "word_max": 2 + }, + "featureCount": 83313, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 0 + }, + "complaint": { + "negative": 187, + "positive": 0 + }, + "confirmationDecision": { + "negative": 107, + "positive": 2 + }, + "followUpReminder": { + "negative": 250, + "positive": 0 + }, + "invitation": { + "negative": 250, + "positive": 0 + }, + "question": { + "negative": 40, + "positive": 5 + }, + "replyableMessage": { + "negative": 2, + "positive": 135 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 0 + }, + "task": { + "negative": 94, + "positive": 0 + } + }, + "round": 8, + "selectionObjective": 0.7837, + "syntheticRegression": { + "intentsMeeting95Precision": 4, + "macroF1": 0.6947, + "macroPrecision": 0.9526, + "macroRecall": 0.6066, + "minimumPrecision": 0.8509, + "perIntent": { + "blessing": { + "f1": 0.125, + "falseNegative": 84, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 6, + "recall": 0.0667, + "total": 2970, + "trueNegative": 2880, + "truePositive": 6, + "wilsonPrecisionLower95": 0.6097 + }, + "complaint": { + "f1": 0.9744, + "falseNegative": 18, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 342, + "recall": 0.95, + "total": 2970, + "trueNegative": 2610, + "truePositive": 342, + "wilsonPrecisionLower95": 0.9889 + }, + "confirmationDecision": { + "f1": 0.8035, + "falseNegative": 43, + "falsePositive": 24, + "precision": 0.8509, + "predictedPositive": 161, + "recall": 0.7611, + "total": 2970, + "trueNegative": 2766, + "truePositive": 137, + "wilsonPrecisionLower95": 0.7878 + }, + "followUpReminder": { + "f1": 0.5422, + "falseNegative": 222, + "falsePositive": 11, + "precision": 0.9262, + "predictedPositive": 149, + "recall": 0.3833, + "total": 2970, + "trueNegative": 2599, + "truePositive": 138, + "wilsonPrecisionLower95": 0.8726 + }, + "invitation": { + "f1": 0.5062, + "falseNegative": 119, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 61, + "recall": 0.3389, + "total": 2970, + "trueNegative": 2790, + "truePositive": 61, + "wilsonPrecisionLower95": 0.9408 + }, + "question": { + "f1": 0.7719, + "falseNegative": 229, + "falsePositive": 8, + "precision": 0.9804, + "predictedPositive": 409, + "recall": 0.6365, + "total": 2970, + "trueNegative": 2332, + "truePositive": 401, + "wilsonPrecisionLower95": 0.9619 + }, + "replyableMessage": { + "f1": 0.8832, + "falseNegative": 366, + "falsePositive": 37, + "precision": 0.9763, + "predictedPositive": 1561, + "recall": 0.8063, + "total": 2970, + "trueNegative": 1043, + "truePositive": 1524, + "wilsonPrecisionLower95": 0.9675 + }, + "scheduleNegotiation": { + "f1": 0.9162, + "falseNegative": 8, + "falsePositive": 7, + "precision": 0.9213, + "predictedPositive": 89, + "recall": 0.9111, + "total": 2970, + "trueNegative": 2873, + "truePositive": 82, + "wilsonPrecisionLower95": 0.8464 + }, + "task": { + "f1": 0.7297, + "falseNegative": 284, + "falsePositive": 39, + "precision": 0.9179, + "predictedPositive": 475, + "recall": 0.6056, + "total": 2970, + "trueNegative": 2211, + "truePositive": 436, + "wilsonPrecisionLower95": 0.8897 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.989, + "falseNegative": 1, + "falsePositive": 3, + "precision": 0.9835, + "predictedPositive": 182, + "recall": 0.9944, + "total": 1485, + "trueNegative": 1302, + "truePositive": 179, + "wilsonPrecisionLower95": 0.9527 + }, + "threshold": 0.97 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9405, + "falseNegative": 14, + "falsePositive": 7, + "precision": 0.9595, + "predictedPositive": 173, + "recall": 0.9222, + "total": 1485, + "trueNegative": 1298, + "truePositive": 166, + "wilsonPrecisionLower95": 0.9188 + }, + "threshold": 0.54 + } + }, + "metrics": { + "f1": 0.9237, + "falseNegative": 39, + "falsePositive": 14, + "precision": 0.9582, + "predictedPositive": 335, + "recall": 0.8917, + "total": 2970, + "trueNegative": 2596, + "truePositive": 321, + "wilsonPrecisionLower95": 0.9311 + }, + "threshold": 0.925 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9613, + "falseNegative": 3, + "falsePositive": 4, + "precision": 0.956, + "predictedPositive": 91, + "recall": 0.9667, + "total": 1485, + "trueNegative": 1391, + "truePositive": 87, + "wilsonPrecisionLower95": 0.8924 + }, + "threshold": 0.72 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.5556, + "falseNegative": 55, + "falsePositive": 1, + "precision": 0.9722, + "predictedPositive": 36, + "recall": 0.3889, + "total": 1485, + "trueNegative": 1394, + "truePositive": 35, + "wilsonPrecisionLower95": 0.8583 + }, + "threshold": 0.905 + } + }, + "metrics": { + "f1": 0.7922, + "falseNegative": 58, + "falsePositive": 6, + "precision": 0.9531, + "predictedPositive": 128, + "recall": 0.6778, + "total": 2970, + "trueNegative": 2784, + "truePositive": 122, + "wilsonPrecisionLower95": 0.9015 + }, + "threshold": 0.88 + }, + "followUpReminder": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8437, + "falseNegative": 45, + "falsePositive": 5, + "precision": 0.9643, + "predictedPositive": 140, + "recall": 0.75, + "total": 1485, + "trueNegative": 1300, + "truePositive": 135, + "wilsonPrecisionLower95": 0.9191 + }, + "threshold": 0.96 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.7063, + "falseNegative": 158, + "falsePositive": 10, + "precision": 0.9528, + "predictedPositive": 212, + "recall": 0.5611, + "total": 2970, + "trueNegative": 2600, + "truePositive": 202, + "wilsonPrecisionLower95": 0.9154 + }, + "threshold": 0.96 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.7157, + "falseNegative": 135, + "falsePositive": 8, + "precision": 0.9574, + "predictedPositive": 188, + "recall": 0.5714, + "total": 1485, + "trueNegative": 1162, + "truePositive": 180, + "wilsonPrecisionLower95": 0.9183 + }, + "threshold": 0.975 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.822, + "falseNegative": 91, + "falsePositive": 6, + "precision": 0.9739, + "predictedPositive": 230, + "recall": 0.7111, + "total": 1485, + "trueNegative": 1164, + "truePositive": 224, + "wilsonPrecisionLower95": 0.9443 + }, + "threshold": 0.505 + } + }, + "metrics": { + "f1": 0.8095, + "falseNegative": 186, + "falsePositive": 23, + "precision": 0.9507, + "predictedPositive": 467, + "recall": 0.7048, + "total": 2970, + "trueNegative": 2317, + "truePositive": 444, + "wilsonPrecisionLower95": 0.9272 + }, + "threshold": 0.71 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8741, + "falseNegative": 181, + "falsePositive": 39, + "precision": 0.9514, + "predictedPositive": 803, + "recall": 0.8085, + "total": 1485, + "trueNegative": 501, + "truePositive": 764, + "wilsonPrecisionLower95": 0.9343 + }, + "threshold": 0.95 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8775, + "falseNegative": 175, + "falsePositive": 40, + "precision": 0.9506, + "predictedPositive": 810, + "recall": 0.8148, + "total": 1485, + "trueNegative": 500, + "truePositive": 770, + "wilsonPrecisionLower95": 0.9335 + }, + "threshold": 0.88 + } + }, + "metrics": { + "f1": 0.8762, + "falseNegative": 354, + "falsePositive": 80, + "precision": 0.9505, + "predictedPositive": 1616, + "recall": 0.8127, + "total": 2970, + "trueNegative": 1000, + "truePositive": 1536, + "wilsonPrecisionLower95": 0.9388 + }, + "threshold": 0.905 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 45, + "recall": 1.0, + "total": 1485, + "trueNegative": 1440, + "truePositive": 45, + "wilsonPrecisionLower95": 0.9213 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9556, + "falseNegative": 2, + "falsePositive": 2, + "precision": 0.9556, + "predictedPositive": 45, + "recall": 0.9556, + "total": 1485, + "trueNegative": 1438, + "truePositive": 43, + "wilsonPrecisionLower95": 0.8517 + }, + "threshold": 0.75 + } + }, + "metrics": { + "f1": 0.9724, + "falseNegative": 2, + "falsePositive": 3, + "precision": 0.967, + "predictedPositive": 91, + "recall": 0.9778, + "total": 2970, + "trueNegative": 2877, + "truePositive": 88, + "wilsonPrecisionLower95": 0.9075 + }, + "threshold": 0.505 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8331, + "falseNegative": 93, + "falsePositive": 14, + "precision": 0.9502, + "predictedPositive": 281, + "recall": 0.7417, + "total": 1485, + "trueNegative": 1111, + "truePositive": 267, + "wilsonPrecisionLower95": 0.9181 + }, + "threshold": 0.95 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7445, + "falseNegative": 140, + "falsePositive": 11, + "precision": 0.9524, + "predictedPositive": 231, + "recall": 0.6111, + "total": 1485, + "trueNegative": 1114, + "truePositive": 220, + "wilsonPrecisionLower95": 0.9168 + }, + "threshold": 0.86 + } + }, + "metrics": { + "f1": 0.7672, + "falseNegative": 257, + "falsePositive": 24, + "precision": 0.9507, + "predictedPositive": 487, + "recall": 0.6431, + "total": 2970, + "trueNegative": 2226, + "truePositive": 463, + "wilsonPrecisionLower95": 0.9277 + }, + "threshold": 0.92 + } + }, + "trainingRecords": 17762, + "validation": { + "intentsMeeting95Precision": 8, + "macroF1": 0.698, + "macroPrecision": 0.9692, + "macroRecall": 0.6056, + "minimumPrecision": 0.9444, + "perIntent": { + "blessing": { + "f1": 0.2524, + "falseNegative": 77, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 13, + "recall": 0.1444, + "total": 2970, + "trueNegative": 2880, + "truePositive": 13, + "wilsonPrecisionLower95": 0.7719 + }, + "complaint": { + "f1": 0.965, + "falseNegative": 15, + "falsePositive": 10, + "precision": 0.9718, + "predictedPositive": 355, + "recall": 0.9583, + "total": 2970, + "trueNegative": 2600, + "truePositive": 345, + "wilsonPrecisionLower95": 0.9489 + }, + "confirmationDecision": { + "f1": 0.7948, + "falseNegative": 58, + "falsePositive": 5, + "precision": 0.9606, + "predictedPositive": 127, + "recall": 0.6778, + "total": 2970, + "trueNegative": 2785, + "truePositive": 122, + "wilsonPrecisionLower95": 0.9111 + }, + "followUpReminder": { + "f1": 0.5397, + "falseNegative": 224, + "falsePositive": 8, + "precision": 0.9444, + "predictedPositive": 144, + "recall": 0.3778, + "total": 2970, + "trueNegative": 2602, + "truePositive": 136, + "wilsonPrecisionLower95": 0.8942 + }, + "invitation": { + "f1": 0.3178, + "falseNegative": 146, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 34, + "recall": 0.1889, + "total": 2970, + "trueNegative": 2790, + "truePositive": 34, + "wilsonPrecisionLower95": 0.8985 + }, + "question": { + "f1": 0.771, + "falseNegative": 226, + "falsePositive": 14, + "precision": 0.9665, + "predictedPositive": 418, + "recall": 0.6413, + "total": 2970, + "trueNegative": 2326, + "truePositive": 404, + "wilsonPrecisionLower95": 0.9446 + }, + "replyableMessage": { + "f1": 0.8758, + "falseNegative": 356, + "falsePositive": 79, + "precision": 0.951, + "predictedPositive": 1613, + "recall": 0.8116, + "total": 2970, + "trueNegative": 1001, + "truePositive": 1534, + "wilsonPrecisionLower95": 0.9394 + }, + "scheduleNegotiation": { + "f1": 0.9778, + "falseNegative": 2, + "falsePositive": 2, + "precision": 0.9778, + "predictedPositive": 90, + "recall": 0.9778, + "total": 2970, + "trueNegative": 2878, + "truePositive": 88, + "wilsonPrecisionLower95": 0.9226 + }, + "task": { + "f1": 0.7876, + "falseNegative": 236, + "falsePositive": 25, + "precision": 0.9509, + "predictedPositive": 509, + "recall": 0.6722, + "total": 2970, + "trueNegative": 2225, + "truePositive": 484, + "wilsonPrecisionLower95": 0.9285 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 4e-06, + "augmentation": "punctuation", + "char_max": 7, + "char_min": 3, + "external_weight": 0.73, + "hard_example_weight": 1.7, + "l1_ratio": 0.15, + "max_char_features": 70000, + "max_word_features": 35000, + "min_df": 2, + "pseudo_weight": 0.28500000000000003, + "round": 9, + "word_max": 3 + }, + "featureCount": 92641, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 0 + }, + "complaint": { + "negative": 250, + "positive": 0 + }, + "confirmationDecision": { + "negative": 248, + "positive": 6 + }, + "followUpReminder": { + "negative": 250, + "positive": 1 + }, + "invitation": { + "negative": 250, + "positive": 0 + }, + "question": { + "negative": 146, + "positive": 8 + }, + "replyableMessage": { + "negative": 8, + "positive": 250 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 0 + }, + "task": { + "negative": 208, + "positive": 0 + } + }, + "round": 9, + "selectionObjective": 0.7717, + "syntheticRegression": { + "intentsMeeting95Precision": 4, + "macroF1": 0.6445, + "macroPrecision": 0.9357, + "macroRecall": 0.5386, + "minimumPrecision": 0.8583, + "perIntent": { + "blessing": { + "f1": 0.125, + "falseNegative": 84, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 6, + "recall": 0.0667, + "total": 2970, + "trueNegative": 2880, + "truePositive": 6, + "wilsonPrecisionLower95": 0.6097 + }, + "complaint": { + "f1": 0.949, + "falseNegative": 25, + "falsePositive": 11, + "precision": 0.9682, + "predictedPositive": 346, + "recall": 0.9306, + "total": 2970, + "trueNegative": 2599, + "truePositive": 335, + "wilsonPrecisionLower95": 0.944 + }, + "confirmationDecision": { + "f1": 0.7101, + "falseNegative": 71, + "falsePositive": 18, + "precision": 0.8583, + "predictedPositive": 127, + "recall": 0.6056, + "total": 2970, + "trueNegative": 2772, + "truePositive": 109, + "wilsonPrecisionLower95": 0.7871 + }, + "followUpReminder": { + "f1": 0.5244, + "falseNegative": 231, + "falsePositive": 3, + "precision": 0.9773, + "predictedPositive": 132, + "recall": 0.3583, + "total": 2970, + "trueNegative": 2607, + "truePositive": 129, + "wilsonPrecisionLower95": 0.9353 + }, + "invitation": { + "f1": 0.6286, + "falseNegative": 92, + "falsePositive": 12, + "precision": 0.88, + "predictedPositive": 100, + "recall": 0.4889, + "total": 2970, + "trueNegative": 2778, + "truePositive": 88, + "wilsonPrecisionLower95": 0.8019 + }, + "question": { + "f1": 0.6667, + "falseNegative": 310, + "falsePositive": 10, + "precision": 0.9697, + "predictedPositive": 330, + "recall": 0.5079, + "total": 2970, + "trueNegative": 2330, + "truePositive": 320, + "wilsonPrecisionLower95": 0.9451 + }, + "replyableMessage": { + "f1": 0.7919, + "falseNegative": 594, + "falsePositive": 87, + "precision": 0.9371, + "predictedPositive": 1383, + "recall": 0.6857, + "total": 2970, + "trueNegative": 993, + "truePositive": 1296, + "wilsonPrecisionLower95": 0.923 + }, + "scheduleNegotiation": { + "f1": 0.8941, + "falseNegative": 14, + "falsePositive": 4, + "precision": 0.95, + "predictedPositive": 80, + "recall": 0.8444, + "total": 2970, + "trueNegative": 2876, + "truePositive": 76, + "wilsonPrecisionLower95": 0.8784 + }, + "task": { + "f1": 0.5108, + "falseNegative": 461, + "falsePositive": 35, + "precision": 0.881, + "predictedPositive": 294, + "recall": 0.3597, + "total": 2970, + "trueNegative": 2215, + "truePositive": 259, + "wilsonPrecisionLower95": 0.8389 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9862, + "falseNegative": 1, + "falsePositive": 4, + "precision": 0.9781, + "predictedPositive": 183, + "recall": 0.9944, + "total": 1485, + "trueNegative": 1301, + "truePositive": 179, + "wilsonPrecisionLower95": 0.9452 + }, + "threshold": 0.74 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9222, + "falseNegative": 20, + "falsePositive": 7, + "precision": 0.9581, + "predictedPositive": 167, + "recall": 0.8889, + "total": 1485, + "trueNegative": 1298, + "truePositive": 160, + "wilsonPrecisionLower95": 0.916 + }, + "threshold": 0.62 + } + }, + "metrics": { + "f1": 0.9528, + "falseNegative": 17, + "falsePositive": 17, + "precision": 0.9528, + "predictedPositive": 360, + "recall": 0.9528, + "total": 2970, + "trueNegative": 2593, + "truePositive": 343, + "wilsonPrecisionLower95": 0.9257 + }, + "threshold": 0.5 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9492, + "falseNegative": 6, + "falsePositive": 3, + "precision": 0.9655, + "predictedPositive": 87, + "recall": 0.9333, + "total": 1485, + "trueNegative": 1392, + "truePositive": 84, + "wilsonPrecisionLower95": 0.9035 + }, + "threshold": 0.965 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7042, + "falseNegative": 40, + "falsePositive": 2, + "precision": 0.9615, + "predictedPositive": 52, + "recall": 0.5556, + "total": 1485, + "trueNegative": 1393, + "truePositive": 50, + "wilsonPrecisionLower95": 0.8702 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.76, + "falseNegative": 66, + "falsePositive": 6, + "precision": 0.95, + "predictedPositive": 120, + "recall": 0.6333, + "total": 2970, + "trueNegative": 2784, + "truePositive": 114, + "wilsonPrecisionLower95": 0.8952 + }, + "threshold": 0.945 + }, + "followUpReminder": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8243, + "falseNegative": 51, + "falsePositive": 4, + "precision": 0.9699, + "predictedPositive": 133, + "recall": 0.7167, + "total": 1485, + "trueNegative": 1301, + "truePositive": 129, + "wilsonPrecisionLower95": 0.9252 + }, + "threshold": 0.99 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.6316, + "falseNegative": 192, + "falsePositive": 4, + "precision": 0.9767, + "predictedPositive": 172, + "recall": 0.4667, + "total": 2970, + "trueNegative": 2606, + "truePositive": 168, + "wilsonPrecisionLower95": 0.9417 + }, + "threshold": 0.995 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.544, + "falseNegative": 56, + "falsePositive": 1, + "precision": 0.9714, + "predictedPositive": 35, + "recall": 0.3778, + "total": 1485, + "trueNegative": 1394, + "truePositive": 34, + "wilsonPrecisionLower95": 0.8547 + }, + "threshold": 0.82 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.6694, + "falseNegative": 153, + "falsePositive": 7, + "precision": 0.9586, + "predictedPositive": 169, + "recall": 0.5143, + "total": 1485, + "trueNegative": 1163, + "truePositive": 162, + "wilsonPrecisionLower95": 0.917 + }, + "threshold": 0.985 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6762, + "falseNegative": 150, + "falsePositive": 8, + "precision": 0.9538, + "predictedPositive": 173, + "recall": 0.5238, + "total": 1485, + "trueNegative": 1162, + "truePositive": 165, + "wilsonPrecisionLower95": 0.9114 + }, + "threshold": 0.64 + } + }, + "metrics": { + "f1": 0.5392, + "falseNegative": 393, + "falsePositive": 12, + "precision": 0.9518, + "predictedPositive": 249, + "recall": 0.3762, + "total": 2970, + "trueNegative": 2328, + "truePositive": 237, + "wilsonPrecisionLower95": 0.9177 + }, + "threshold": 0.965 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9407, + "falseNegative": 65, + "falsePositive": 46, + "precision": 0.9503, + "predictedPositive": 926, + "recall": 0.9312, + "total": 1485, + "trueNegative": 494, + "truePositive": 880, + "wilsonPrecisionLower95": 0.9344 + }, + "threshold": 0.745 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6118, + "falseNegative": 521, + "falsePositive": 17, + "precision": 0.9615, + "predictedPositive": 441, + "recall": 0.4487, + "total": 1485, + "trueNegative": 523, + "truePositive": 424, + "wilsonPrecisionLower95": 0.9391 + }, + "threshold": 0.965 + } + }, + "metrics": { + "f1": 0.8037, + "falseNegative": 576, + "falsePositive": 66, + "precision": 0.9522, + "predictedPositive": 1380, + "recall": 0.6952, + "total": 2970, + "trueNegative": 1014, + "truePositive": 1314, + "wilsonPrecisionLower95": 0.9396 + }, + "threshold": 0.95 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9286, + "falseNegative": 6, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 39, + "recall": 0.8667, + "total": 1485, + "trueNegative": 1440, + "truePositive": 39, + "wilsonPrecisionLower95": 0.9103 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9302, + "falseNegative": 5, + "falsePositive": 1, + "precision": 0.9756, + "predictedPositive": 41, + "recall": 0.8889, + "total": 1485, + "trueNegative": 1439, + "truePositive": 40, + "wilsonPrecisionLower95": 0.874 + }, + "threshold": 0.61 + } + }, + "metrics": { + "f1": 0.9294, + "falseNegative": 11, + "falsePositive": 1, + "precision": 0.9875, + "predictedPositive": 80, + "recall": 0.8778, + "total": 2970, + "trueNegative": 2879, + "truePositive": 79, + "wilsonPrecisionLower95": 0.9325 + }, + "threshold": 0.61 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8403, + "falseNegative": 89, + "falsePositive": 14, + "precision": 0.9509, + "predictedPositive": 285, + "recall": 0.7528, + "total": 1485, + "trueNegative": 1111, + "truePositive": 271, + "wilsonPrecisionLower95": 0.9192 + }, + "threshold": 0.93 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.2384, + "falseNegative": 311, + "falsePositive": 2, + "precision": 0.9608, + "predictedPositive": 51, + "recall": 0.1361, + "total": 1485, + "trueNegative": 1123, + "truePositive": 49, + "wilsonPrecisionLower95": 0.8678 + }, + "threshold": 0.995 + } + }, + "metrics": { + "f1": 0.6415, + "falseNegative": 372, + "falsePositive": 17, + "precision": 0.9534, + "predictedPositive": 365, + "recall": 0.4833, + "total": 2970, + "trueNegative": 2233, + "truePositive": 348, + "wilsonPrecisionLower95": 0.9267 + }, + "threshold": 0.975 + } + }, + "trainingRecords": 17823, + "validation": { + "intentsMeeting95Precision": 8, + "macroF1": 0.6858, + "macroPrecision": 0.9638, + "macroRecall": 0.5714, + "minimumPrecision": 0.9211, + "perIntent": { + "blessing": { + "f1": 0.2857, + "falseNegative": 75, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 15, + "recall": 0.1667, + "total": 2970, + "trueNegative": 2880, + "truePositive": 15, + "wilsonPrecisionLower95": 0.7961 + }, + "complaint": { + "f1": 0.9549, + "falseNegative": 21, + "falsePositive": 11, + "precision": 0.9686, + "predictedPositive": 350, + "recall": 0.9417, + "total": 2970, + "trueNegative": 2599, + "truePositive": 339, + "wilsonPrecisionLower95": 0.9446 + }, + "confirmationDecision": { + "f1": 0.8401, + "falseNegative": 46, + "falsePositive": 5, + "precision": 0.964, + "predictedPositive": 139, + "recall": 0.7444, + "total": 2970, + "trueNegative": 2785, + "truePositive": 134, + "wilsonPrecisionLower95": 0.9186 + }, + "followUpReminder": { + "f1": 0.5352, + "falseNegative": 227, + "falsePositive": 4, + "precision": 0.9708, + "predictedPositive": 137, + "recall": 0.3694, + "total": 2970, + "trueNegative": 2606, + "truePositive": 133, + "wilsonPrecisionLower95": 0.9273 + }, + "invitation": { + "f1": 0.5469, + "falseNegative": 110, + "falsePositive": 6, + "precision": 0.9211, + "predictedPositive": 76, + "recall": 0.3889, + "total": 2970, + "trueNegative": 2784, + "truePositive": 70, + "wilsonPrecisionLower95": 0.8383 + }, + "question": { + "f1": 0.6728, + "falseNegative": 303, + "falsePositive": 15, + "precision": 0.9561, + "predictedPositive": 342, + "recall": 0.519, + "total": 2970, + "trueNegative": 2325, + "truePositive": 327, + "wilsonPrecisionLower95": 0.9289 + }, + "replyableMessage": { + "f1": 0.8007, + "falseNegative": 586, + "falsePositive": 63, + "precision": 0.9539, + "predictedPositive": 1367, + "recall": 0.6899, + "total": 2970, + "trueNegative": 1017, + "truePositive": 1304, + "wilsonPrecisionLower95": 0.9415 + }, + "scheduleNegotiation": { + "f1": 0.9294, + "falseNegative": 11, + "falsePositive": 1, + "precision": 0.9875, + "predictedPositive": 80, + "recall": 0.8778, + "total": 2970, + "trueNegative": 2879, + "truePositive": 79, + "wilsonPrecisionLower95": 0.9325 + }, + "task": { + "f1": 0.6061, + "falseNegative": 400, + "falsePositive": 16, + "precision": 0.9524, + "predictedPositive": 336, + "recall": 0.4444, + "total": 2970, + "trueNegative": 2234, + "truePositive": 320, + "wilsonPrecisionLower95": 0.9241 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 6e-06, + "augmentation": "punctuation", + "char_max": 6, + "char_min": 2, + "external_weight": 0.75, + "hard_example_weight": 1.8, + "l1_ratio": 0.2, + "max_char_features": 75000, + "max_word_features": 38000, + "min_df": 1, + "pseudo_weight": 0.3, + "round": 10, + "word_max": 3 + }, + "featureCount": 113000, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 0 + }, + "complaint": { + "negative": 250, + "positive": 0 + }, + "confirmationDecision": { + "negative": 250, + "positive": 11 + }, + "followUpReminder": { + "negative": 250, + "positive": 4 + }, + "invitation": { + "negative": 250, + "positive": 0 + }, + "question": { + "negative": 182, + "positive": 10 + }, + "replyableMessage": { + "negative": 11, + "positive": 250 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 0 + }, + "task": { + "negative": 250, + "positive": 1 + } + }, + "round": 10, + "selectionObjective": 0.7542, + "syntheticRegression": { + "intentsMeeting95Precision": 4, + "macroF1": 0.6448, + "macroPrecision": 0.9609, + "macroRecall": 0.5363, + "minimumPrecision": 0.8734, + "perIntent": { + "blessing": { + "f1": 0.125, + "falseNegative": 84, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 6, + "recall": 0.0667, + "total": 2970, + "trueNegative": 2880, + "truePositive": 6, + "wilsonPrecisionLower95": 0.6097 + }, + "complaint": { + "f1": 0.9366, + "falseNegative": 20, + "falsePositive": 26, + "precision": 0.929, + "predictedPositive": 366, + "recall": 0.9444, + "total": 2970, + "trueNegative": 2584, + "truePositive": 340, + "wilsonPrecisionLower95": 0.8979 + }, + "confirmationDecision": { + "f1": 0.8166, + "falseNegative": 42, + "falsePositive": 20, + "precision": 0.8734, + "predictedPositive": 158, + "recall": 0.7667, + "total": 2970, + "trueNegative": 2770, + "truePositive": 138, + "wilsonPrecisionLower95": 0.8126 + }, + "followUpReminder": { + "f1": 0.5397, + "falseNegative": 224, + "falsePositive": 8, + "precision": 0.9444, + "predictedPositive": 144, + "recall": 0.3778, + "total": 2970, + "trueNegative": 2602, + "truePositive": 136, + "wilsonPrecisionLower95": 0.8942 + }, + "invitation": { + "f1": 0.5124, + "falseNegative": 118, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 62, + "recall": 0.3444, + "total": 2970, + "trueNegative": 2790, + "truePositive": 62, + "wilsonPrecisionLower95": 0.9417 + }, + "question": { + "f1": 0.5488, + "falseNegative": 391, + "falsePositive": 2, + "precision": 0.9917, + "predictedPositive": 241, + "recall": 0.3794, + "total": 2970, + "trueNegative": 2338, + "truePositive": 239, + "wilsonPrecisionLower95": 0.9703 + }, + "replyableMessage": { + "f1": 0.7216, + "falseNegative": 808, + "falsePositive": 27, + "precision": 0.9757, + "predictedPositive": 1109, + "recall": 0.5725, + "total": 2970, + "trueNegative": 1053, + "truePositive": 1082, + "wilsonPrecisionLower95": 0.9648 + }, + "scheduleNegotiation": { + "f1": 0.882, + "falseNegative": 19, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 71, + "recall": 0.7889, + "total": 2970, + "trueNegative": 2880, + "truePositive": 71, + "wilsonPrecisionLower95": 0.9487 + }, + "task": { + "f1": 0.7201, + "falseNegative": 298, + "falsePositive": 30, + "precision": 0.9336, + "predictedPositive": 452, + "recall": 0.5861, + "total": 2970, + "trueNegative": 2220, + "truePositive": 422, + "wilsonPrecisionLower95": 0.9068 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9807, + "falseNegative": 2, + "falsePositive": 5, + "precision": 0.9727, + "predictedPositive": 183, + "recall": 0.9889, + "total": 1485, + "trueNegative": 1300, + "truePositive": 178, + "wilsonPrecisionLower95": 0.9376 + }, + "threshold": 0.765 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9288, + "falseNegative": 17, + "falsePositive": 8, + "precision": 0.9532, + "predictedPositive": 171, + "recall": 0.9056, + "total": 1485, + "trueNegative": 1297, + "truePositive": 163, + "wilsonPrecisionLower95": 0.9104 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.9455, + "falseNegative": 22, + "falsePositive": 17, + "precision": 0.9521, + "predictedPositive": 355, + "recall": 0.9389, + "total": 2970, + "trueNegative": 2593, + "truePositive": 338, + "wilsonPrecisionLower95": 0.9247 + }, + "threshold": 0.595 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.7568, + "falseNegative": 34, + "falsePositive": 2, + "precision": 0.9655, + "predictedPositive": 58, + "recall": 0.6222, + "total": 1485, + "trueNegative": 1393, + "truePositive": 56, + "wilsonPrecisionLower95": 0.8827 + }, + "threshold": 0.955 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7733, + "falseNegative": 32, + "falsePositive": 2, + "precision": 0.9667, + "predictedPositive": 60, + "recall": 0.6444, + "total": 1485, + "trueNegative": 1393, + "truePositive": 58, + "wilsonPrecisionLower95": 0.8864 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.7723, + "falseNegative": 63, + "falsePositive": 6, + "precision": 0.9512, + "predictedPositive": 123, + "recall": 0.65, + "total": 2970, + "trueNegative": 2784, + "truePositive": 117, + "wilsonPrecisionLower95": 0.8977 + }, + "threshold": 0.765 + }, + "followUpReminder": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8483, + "falseNegative": 43, + "falsePositive": 6, + "precision": 0.958, + "predictedPositive": 143, + "recall": 0.7611, + "total": 1485, + "trueNegative": 1299, + "truePositive": 137, + "wilsonPrecisionLower95": 0.9115 + }, + "threshold": 0.995 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.3529, + "falseNegative": 141, + "falsePositive": 2, + "precision": 0.9512, + "predictedPositive": 41, + "recall": 0.2167, + "total": 1485, + "trueNegative": 1303, + "truePositive": 39, + "wilsonPrecisionLower95": 0.8386 + }, + "threshold": 0.99 + } + }, + "metrics": { + "f1": 0.6292, + "falseNegative": 192, + "falsePositive": 6, + "precision": 0.9655, + "predictedPositive": 174, + "recall": 0.4667, + "total": 2970, + "trueNegative": 2604, + "truePositive": 168, + "wilsonPrecisionLower95": 0.9268 + }, + "threshold": 0.995 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.3894, + "falseNegative": 68, + "falsePositive": 1, + "precision": 0.9565, + "predictedPositive": 23, + "recall": 0.2444, + "total": 1485, + "trueNegative": 1394, + "truePositive": 22, + "wilsonPrecisionLower95": 0.7901 + }, + "threshold": 0.98 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8175, + "falseNegative": 91, + "falsePositive": 9, + "precision": 0.9614, + "predictedPositive": 233, + "recall": 0.7111, + "total": 1485, + "trueNegative": 1161, + "truePositive": 224, + "wilsonPrecisionLower95": 0.9282 + }, + "threshold": 0.6 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.7182, + "falseNegative": 401, + "falsePositive": 26, + "precision": 0.9544, + "predictedPositive": 570, + "recall": 0.5757, + "total": 1485, + "trueNegative": 514, + "truePositive": 544, + "wilsonPrecisionLower95": 0.934 + }, + "threshold": 0.985 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6667, + "falseNegative": 460, + "falsePositive": 25, + "precision": 0.951, + "predictedPositive": 510, + "recall": 0.5132, + "total": 1485, + "trueNegative": 515, + "truePositive": 485, + "wilsonPrecisionLower95": 0.9286 + }, + "threshold": 0.945 + } + }, + "metrics": { + "f1": 0.7151, + "falseNegative": 807, + "falsePositive": 56, + "precision": 0.9508, + "predictedPositive": 1139, + "recall": 0.573, + "total": 2970, + "trueNegative": 1024, + "truePositive": 1083, + "wilsonPrecisionLower95": 0.9367 + }, + "threshold": 0.96 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9412, + "falseNegative": 5, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 40, + "recall": 0.8889, + "total": 1485, + "trueNegative": 1440, + "truePositive": 40, + "wilsonPrecisionLower95": 0.9124 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8052, + "falseNegative": 14, + "falsePositive": 1, + "precision": 0.9688, + "predictedPositive": 32, + "recall": 0.6889, + "total": 1485, + "trueNegative": 1439, + "truePositive": 31, + "wilsonPrecisionLower95": 0.8426 + }, + "threshold": 0.98 + } + }, + "metrics": { + "f1": 0.8375, + "falseNegative": 23, + "falsePositive": 3, + "precision": 0.9571, + "predictedPositive": 70, + "recall": 0.7444, + "total": 2970, + "trueNegative": 2877, + "truePositive": 67, + "wilsonPrecisionLower95": 0.8814 + }, + "threshold": 0.92 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.7783, + "falseNegative": 123, + "falsePositive": 12, + "precision": 0.9518, + "predictedPositive": 249, + "recall": 0.6583, + "total": 1485, + "trueNegative": 1113, + "truePositive": 237, + "wilsonPrecisionLower95": 0.9177 + }, + "threshold": 0.84 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7994, + "falseNegative": 113, + "falsePositive": 11, + "precision": 0.9574, + "predictedPositive": 258, + "recall": 0.6861, + "total": 1485, + "trueNegative": 1114, + "truePositive": 247, + "wilsonPrecisionLower95": 0.9253 + }, + "threshold": 0.755 + } + }, + "metrics": { + "f1": 0.7977, + "falseNegative": 225, + "falsePositive": 26, + "precision": 0.9501, + "predictedPositive": 521, + "recall": 0.6875, + "total": 2970, + "trueNegative": 2224, + "truePositive": 495, + "wilsonPrecisionLower95": 0.9279 + }, + "threshold": 0.77 + } + }, + "trainingRecords": 17763, + "validation": { + "intentsMeeting95Precision": 8, + "macroF1": 0.6563, + "macroPrecision": 0.9695, + "macroRecall": 0.5386, + "minimumPrecision": 0.9528, + "perIntent": { + "blessing": { + "f1": 0.1818, + "falseNegative": 81, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 9, + "recall": 0.1, + "total": 2970, + "trueNegative": 2880, + "truePositive": 9, + "wilsonPrecisionLower95": 0.7009 + }, + "complaint": { + "f1": 0.9552, + "falseNegative": 19, + "falsePositive": 13, + "precision": 0.9633, + "predictedPositive": 354, + "recall": 0.9472, + "total": 2970, + "trueNegative": 2597, + "truePositive": 341, + "wilsonPrecisionLower95": 0.9382 + }, + "confirmationDecision": { + "f1": 0.7651, + "falseNegative": 66, + "falsePositive": 4, + "precision": 0.9661, + "predictedPositive": 118, + "recall": 0.6333, + "total": 2970, + "trueNegative": 2786, + "truePositive": 114, + "wilsonPrecisionLower95": 0.9161 + }, + "followUpReminder": { + "f1": 0.6471, + "falseNegative": 184, + "falsePositive": 8, + "precision": 0.9565, + "predictedPositive": 184, + "recall": 0.4889, + "total": 2970, + "trueNegative": 2602, + "truePositive": 176, + "wilsonPrecisionLower95": 0.9166 + }, + "invitation": { + "f1": 0.479, + "falseNegative": 123, + "falsePositive": 1, + "precision": 0.9828, + "predictedPositive": 58, + "recall": 0.3167, + "total": 2970, + "trueNegative": 2789, + "truePositive": 57, + "wilsonPrecisionLower95": 0.9086 + }, + "question": { + "f1": 0.5191, + "falseNegative": 406, + "falsePositive": 9, + "precision": 0.9614, + "predictedPositive": 233, + "recall": 0.3556, + "total": 2970, + "trueNegative": 2331, + "truePositive": 224, + "wilsonPrecisionLower95": 0.9282 + }, + "replyableMessage": { + "f1": 0.6929, + "falseNegative": 861, + "falsePositive": 51, + "precision": 0.9528, + "predictedPositive": 1080, + "recall": 0.5444, + "total": 2970, + "trueNegative": 1029, + "truePositive": 1029, + "wilsonPrecisionLower95": 0.9384 + }, + "scheduleNegotiation": { + "f1": 0.8765, + "falseNegative": 19, + "falsePositive": 1, + "precision": 0.9861, + "predictedPositive": 72, + "recall": 0.7889, + "total": 2970, + "trueNegative": 2879, + "truePositive": 71, + "wilsonPrecisionLower95": 0.9254 + }, + "task": { + "f1": 0.7896, + "falseNegative": 236, + "falsePositive": 22, + "precision": 0.9565, + "predictedPositive": 506, + "recall": 0.6722, + "total": 2970, + "trueNegative": 2228, + "truePositive": 484, + "wilsonPrecisionLower95": 0.935 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 5e-06, + "augmentation": "prefix", + "char_max": 6, + "char_min": 2, + "external_weight": 0.77, + "hard_example_weight": 1.8, + "l1_ratio": 0.1, + "max_char_features": 70000, + "max_word_features": 30000, + "min_df": 2, + "pseudo_weight": 0.31499999999999995, + "round": 11, + "word_max": 2 + }, + "featureCount": 81451, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 0 + }, + "complaint": { + "negative": 250, + "positive": 0 + }, + "confirmationDecision": { + "negative": 250, + "positive": 14 + }, + "followUpReminder": { + "negative": 250, + "positive": 2 + }, + "invitation": { + "negative": 250, + "positive": 1 + }, + "question": { + "negative": 233, + "positive": 0 + }, + "replyableMessage": { + "negative": 8, + "positive": 250 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 0 + }, + "task": { + "negative": 250, + "positive": 0 + } + }, + "round": 11, + "selectionObjective": 0.8015, + "syntheticRegression": { + "intentsMeeting95Precision": 4, + "macroF1": 0.6807, + "macroPrecision": 0.963, + "macroRecall": 0.6053, + "minimumPrecision": 0.8782, + "perIntent": { + "blessing": { + "f1": 0.125, + "falseNegative": 84, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 6, + "recall": 0.0667, + "total": 2970, + "trueNegative": 2880, + "truePositive": 6, + "wilsonPrecisionLower95": 0.6097 + }, + "complaint": { + "f1": 0.9655, + "falseNegative": 24, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 336, + "recall": 0.9333, + "total": 2970, + "trueNegative": 2610, + "truePositive": 336, + "wilsonPrecisionLower95": 0.9887 + }, + "confirmationDecision": { + "f1": 0.8155, + "falseNegative": 43, + "falsePositive": 19, + "precision": 0.8782, + "predictedPositive": 156, + "recall": 0.7611, + "total": 2970, + "trueNegative": 2771, + "truePositive": 137, + "wilsonPrecisionLower95": 0.8176 + }, + "followUpReminder": { + "f1": 0.2439, + "falseNegative": 310, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 50, + "recall": 0.1389, + "total": 2970, + "trueNegative": 2610, + "truePositive": 50, + "wilsonPrecisionLower95": 0.9287 + }, + "invitation": { + "f1": 0.5425, + "falseNegative": 113, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 67, + "recall": 0.3722, + "total": 2970, + "trueNegative": 2790, + "truePositive": 67, + "wilsonPrecisionLower95": 0.9458 + }, + "question": { + "f1": 0.8494, + "falseNegative": 162, + "falsePositive": 4, + "precision": 0.9915, + "predictedPositive": 472, + "recall": 0.7429, + "total": 2970, + "trueNegative": 2336, + "truePositive": 468, + "wilsonPrecisionLower95": 0.9784 + }, + "replyableMessage": { + "f1": 0.8975, + "falseNegative": 253, + "falsePositive": 121, + "precision": 0.9312, + "predictedPositive": 1758, + "recall": 0.8661, + "total": 2970, + "trueNegative": 959, + "truePositive": 1637, + "wilsonPrecisionLower95": 0.9184 + }, + "scheduleNegotiation": { + "f1": 0.9444, + "falseNegative": 5, + "falsePositive": 5, + "precision": 0.9444, + "predictedPositive": 90, + "recall": 0.9444, + "total": 2970, + "trueNegative": 2875, + "truePositive": 85, + "wilsonPrecisionLower95": 0.8765 + }, + "task": { + "f1": 0.743, + "falseNegative": 272, + "falsePositive": 38, + "precision": 0.9218, + "predictedPositive": 486, + "recall": 0.6222, + "total": 2970, + "trueNegative": 2212, + "truePositive": 448, + "wilsonPrecisionLower95": 0.8945 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9917, + "falseNegative": 0, + "falsePositive": 3, + "precision": 0.9836, + "predictedPositive": 183, + "recall": 1.0, + "total": 1485, + "trueNegative": 1302, + "truePositive": 180, + "wilsonPrecisionLower95": 0.9529 + }, + "threshold": 0.99 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9318, + "falseNegative": 16, + "falsePositive": 8, + "precision": 0.9535, + "predictedPositive": 172, + "recall": 0.9111, + "total": 1485, + "trueNegative": 1297, + "truePositive": 164, + "wilsonPrecisionLower95": 0.9109 + }, + "threshold": 0.84 + } + }, + "metrics": { + "f1": 0.9529, + "falseNegative": 16, + "falsePositive": 18, + "precision": 0.9503, + "predictedPositive": 362, + "recall": 0.9556, + "total": 2970, + "trueNegative": 2592, + "truePositive": 344, + "wilsonPrecisionLower95": 0.9228 + }, + "threshold": 0.84 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.931, + "falseNegative": 9, + "falsePositive": 3, + "precision": 0.9643, + "predictedPositive": 84, + "recall": 0.9, + "total": 1485, + "trueNegative": 1392, + "truePositive": 81, + "wilsonPrecisionLower95": 0.9002 + }, + "threshold": 0.97 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7222, + "falseNegative": 38, + "falsePositive": 2, + "precision": 0.963, + "predictedPositive": 54, + "recall": 0.5778, + "total": 1485, + "trueNegative": 1393, + "truePositive": 52, + "wilsonPrecisionLower95": 0.8746 + }, + "threshold": 0.86 + } + }, + "metrics": { + "f1": 0.8077, + "falseNegative": 54, + "falsePositive": 6, + "precision": 0.9545, + "predictedPositive": 132, + "recall": 0.7, + "total": 2970, + "trueNegative": 2784, + "truePositive": 126, + "wilsonPrecisionLower95": 0.9044 + }, + "threshold": 0.95 + }, + "followUpReminder": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.4833, + "falseNegative": 61, + "falsePositive": 1, + "precision": 0.9667, + "predictedPositive": 30, + "recall": 0.3222, + "total": 1485, + "trueNegative": 1394, + "truePositive": 29, + "wilsonPrecisionLower95": 0.8333 + }, + "threshold": 0.965 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.7993, + "falseNegative": 98, + "falsePositive": 11, + "precision": 0.9518, + "predictedPositive": 228, + "recall": 0.6889, + "total": 1485, + "trueNegative": 1159, + "truePositive": 217, + "wilsonPrecisionLower95": 0.9157 + }, + "threshold": 0.92 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7572, + "falseNegative": 117, + "falsePositive": 10, + "precision": 0.9519, + "predictedPositive": 208, + "recall": 0.6286, + "total": 1485, + "trueNegative": 1160, + "truePositive": 198, + "wilsonPrecisionLower95": 0.9138 + }, + "threshold": 0.735 + } + }, + "metrics": { + "f1": 0.797, + "falseNegative": 198, + "falsePositive": 22, + "precision": 0.9515, + "predictedPositive": 454, + "recall": 0.6857, + "total": 2970, + "trueNegative": 2318, + "truePositive": 432, + "wilsonPrecisionLower95": 0.9277 + }, + "threshold": 0.8 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9373, + "falseNegative": 71, + "falsePositive": 46, + "precision": 0.95, + "predictedPositive": 920, + "recall": 0.9249, + "total": 1485, + "trueNegative": 494, + "truePositive": 874, + "wilsonPrecisionLower95": 0.934 + }, + "threshold": 0.79 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8754, + "falseNegative": 179, + "falsePositive": 39, + "precision": 0.9516, + "predictedPositive": 805, + "recall": 0.8106, + "total": 1485, + "trueNegative": 501, + "truePositive": 766, + "wilsonPrecisionLower95": 0.9345 + }, + "threshold": 0.91 + } + }, + "metrics": { + "f1": 0.9105, + "falseNegative": 242, + "falsePositive": 82, + "precision": 0.9526, + "predictedPositive": 1730, + "recall": 0.872, + "total": 2970, + "trueNegative": 998, + "truePositive": 1648, + "wilsonPrecisionLower95": 0.9415 + }, + "threshold": 0.895 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9773, + "falseNegative": 2, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 43, + "recall": 0.9556, + "total": 1485, + "trueNegative": 1440, + "truePositive": 43, + "wilsonPrecisionLower95": 0.918 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9318, + "falseNegative": 4, + "falsePositive": 2, + "precision": 0.9535, + "predictedPositive": 43, + "recall": 0.9111, + "total": 1485, + "trueNegative": 1438, + "truePositive": 41, + "wilsonPrecisionLower95": 0.8454 + }, + "threshold": 0.9 + } + }, + "metrics": { + "f1": 0.9551, + "falseNegative": 5, + "falsePositive": 3, + "precision": 0.9659, + "predictedPositive": 88, + "recall": 0.9444, + "total": 2970, + "trueNegative": 2877, + "truePositive": 85, + "wilsonPrecisionLower95": 0.9045 + }, + "threshold": 0.885 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8315, + "falseNegative": 96, + "falsePositive": 11, + "precision": 0.96, + "predictedPositive": 275, + "recall": 0.7333, + "total": 1485, + "trueNegative": 1114, + "truePositive": 264, + "wilsonPrecisionLower95": 0.9298 + }, + "threshold": 0.985 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8349, + "falseNegative": 92, + "falsePositive": 14, + "precision": 0.9504, + "predictedPositive": 282, + "recall": 0.7444, + "total": 1485, + "trueNegative": 1111, + "truePositive": 268, + "wilsonPrecisionLower95": 0.9184 + }, + "threshold": 0.845 + } + }, + "metrics": { + "f1": 0.7688, + "falseNegative": 256, + "falsePositive": 23, + "precision": 0.9528, + "predictedPositive": 487, + "recall": 0.6444, + "total": 2970, + "trueNegative": 2227, + "truePositive": 464, + "wilsonPrecisionLower95": 0.9301 + }, + "threshold": 0.975 + } + }, + "trainingRecords": 15345, + "validation": { + "intentsMeeting95Precision": 9, + "macroF1": 0.7269, + "macroPrecision": 0.9659, + "macroRecall": 0.6366, + "minimumPrecision": 0.9507, + "perIntent": { + "blessing": { + "f1": 0.2692, + "falseNegative": 76, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 14, + "recall": 0.1556, + "total": 2970, + "trueNegative": 2880, + "truePositive": 14, + "wilsonPrecisionLower95": 0.7847 + }, + "complaint": { + "f1": 0.9622, + "falseNegative": 16, + "falsePositive": 11, + "precision": 0.969, + "predictedPositive": 355, + "recall": 0.9556, + "total": 2970, + "trueNegative": 2599, + "truePositive": 344, + "wilsonPrecisionLower95": 0.9454 + }, + "confirmationDecision": { + "f1": 0.8365, + "falseNegative": 47, + "falsePositive": 5, + "precision": 0.9638, + "predictedPositive": 138, + "recall": 0.7389, + "total": 2970, + "trueNegative": 2785, + "truePositive": 133, + "wilsonPrecisionLower95": 0.918 + }, + "followUpReminder": { + "f1": 0.4335, + "falseNegative": 259, + "falsePositive": 5, + "precision": 0.9528, + "predictedPositive": 106, + "recall": 0.2806, + "total": 2970, + "trueNegative": 2605, + "truePositive": 101, + "wilsonPrecisionLower95": 0.8943 + }, + "invitation": { + "f1": 0.5669, + "falseNegative": 108, + "falsePositive": 2, + "precision": 0.973, + "predictedPositive": 74, + "recall": 0.4, + "total": 2970, + "trueNegative": 2788, + "truePositive": 72, + "wilsonPrecisionLower95": 0.9067 + }, + "question": { + "f1": 0.7786, + "falseNegative": 215, + "falsePositive": 21, + "precision": 0.9518, + "predictedPositive": 436, + "recall": 0.6587, + "total": 2970, + "trueNegative": 2319, + "truePositive": 415, + "wilsonPrecisionLower95": 0.9275 + }, + "replyableMessage": { + "f1": 0.9073, + "falseNegative": 250, + "falsePositive": 85, + "precision": 0.9507, + "predictedPositive": 1725, + "recall": 0.8677, + "total": 2970, + "trueNegative": 995, + "truePositive": 1640, + "wilsonPrecisionLower95": 0.9395 + }, + "scheduleNegotiation": { + "f1": 0.9545, + "falseNegative": 6, + "falsePositive": 2, + "precision": 0.9767, + "predictedPositive": 86, + "recall": 0.9333, + "total": 2970, + "trueNegative": 2878, + "truePositive": 84, + "wilsonPrecisionLower95": 0.9191 + }, + "task": { + "f1": 0.8332, + "falseNegative": 188, + "falsePositive": 25, + "precision": 0.9551, + "predictedPositive": 557, + "recall": 0.7389, + "total": 2970, + "trueNegative": 2225, + "truePositive": 532, + "wilsonPrecisionLower95": 0.9346 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 4e-06, + "augmentation": "prefix", + "char_max": 6, + "char_min": 3, + "external_weight": 0.79, + "hard_example_weight": 2.0, + "l1_ratio": 0.15, + "max_char_features": 75000, + "max_word_features": 35000, + "min_df": 2, + "pseudo_weight": 0.32999999999999996, + "round": 12, + "word_max": 3 + }, + "featureCount": 94458, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 0 + }, + "complaint": { + "negative": 250, + "positive": 0 + }, + "confirmationDecision": { + "negative": 250, + "positive": 15 + }, + "followUpReminder": { + "negative": 250, + "positive": 0 + }, + "invitation": { + "negative": 250, + "positive": 0 + }, + "question": { + "negative": 232, + "positive": 0 + }, + "replyableMessage": { + "negative": 10, + "positive": 250 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 0 + }, + "task": { + "negative": 250, + "positive": 0 + } + }, + "round": 12, + "selectionObjective": 0.7563, + "syntheticRegression": { + "intentsMeeting95Precision": 5, + "macroF1": 0.6379, + "macroPrecision": 0.9476, + "macroRecall": 0.5322, + "minimumPrecision": 0.8452, + "perIntent": { + "blessing": { + "f1": 0.1053, + "falseNegative": 85, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 5, + "recall": 0.0556, + "total": 2970, + "trueNegative": 2880, + "truePositive": 5, + "wilsonPrecisionLower95": 0.5655 + }, + "complaint": { + "f1": 0.9625, + "falseNegative": 26, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 334, + "recall": 0.9278, + "total": 2970, + "trueNegative": 2610, + "truePositive": 334, + "wilsonPrecisionLower95": 0.9886 + }, + "confirmationDecision": { + "f1": 0.7821, + "falseNegative": 49, + "falsePositive": 24, + "precision": 0.8452, + "predictedPositive": 155, + "recall": 0.7278, + "total": 2970, + "trueNegative": 2766, + "truePositive": 131, + "wilsonPrecisionLower95": 0.7799 + }, + "followUpReminder": { + "f1": 0.6142, + "falseNegative": 200, + "falsePositive": 1, + "precision": 0.9938, + "predictedPositive": 161, + "recall": 0.4444, + "total": 2970, + "trueNegative": 2609, + "truePositive": 160, + "wilsonPrecisionLower95": 0.9657 + }, + "invitation": { + "f1": 0.5992, + "falseNegative": 103, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 77, + "recall": 0.4278, + "total": 2970, + "trueNegative": 2790, + "truePositive": 77, + "wilsonPrecisionLower95": 0.9525 + }, + "question": { + "f1": 0.4319, + "falseNegative": 454, + "falsePositive": 9, + "precision": 0.9514, + "predictedPositive": 185, + "recall": 0.2794, + "total": 2970, + "trueNegative": 2331, + "truePositive": 176, + "wilsonPrecisionLower95": 0.9101 + }, + "replyableMessage": { + "f1": 0.7888, + "falseNegative": 624, + "falsePositive": 54, + "precision": 0.9591, + "predictedPositive": 1320, + "recall": 0.6698, + "total": 2970, + "trueNegative": 1026, + "truePositive": 1266, + "wilsonPrecisionLower95": 0.947 + }, + "scheduleNegotiation": { + "f1": 0.8144, + "falseNegative": 22, + "falsePositive": 9, + "precision": 0.8831, + "predictedPositive": 77, + "recall": 0.7556, + "total": 2970, + "trueNegative": 2871, + "truePositive": 68, + "wilsonPrecisionLower95": 0.7925 + }, + "task": { + "f1": 0.6429, + "falseNegative": 359, + "falsePositive": 42, + "precision": 0.8958, + "predictedPositive": 403, + "recall": 0.5014, + "total": 2970, + "trueNegative": 2208, + "truePositive": 361, + "wilsonPrecisionLower95": 0.8621 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9917, + "falseNegative": 0, + "falsePositive": 3, + "precision": 0.9836, + "predictedPositive": 183, + "recall": 1.0, + "total": 1485, + "trueNegative": 1302, + "truePositive": 180, + "wilsonPrecisionLower95": 0.9529 + }, + "threshold": 0.885 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8974, + "falseNegative": 27, + "falsePositive": 8, + "precision": 0.9503, + "predictedPositive": 161, + "recall": 0.85, + "total": 1485, + "trueNegative": 1297, + "truePositive": 153, + "wilsonPrecisionLower95": 0.905 + }, + "threshold": 0.9 + } + }, + "metrics": { + "f1": 0.9515, + "falseNegative": 17, + "falsePositive": 18, + "precision": 0.9501, + "predictedPositive": 361, + "recall": 0.9528, + "total": 2970, + "trueNegative": 2592, + "truePositive": 343, + "wilsonPrecisionLower95": 0.9226 + }, + "threshold": 0.77 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9944, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 89, + "recall": 0.9889, + "total": 1485, + "trueNegative": 1395, + "truePositive": 89, + "wilsonPrecisionLower95": 0.9586 + }, + "threshold": 0.915 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.4959, + "falseNegative": 60, + "falsePositive": 1, + "precision": 0.9677, + "predictedPositive": 31, + "recall": 0.3333, + "total": 1485, + "trueNegative": 1394, + "truePositive": 30, + "wilsonPrecisionLower95": 0.8381 + }, + "threshold": 0.985 + } + }, + "metrics": { + "f1": 0.8217, + "falseNegative": 51, + "falsePositive": 5, + "precision": 0.9627, + "predictedPositive": 134, + "recall": 0.7167, + "total": 2970, + "trueNegative": 2785, + "truePositive": 129, + "wilsonPrecisionLower95": 0.9156 + }, + "threshold": 0.915 + }, + "followUpReminder": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.5647, + "falseNegative": 108, + "falsePositive": 3, + "precision": 0.96, + "predictedPositive": 75, + "recall": 0.4, + "total": 1485, + "trueNegative": 1302, + "truePositive": 72, + "wilsonPrecisionLower95": 0.8889 + }, + "threshold": 0.995 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6542, + "falseNegative": 158, + "falsePositive": 8, + "precision": 0.9515, + "predictedPositive": 165, + "recall": 0.4984, + "total": 1485, + "trueNegative": 1162, + "truePositive": 157, + "wilsonPrecisionLower95": 0.9073 + }, + "threshold": 0.805 + } + }, + "metrics": { + "f1": 0.5711, + "falseNegative": 373, + "falsePositive": 13, + "precision": 0.9519, + "predictedPositive": 270, + "recall": 0.4079, + "total": 2970, + "trueNegative": 2327, + "truePositive": 257, + "wilsonPrecisionLower95": 0.9194 + }, + "threshold": 0.985 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8956, + "falseNegative": 147, + "falsePositive": 39, + "precision": 0.9534, + "predictedPositive": 837, + "recall": 0.8444, + "total": 1485, + "trueNegative": 501, + "truePositive": 798, + "wilsonPrecisionLower95": 0.9369 + }, + "threshold": 0.975 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6735, + "falseNegative": 453, + "falsePositive": 24, + "precision": 0.9535, + "predictedPositive": 516, + "recall": 0.5206, + "total": 1485, + "trueNegative": 516, + "truePositive": 492, + "wilsonPrecisionLower95": 0.9317 + }, + "threshold": 0.985 + } + }, + "metrics": { + "f1": 0.7866, + "falseNegative": 626, + "falsePositive": 60, + "precision": 0.9547, + "predictedPositive": 1324, + "recall": 0.6688, + "total": 2970, + "trueNegative": 1020, + "truePositive": 1264, + "wilsonPrecisionLower95": 0.9421 + }, + "threshold": 0.985 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9888, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 44, + "recall": 0.9778, + "total": 1485, + "trueNegative": 1440, + "truePositive": 44, + "wilsonPrecisionLower95": 0.9197 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8052, + "falseNegative": 14, + "falsePositive": 1, + "precision": 0.9688, + "predictedPositive": 32, + "recall": 0.6889, + "total": 1485, + "trueNegative": 1439, + "truePositive": 31, + "wilsonPrecisionLower95": 0.8426 + }, + "threshold": 0.94 + } + }, + "metrics": { + "f1": 0.9364, + "falseNegative": 9, + "falsePositive": 2, + "precision": 0.9759, + "predictedPositive": 83, + "recall": 0.9, + "total": 2970, + "trueNegative": 2878, + "truePositive": 81, + "wilsonPrecisionLower95": 0.9163 + }, + "threshold": 0.715 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8475, + "falseNegative": 85, + "falsePositive": 14, + "precision": 0.9516, + "predictedPositive": 289, + "recall": 0.7639, + "total": 1485, + "trueNegative": 1111, + "truePositive": 275, + "wilsonPrecisionLower95": 0.9203 + }, + "threshold": 0.985 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.501, + "falseNegative": 238, + "falsePositive": 5, + "precision": 0.9606, + "predictedPositive": 127, + "recall": 0.3389, + "total": 1485, + "trueNegative": 1120, + "truePositive": 122, + "wilsonPrecisionLower95": 0.9111 + }, + "threshold": 0.985 + } + }, + "metrics": { + "f1": 0.6989, + "falseNegative": 323, + "falsePositive": 19, + "precision": 0.9543, + "predictedPositive": 416, + "recall": 0.5514, + "total": 2970, + "trueNegative": 2231, + "truePositive": 397, + "wilsonPrecisionLower95": 0.9298 + }, + "threshold": 0.985 + } + }, + "trainingRecords": 15345, + "validation": { + "intentsMeeting95Precision": 8, + "macroF1": 0.6603, + "macroPrecision": 0.9686, + "macroRecall": 0.5413, + "minimumPrecision": 0.9298, + "perIntent": { + "blessing": { + "f1": 0.2857, + "falseNegative": 75, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 15, + "recall": 0.1667, + "total": 2970, + "trueNegative": 2880, + "truePositive": 15, + "wilsonPrecisionLower95": 0.7961 + }, + "complaint": { + "f1": 0.946, + "falseNegative": 27, + "falsePositive": 11, + "precision": 0.968, + "predictedPositive": 344, + "recall": 0.925, + "total": 2970, + "trueNegative": 2599, + "truePositive": 333, + "wilsonPrecisionLower95": 0.9437 + }, + "confirmationDecision": { + "f1": 0.7933, + "falseNegative": 61, + "falsePositive": 1, + "precision": 0.9917, + "predictedPositive": 120, + "recall": 0.6611, + "total": 2970, + "trueNegative": 2789, + "truePositive": 119, + "wilsonPrecisionLower95": 0.9543 + }, + "followUpReminder": { + "f1": 0.6494, + "falseNegative": 185, + "falsePositive": 4, + "precision": 0.9777, + "predictedPositive": 179, + "recall": 0.4861, + "total": 2970, + "trueNegative": 2606, + "truePositive": 175, + "wilsonPrecisionLower95": 0.944 + }, + "invitation": { + "f1": 0.4473, + "falseNegative": 127, + "falsePositive": 4, + "precision": 0.9298, + "predictedPositive": 57, + "recall": 0.2944, + "total": 2970, + "trueNegative": 2786, + "truePositive": 53, + "wilsonPrecisionLower95": 0.833 + }, + "question": { + "f1": 0.4227, + "falseNegative": 459, + "falsePositive": 8, + "precision": 0.9553, + "predictedPositive": 179, + "recall": 0.2714, + "total": 2970, + "trueNegative": 2332, + "truePositive": 171, + "wilsonPrecisionLower95": 0.9143 + }, + "replyableMessage": { + "f1": 0.7956, + "falseNegative": 600, + "falsePositive": 63, + "precision": 0.9534, + "predictedPositive": 1353, + "recall": 0.6825, + "total": 2970, + "trueNegative": 1017, + "truePositive": 1290, + "wilsonPrecisionLower95": 0.9409 + }, + "scheduleNegotiation": { + "f1": 0.9036, + "falseNegative": 15, + "falsePositive": 1, + "precision": 0.9868, + "predictedPositive": 76, + "recall": 0.8333, + "total": 2970, + "trueNegative": 2879, + "truePositive": 75, + "wilsonPrecisionLower95": 0.9292 + }, + "task": { + "f1": 0.6989, + "falseNegative": 323, + "falsePositive": 19, + "precision": 0.9543, + "predictedPositive": 416, + "recall": 0.5514, + "total": 2970, + "trueNegative": 2231, + "truePositive": 397, + "wilsonPrecisionLower95": 0.9298 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 3e-06, + "augmentation": "prefix", + "char_max": 7, + "char_min": 2, + "external_weight": 0.81, + "hard_example_weight": 2.0, + "l1_ratio": 0.15, + "max_char_features": 80000, + "max_word_features": 38000, + "min_df": 2, + "pseudo_weight": 0.345, + "round": 13, + "word_max": 3 + }, + "featureCount": 99458, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 0 + }, + "complaint": { + "negative": 250, + "positive": 1 + }, + "confirmationDecision": { + "negative": 250, + "positive": 18 + }, + "followUpReminder": { + "negative": 250, + "positive": 0 + }, + "invitation": { + "negative": 250, + "positive": 0 + }, + "question": { + "negative": 250, + "positive": 0 + }, + "replyableMessage": { + "negative": 12, + "positive": 250 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 0 + }, + "task": { + "negative": 250, + "positive": 0 + } + }, + "round": 13, + "selectionObjective": 0.768, + "syntheticRegression": { + "intentsMeeting95Precision": 4, + "macroF1": 0.6577, + "macroPrecision": 0.9568, + "macroRecall": 0.5722, + "minimumPrecision": 0.859, + "perIntent": { + "blessing": { + "f1": 0.1443, + "falseNegative": 83, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 7, + "recall": 0.0778, + "total": 2970, + "trueNegative": 2880, + "truePositive": 7, + "wilsonPrecisionLower95": 0.6457 + }, + "complaint": { + "f1": 0.9744, + "falseNegative": 17, + "falsePositive": 1, + "precision": 0.9971, + "predictedPositive": 344, + "recall": 0.9528, + "total": 2970, + "trueNegative": 2609, + "truePositive": 343, + "wilsonPrecisionLower95": 0.9837 + }, + "confirmationDecision": { + "f1": 0.7976, + "falseNegative": 46, + "falsePositive": 22, + "precision": 0.859, + "predictedPositive": 156, + "recall": 0.7444, + "total": 2970, + "trueNegative": 2768, + "truePositive": 134, + "wilsonPrecisionLower95": 0.7957 + }, + "followUpReminder": { + "f1": 0.2972, + "falseNegative": 297, + "falsePositive": 1, + "precision": 0.9844, + "predictedPositive": 64, + "recall": 0.175, + "total": 2970, + "trueNegative": 2609, + "truePositive": 63, + "wilsonPrecisionLower95": 0.9167 + }, + "invitation": { + "f1": 0.6077, + "falseNegative": 101, + "falsePositive": 1, + "precision": 0.9875, + "predictedPositive": 80, + "recall": 0.4389, + "total": 2970, + "trueNegative": 2789, + "truePositive": 79, + "wilsonPrecisionLower95": 0.9325 + }, + "question": { + "f1": 0.5229, + "falseNegative": 407, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 223, + "recall": 0.354, + "total": 2970, + "trueNegative": 2340, + "truePositive": 223, + "wilsonPrecisionLower95": 0.9831 + }, + "replyableMessage": { + "f1": 0.9179, + "falseNegative": 202, + "falsePositive": 100, + "precision": 0.9441, + "predictedPositive": 1788, + "recall": 0.8931, + "total": 2970, + "trueNegative": 980, + "truePositive": 1688, + "wilsonPrecisionLower95": 0.9324 + }, + "scheduleNegotiation": { + "f1": 0.8889, + "falseNegative": 14, + "falsePositive": 5, + "precision": 0.9383, + "predictedPositive": 81, + "recall": 0.8444, + "total": 2970, + "trueNegative": 2875, + "truePositive": 76, + "wilsonPrecisionLower95": 0.8635 + }, + "task": { + "f1": 0.7681, + "falseNegative": 238, + "falsePositive": 53, + "precision": 0.9009, + "predictedPositive": 535, + "recall": 0.6694, + "total": 2970, + "trueNegative": 2197, + "truePositive": 482, + "wilsonPrecisionLower95": 0.8727 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9727, + "falseNegative": 2, + "falsePositive": 8, + "precision": 0.957, + "predictedPositive": 186, + "recall": 0.9889, + "total": 1485, + "trueNegative": 1297, + "truePositive": 178, + "wilsonPrecisionLower95": 0.9174 + }, + "threshold": 0.98 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.95, + "falseNegative": 9, + "falsePositive": 9, + "precision": 0.95, + "predictedPositive": 180, + "recall": 0.95, + "total": 1485, + "trueNegative": 1296, + "truePositive": 171, + "wilsonPrecisionLower95": 0.9077 + }, + "threshold": 0.905 + } + }, + "metrics": { + "f1": 0.9529, + "falseNegative": 16, + "falsePositive": 18, + "precision": 0.9503, + "predictedPositive": 362, + "recall": 0.9556, + "total": 2970, + "trueNegative": 2592, + "truePositive": 344, + "wilsonPrecisionLower95": 0.9228 + }, + "threshold": 0.965 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.828, + "falseNegative": 25, + "falsePositive": 2, + "precision": 0.9701, + "predictedPositive": 67, + "recall": 0.7222, + "total": 1485, + "trueNegative": 1393, + "truePositive": 65, + "wilsonPrecisionLower95": 0.8975 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6763, + "falseNegative": 43, + "falsePositive": 2, + "precision": 0.9592, + "predictedPositive": 49, + "recall": 0.5222, + "total": 1485, + "trueNegative": 1393, + "truePositive": 47, + "wilsonPrecisionLower95": 0.8629 + }, + "threshold": 0.93 + } + }, + "metrics": { + "f1": 0.7855, + "falseNegative": 61, + "falsePositive": 4, + "precision": 0.9675, + "predictedPositive": 123, + "recall": 0.6611, + "total": 2970, + "trueNegative": 2786, + "truePositive": 119, + "wilsonPrecisionLower95": 0.9194 + }, + "threshold": 0.82 + }, + "followUpReminder": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7452, + "falseNegative": 122, + "falsePositive": 10, + "precision": 0.9507, + "predictedPositive": 203, + "recall": 0.6127, + "total": 1485, + "trueNegative": 1160, + "truePositive": 193, + "wilsonPrecisionLower95": 0.9117 + }, + "threshold": 0.915 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9441, + "falseNegative": 59, + "falsePositive": 46, + "precision": 0.9506, + "predictedPositive": 932, + "recall": 0.9376, + "total": 1485, + "trueNegative": 494, + "truePositive": 886, + "wilsonPrecisionLower95": 0.9348 + }, + "threshold": 0.935 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9067, + "falseNegative": 129, + "falsePositive": 39, + "precision": 0.9544, + "predictedPositive": 855, + "recall": 0.8635, + "total": 1485, + "trueNegative": 501, + "truePositive": 816, + "wilsonPrecisionLower95": 0.9383 + }, + "threshold": 0.905 + } + }, + "metrics": { + "f1": 0.9267, + "falseNegative": 182, + "falsePositive": 88, + "precision": 0.951, + "predictedPositive": 1796, + "recall": 0.9037, + "total": 2970, + "trueNegative": 992, + "truePositive": 1708, + "wilsonPrecisionLower95": 0.94 + }, + "threshold": 0.905 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9888, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 44, + "recall": 0.9778, + "total": 1485, + "trueNegative": 1440, + "truePositive": 44, + "wilsonPrecisionLower95": 0.9197 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9655, + "falseNegative": 3, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 42, + "recall": 0.9333, + "total": 1485, + "trueNegative": 1440, + "truePositive": 42, + "wilsonPrecisionLower95": 0.9162 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.9773, + "falseNegative": 4, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 86, + "recall": 0.9556, + "total": 2970, + "trueNegative": 2880, + "truePositive": 86, + "wilsonPrecisionLower95": 0.9572 + }, + "threshold": 0.5 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.7876, + "falseNegative": 119, + "falsePositive": 11, + "precision": 0.9563, + "predictedPositive": 252, + "recall": 0.6694, + "total": 1485, + "trueNegative": 1114, + "truePositive": 241, + "wilsonPrecisionLower95": 0.9235 + }, + "threshold": 0.995 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8416, + "falseNegative": 89, + "falsePositive": 13, + "precision": 0.9542, + "predictedPositive": 284, + "recall": 0.7528, + "total": 1485, + "trueNegative": 1112, + "truePositive": 271, + "wilsonPrecisionLower95": 0.9233 + }, + "threshold": 0.895 + } + }, + "metrics": { + "f1": 0.6962, + "falseNegative": 327, + "falsePositive": 16, + "precision": 0.9609, + "predictedPositive": 409, + "recall": 0.5458, + "total": 2970, + "trueNegative": 2234, + "truePositive": 393, + "wilsonPrecisionLower95": 0.9374 + }, + "threshold": 0.995 + } + }, + "trainingRecords": 15346, + "validation": { + "intentsMeeting95Precision": 9, + "macroF1": 0.6724, + "macroPrecision": 0.9724, + "macroRecall": 0.5779, + "minimumPrecision": 0.9524, + "perIntent": { + "blessing": { + "f1": 0.2178, + "falseNegative": 79, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 11, + "recall": 0.1222, + "total": 2970, + "trueNegative": 2880, + "truePositive": 11, + "wilsonPrecisionLower95": 0.7412 + }, + "complaint": { + "f1": 0.9614, + "falseNegative": 11, + "falsePositive": 17, + "precision": 0.9536, + "predictedPositive": 366, + "recall": 0.9694, + "total": 2970, + "trueNegative": 2593, + "truePositive": 349, + "wilsonPrecisionLower95": 0.9269 + }, + "confirmationDecision": { + "f1": 0.7568, + "falseNegative": 68, + "falsePositive": 4, + "precision": 0.9655, + "predictedPositive": 116, + "recall": 0.6222, + "total": 2970, + "trueNegative": 2786, + "truePositive": 112, + "wilsonPrecisionLower95": 0.9147 + }, + "followUpReminder": { + "f1": 0.4895, + "falseNegative": 243, + "falsePositive": 1, + "precision": 0.9915, + "predictedPositive": 118, + "recall": 0.325, + "total": 2970, + "trueNegative": 2609, + "truePositive": 117, + "wilsonPrecisionLower95": 0.9536 + }, + "invitation": { + "f1": 0.4192, + "falseNegative": 132, + "falsePositive": 1, + "precision": 0.9796, + "predictedPositive": 49, + "recall": 0.2667, + "total": 2970, + "trueNegative": 2789, + "truePositive": 48, + "wilsonPrecisionLower95": 0.8931 + }, + "question": { + "f1": 0.4888, + "falseNegative": 423, + "falsePositive": 10, + "precision": 0.9539, + "predictedPositive": 217, + "recall": 0.3286, + "total": 2970, + "trueNegative": 2330, + "truePositive": 207, + "wilsonPrecisionLower95": 0.9173 + }, + "replyableMessage": { + "f1": 0.9258, + "falseNegative": 188, + "falsePositive": 85, + "precision": 0.9524, + "predictedPositive": 1787, + "recall": 0.9005, + "total": 2970, + "trueNegative": 995, + "truePositive": 1702, + "wilsonPrecisionLower95": 0.9416 + }, + "scheduleNegotiation": { + "f1": 0.9773, + "falseNegative": 4, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 86, + "recall": 0.9556, + "total": 2970, + "trueNegative": 2880, + "truePositive": 86, + "wilsonPrecisionLower95": 0.9572 + }, + "task": { + "f1": 0.8153, + "falseNegative": 208, + "falsePositive": 24, + "precision": 0.9552, + "predictedPositive": 536, + "recall": 0.7111, + "total": 2970, + "trueNegative": 2226, + "truePositive": 512, + "wilsonPrecisionLower95": 0.9342 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 2.5e-06, + "augmentation": "prefix", + "char_max": 7, + "char_min": 3, + "external_weight": 0.8300000000000001, + "hard_example_weight": 2.2, + "l1_ratio": 0.2, + "max_char_features": 80000, + "max_word_features": 40000, + "min_df": 1, + "pseudo_weight": 0.36, + "round": 14, + "word_max": 3 + }, + "featureCount": 120000, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 0 + }, + "complaint": { + "negative": 250, + "positive": 8 + }, + "confirmationDecision": { + "negative": 250, + "positive": 26 + }, + "followUpReminder": { + "negative": 250, + "positive": 0 + }, + "invitation": { + "negative": 250, + "positive": 0 + }, + "question": { + "negative": 250, + "positive": 0 + }, + "replyableMessage": { + "negative": 34, + "positive": 250 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 0 + }, + "task": { + "negative": 250, + "positive": 3 + } + }, + "round": 14, + "selectionObjective": 0.7391, + "syntheticRegression": { + "intentsMeeting95Precision": 5, + "macroF1": 0.6159, + "macroPrecision": 0.9488, + "macroRecall": 0.5171, + "minimumPrecision": 0.8315, + "perIntent": { + "blessing": { + "f1": 0.1443, + "falseNegative": 83, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 7, + "recall": 0.0778, + "total": 2970, + "trueNegative": 2880, + "truePositive": 7, + "wilsonPrecisionLower95": 0.6457 + }, + "complaint": { + "f1": 0.7987, + "falseNegative": 114, + "falsePositive": 10, + "precision": 0.9609, + "predictedPositive": 256, + "recall": 0.6833, + "total": 2970, + "trueNegative": 2600, + "truePositive": 246, + "wilsonPrecisionLower95": 0.9296 + }, + "confirmationDecision": { + "f1": 0.8407, + "falseNegative": 27, + "falsePositive": 31, + "precision": 0.8315, + "predictedPositive": 184, + "recall": 0.85, + "total": 2970, + "trueNegative": 2759, + "truePositive": 153, + "wilsonPrecisionLower95": 0.7708 + }, + "followUpReminder": { + "f1": 0.7361, + "falseNegative": 148, + "falsePositive": 4, + "precision": 0.9815, + "predictedPositive": 216, + "recall": 0.5889, + "total": 2970, + "trueNegative": 2606, + "truePositive": 212, + "wilsonPrecisionLower95": 0.9534 + }, + "invitation": { + "f1": 0.5591, + "falseNegative": 109, + "falsePositive": 3, + "precision": 0.9595, + "predictedPositive": 74, + "recall": 0.3944, + "total": 2970, + "trueNegative": 2787, + "truePositive": 71, + "wilsonPrecisionLower95": 0.8875 + }, + "question": { + "f1": 0.7917, + "falseNegative": 212, + "falsePositive": 8, + "precision": 0.9812, + "predictedPositive": 426, + "recall": 0.6635, + "total": 2970, + "trueNegative": 2332, + "truePositive": 418, + "wilsonPrecisionLower95": 0.9634 + }, + "replyableMessage": { + "f1": 0.6113, + "falseNegative": 1037, + "falsePositive": 48, + "precision": 0.9467, + "predictedPositive": 901, + "recall": 0.4513, + "total": 2970, + "trueNegative": 1032, + "truePositive": 853, + "wilsonPrecisionLower95": 0.9301 + }, + "scheduleNegotiation": { + "f1": 0.9176, + "falseNegative": 12, + "falsePositive": 2, + "precision": 0.975, + "predictedPositive": 80, + "recall": 0.8667, + "total": 2970, + "trueNegative": 2878, + "truePositive": 78, + "wilsonPrecisionLower95": 0.9134 + }, + "task": { + "f1": 0.1432, + "falseNegative": 664, + "falsePositive": 6, + "precision": 0.9032, + "predictedPositive": 62, + "recall": 0.0778, + "total": 2970, + "trueNegative": 2244, + "truePositive": 56, + "wilsonPrecisionLower95": 0.8045 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9809, + "falseNegative": 0, + "falsePositive": 7, + "precision": 0.9626, + "predictedPositive": 187, + "recall": 1.0, + "total": 1485, + "trueNegative": 1298, + "truePositive": 180, + "wilsonPrecisionLower95": 0.9248 + }, + "threshold": 0.99 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8625, + "falseNegative": 21, + "falsePositive": 1, + "precision": 0.9857, + "predictedPositive": 70, + "recall": 0.7667, + "total": 1485, + "trueNegative": 1394, + "truePositive": 69, + "wilsonPrecisionLower95": 0.9234 + }, + "threshold": 0.905 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.828, + "falseNegative": 25, + "falsePositive": 2, + "precision": 0.9701, + "predictedPositive": 67, + "recall": 0.7222, + "total": 1485, + "trueNegative": 1393, + "truePositive": 65, + "wilsonPrecisionLower95": 0.8975 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.8328, + "falseNegative": 48, + "falsePositive": 5, + "precision": 0.9635, + "predictedPositive": 137, + "recall": 0.7333, + "total": 2970, + "trueNegative": 2785, + "truePositive": 132, + "wilsonPrecisionLower95": 0.9174 + }, + "threshold": 0.685 + }, + "followUpReminder": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8788, + "falseNegative": 35, + "falsePositive": 5, + "precision": 0.9667, + "predictedPositive": 150, + "recall": 0.8056, + "total": 1485, + "trueNegative": 1300, + "truePositive": 145, + "wilsonPrecisionLower95": 0.9243 + }, + "threshold": 0.845 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6113, + "falseNegative": 99, + "falsePositive": 4, + "precision": 0.9529, + "predictedPositive": 85, + "recall": 0.45, + "total": 1485, + "trueNegative": 1301, + "truePositive": 81, + "wilsonPrecisionLower95": 0.8852 + }, + "threshold": 0.97 + } + }, + "metrics": { + "f1": 0.7682, + "falseNegative": 128, + "falsePositive": 12, + "precision": 0.9508, + "predictedPositive": 244, + "recall": 0.6444, + "total": 2970, + "trueNegative": 2598, + "truePositive": 232, + "wilsonPrecisionLower95": 0.916 + }, + "threshold": 0.91 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.847, + "falseNegative": 77, + "falsePositive": 9, + "precision": 0.9636, + "predictedPositive": 247, + "recall": 0.7556, + "total": 1485, + "trueNegative": 1161, + "truePositive": 238, + "wilsonPrecisionLower95": 0.9322 + }, + "threshold": 0.995 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7168, + "falseNegative": 134, + "falsePositive": 9, + "precision": 0.9526, + "predictedPositive": 190, + "recall": 0.5746, + "total": 1485, + "trueNegative": 1161, + "truePositive": 181, + "wilsonPrecisionLower95": 0.9124 + }, + "threshold": 0.865 + } + }, + "metrics": { + "f1": 0.7524, + "falseNegative": 238, + "falsePositive": 20, + "precision": 0.9515, + "predictedPositive": 412, + "recall": 0.6222, + "total": 2970, + "trueNegative": 2320, + "truePositive": 392, + "wilsonPrecisionLower95": 0.9262 + }, + "threshold": 0.975 + }, + "replyableMessage": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9349, + "falseNegative": 76, + "falsePositive": 45, + "precision": 0.9508, + "predictedPositive": 914, + "recall": 0.9196, + "total": 1485, + "trueNegative": 495, + "truePositive": 869, + "wilsonPrecisionLower95": 0.9348 + }, + "threshold": 0.955 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 45, + "recall": 1.0, + "total": 1485, + "trueNegative": 1440, + "truePositive": 45, + "wilsonPrecisionLower95": 0.9213 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.967, + "falseNegative": 1, + "falsePositive": 2, + "precision": 0.9565, + "predictedPositive": 46, + "recall": 0.9778, + "total": 1485, + "trueNegative": 1438, + "truePositive": 44, + "wilsonPrecisionLower95": 0.8547 + }, + "threshold": 0.995 + } + }, + "metrics": { + "f1": 0.9727, + "falseNegative": 1, + "falsePositive": 4, + "precision": 0.957, + "predictedPositive": 93, + "recall": 0.9889, + "total": 2970, + "trueNegative": 2876, + "truePositive": 89, + "wilsonPrecisionLower95": 0.8946 + }, + "threshold": 0.64 + }, + "task": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "trainingRecords": 15345, + "validation": { + "intentsMeeting95Precision": 8, + "macroF1": 0.6332, + "macroPrecision": 0.9668, + "macroRecall": 0.5246, + "minimumPrecision": 0.9, + "perIntent": { + "blessing": { + "f1": 0.3019, + "falseNegative": 74, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 16, + "recall": 0.1778, + "total": 2970, + "trueNegative": 2880, + "truePositive": 16, + "wilsonPrecisionLower95": 0.8064 + }, + "complaint": { + "f1": 0.8199, + "falseNegative": 105, + "falsePositive": 7, + "precision": 0.9733, + "predictedPositive": 262, + "recall": 0.7083, + "total": 2970, + "trueNegative": 2603, + "truePositive": 255, + "wilsonPrecisionLower95": 0.9459 + }, + "confirmationDecision": { + "f1": 0.8454, + "falseNegative": 46, + "falsePositive": 3, + "precision": 0.9781, + "predictedPositive": 137, + "recall": 0.7444, + "total": 2970, + "trueNegative": 2787, + "truePositive": 134, + "wilsonPrecisionLower95": 0.9376 + }, + "followUpReminder": { + "f1": 0.7597, + "falseNegative": 134, + "falsePositive": 9, + "precision": 0.9617, + "predictedPositive": 235, + "recall": 0.6278, + "total": 2970, + "trueNegative": 2601, + "truePositive": 226, + "wilsonPrecisionLower95": 0.9288 + }, + "invitation": { + "f1": 0.3273, + "falseNegative": 144, + "falsePositive": 4, + "precision": 0.9, + "predictedPositive": 40, + "recall": 0.2, + "total": 2970, + "trueNegative": 2786, + "truePositive": 36, + "wilsonPrecisionLower95": 0.7695 + }, + "question": { + "f1": 0.7854, + "falseNegative": 211, + "falsePositive": 18, + "precision": 0.9588, + "predictedPositive": 437, + "recall": 0.6651, + "total": 2970, + "trueNegative": 2322, + "truePositive": 419, + "wilsonPrecisionLower95": 0.9358 + }, + "replyableMessage": { + "f1": 0.6272, + "falseNegative": 1006, + "falsePositive": 45, + "precision": 0.9516, + "predictedPositive": 929, + "recall": 0.4677, + "total": 2970, + "trueNegative": 1035, + "truePositive": 884, + "wilsonPrecisionLower95": 0.9358 + }, + "scheduleNegotiation": { + "f1": 0.9834, + "falseNegative": 1, + "falsePositive": 2, + "precision": 0.978, + "predictedPositive": 91, + "recall": 0.9889, + "total": 2970, + "trueNegative": 2878, + "truePositive": 89, + "wilsonPrecisionLower95": 0.9234 + }, + "task": { + "f1": 0.2482, + "falseNegative": 618, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 102, + "recall": 0.1417, + "total": 2970, + "trueNegative": 2250, + "truePositive": 102, + "wilsonPrecisionLower95": 0.9637 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 3.5e-06, + "augmentation": "prefix", + "char_max": 6, + "char_min": 2, + "external_weight": 0.8500000000000001, + "hard_example_weight": 2.2, + "l1_ratio": 0.2, + "max_char_features": 85000, + "max_word_features": 42000, + "min_df": 1, + "pseudo_weight": 0.375, + "round": 15, + "word_max": 3 + }, + "featureCount": 127000, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 1 + }, + "complaint": { + "negative": 250, + "positive": 8 + }, + "confirmationDecision": { + "negative": 250, + "positive": 30 + }, + "followUpReminder": { + "negative": 250, + "positive": 0 + }, + "invitation": { + "negative": 250, + "positive": 0 + }, + "question": { + "negative": 250, + "positive": 0 + }, + "replyableMessage": { + "negative": 39, + "positive": 250 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 0 + }, + "task": { + "negative": 250, + "positive": 0 + } + }, + "round": 15, + "selectionObjective": 0.7896, + "syntheticRegression": { + "intentsMeeting95Precision": 7, + "macroF1": 0.7019, + "macroPrecision": 0.9601, + "macroRecall": 0.6059, + "minimumPrecision": 0.8634, + "perIntent": { + "blessing": { + "f1": 0.2, + "falseNegative": 80, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 10, + "recall": 0.1111, + "total": 2970, + "trueNegative": 2880, + "truePositive": 10, + "wilsonPrecisionLower95": 0.7225 + }, + "complaint": { + "f1": 0.9641, + "falseNegative": 24, + "falsePositive": 1, + "precision": 0.997, + "predictedPositive": 337, + "recall": 0.9333, + "total": 2970, + "trueNegative": 2609, + "truePositive": 336, + "wilsonPrecisionLower95": 0.9834 + }, + "confirmationDecision": { + "f1": 0.8152, + "falseNegative": 41, + "falsePositive": 22, + "precision": 0.8634, + "predictedPositive": 161, + "recall": 0.7722, + "total": 2970, + "trueNegative": 2768, + "truePositive": 139, + "wilsonPrecisionLower95": 0.8018 + }, + "followUpReminder": { + "f1": 0.6115, + "falseNegative": 201, + "falsePositive": 1, + "precision": 0.9938, + "predictedPositive": 160, + "recall": 0.4417, + "total": 2970, + "trueNegative": 2609, + "truePositive": 159, + "wilsonPrecisionLower95": 0.9655 + }, + "invitation": { + "f1": 0.6084, + "falseNegative": 100, + "falsePositive": 3, + "precision": 0.9639, + "predictedPositive": 83, + "recall": 0.4444, + "total": 2970, + "trueNegative": 2787, + "truePositive": 80, + "wilsonPrecisionLower95": 0.899 + }, + "question": { + "f1": 0.5421, + "falseNegative": 395, + "falsePositive": 2, + "precision": 0.9916, + "predictedPositive": 237, + "recall": 0.373, + "total": 2970, + "trueNegative": 2338, + "truePositive": 235, + "wilsonPrecisionLower95": 0.9698 + }, + "replyableMessage": { + "f1": 0.862, + "falseNegative": 410, + "falsePositive": 64, + "precision": 0.9585, + "predictedPositive": 1544, + "recall": 0.7831, + "total": 2970, + "trueNegative": 1016, + "truePositive": 1480, + "wilsonPrecisionLower95": 0.9474 + }, + "scheduleNegotiation": { + "f1": 0.9379, + "falseNegative": 7, + "falsePositive": 4, + "precision": 0.954, + "predictedPositive": 87, + "recall": 0.9222, + "total": 2970, + "trueNegative": 2876, + "truePositive": 83, + "wilsonPrecisionLower95": 0.8877 + }, + "task": { + "f1": 0.7763, + "falseNegative": 236, + "falsePositive": 43, + "precision": 0.9184, + "predictedPositive": 527, + "recall": 0.6722, + "total": 2970, + "trueNegative": 2207, + "truePositive": 484, + "wilsonPrecisionLower95": 0.8919 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9836, + "falseNegative": 0, + "falsePositive": 6, + "precision": 0.9677, + "predictedPositive": 186, + "recall": 1.0, + "total": 1485, + "trueNegative": 1299, + "truePositive": 180, + "wilsonPrecisionLower95": 0.9314 + }, + "threshold": 0.945 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.928, + "falseNegative": 19, + "falsePositive": 6, + "precision": 0.9641, + "predictedPositive": 167, + "recall": 0.8944, + "total": 1485, + "trueNegative": 1299, + "truePositive": 161, + "wilsonPrecisionLower95": 0.9238 + }, + "threshold": 0.77 + } + }, + "metrics": { + "f1": 0.9365, + "falseNegative": 28, + "falsePositive": 17, + "precision": 0.9513, + "predictedPositive": 349, + "recall": 0.9222, + "total": 2970, + "trueNegative": 2593, + "truePositive": 332, + "wilsonPrecisionLower95": 0.9234 + }, + "threshold": 0.88 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.954, + "falseNegative": 7, + "falsePositive": 1, + "precision": 0.9881, + "predictedPositive": 84, + "recall": 0.9222, + "total": 1485, + "trueNegative": 1394, + "truePositive": 83, + "wilsonPrecisionLower95": 0.9356 + }, + "threshold": 0.85 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6906, + "falseNegative": 42, + "falsePositive": 1, + "precision": 0.9796, + "predictedPositive": 49, + "recall": 0.5333, + "total": 1485, + "trueNegative": 1394, + "truePositive": 48, + "wilsonPrecisionLower95": 0.8931 + }, + "threshold": 0.985 + } + }, + "metrics": { + "f1": 0.8598, + "falseNegative": 39, + "falsePositive": 7, + "precision": 0.9527, + "predictedPositive": 148, + "recall": 0.7833, + "total": 2970, + "trueNegative": 2783, + "truePositive": 141, + "wilsonPrecisionLower95": 0.9056 + }, + "threshold": 0.935 + }, + "followUpReminder": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8328, + "falseNegative": 48, + "falsePositive": 5, + "precision": 0.9635, + "predictedPositive": 137, + "recall": 0.7333, + "total": 1485, + "trueNegative": 1300, + "truePositive": 132, + "wilsonPrecisionLower95": 0.9174 + }, + "threshold": 0.99 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9133, + "falseNegative": 11, + "falsePositive": 4, + "precision": 0.9518, + "predictedPositive": 83, + "recall": 0.8778, + "total": 1485, + "trueNegative": 1391, + "truePositive": 79, + "wilsonPrecisionLower95": 0.8825 + }, + "threshold": 0.97 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7548, + "falseNegative": 118, + "falsePositive": 10, + "precision": 0.9517, + "predictedPositive": 207, + "recall": 0.6254, + "total": 1485, + "trueNegative": 1160, + "truePositive": 197, + "wilsonPrecisionLower95": 0.9134 + }, + "threshold": 0.8 + } + }, + "metrics": { + "f1": 0.7678, + "falseNegative": 225, + "falsePositive": 20, + "precision": 0.9529, + "predictedPositive": 425, + "recall": 0.6429, + "total": 2970, + "trueNegative": 2320, + "truePositive": 405, + "wilsonPrecisionLower95": 0.9284 + }, + "threshold": 0.985 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.907, + "falseNegative": 126, + "falsePositive": 42, + "precision": 0.9512, + "predictedPositive": 861, + "recall": 0.8667, + "total": 1485, + "trueNegative": 498, + "truePositive": 819, + "wilsonPrecisionLower95": 0.9347 + }, + "threshold": 0.98 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8252, + "falseNegative": 258, + "falsePositive": 33, + "precision": 0.9542, + "predictedPositive": 720, + "recall": 0.727, + "total": 1485, + "trueNegative": 507, + "truePositive": 687, + "wilsonPrecisionLower95": 0.9363 + }, + "threshold": 0.96 + } + }, + "metrics": { + "f1": 0.8678, + "falseNegative": 384, + "falsePositive": 75, + "precision": 0.9526, + "predictedPositive": 1581, + "recall": 0.7968, + "total": 2970, + "trueNegative": 1005, + "truePositive": 1506, + "wilsonPrecisionLower95": 0.9409 + }, + "threshold": 0.965 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9157, + "falseNegative": 7, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 38, + "recall": 0.8444, + "total": 1485, + "trueNegative": 1440, + "truePositive": 38, + "wilsonPrecisionLower95": 0.9082 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9556, + "falseNegative": 2, + "falsePositive": 2, + "precision": 0.9556, + "predictedPositive": 45, + "recall": 0.9556, + "total": 1485, + "trueNegative": 1438, + "truePositive": 43, + "wilsonPrecisionLower95": 0.8517 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.9364, + "falseNegative": 9, + "falsePositive": 2, + "precision": 0.9759, + "predictedPositive": 83, + "recall": 0.9, + "total": 2970, + "trueNegative": 2878, + "truePositive": 81, + "wilsonPrecisionLower95": 0.9163 + }, + "threshold": 0.5 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8593, + "falseNegative": 79, + "falsePositive": 13, + "precision": 0.9558, + "predictedPositive": 294, + "recall": 0.7806, + "total": 1485, + "trueNegative": 1112, + "truePositive": 281, + "wilsonPrecisionLower95": 0.9258 + }, + "threshold": 0.98 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8166, + "falseNegative": 104, + "falsePositive": 11, + "precision": 0.9588, + "predictedPositive": 267, + "recall": 0.7111, + "total": 1485, + "trueNegative": 1114, + "truePositive": 256, + "wilsonPrecisionLower95": 0.9277 + }, + "threshold": 0.96 + } + }, + "metrics": { + "f1": 0.8434, + "falseNegative": 176, + "falsePositive": 26, + "precision": 0.9544, + "predictedPositive": 570, + "recall": 0.7556, + "total": 2970, + "trueNegative": 2224, + "truePositive": 544, + "wilsonPrecisionLower95": 0.934 + }, + "threshold": 0.965 + } + }, + "trainingRecords": 15345, + "validation": { + "intentsMeeting95Precision": 8, + "macroF1": 0.711, + "macroPrecision": 0.9652, + "macroRecall": 0.6071, + "minimumPrecision": 0.9441, + "perIntent": { + "blessing": { + "f1": 0.3333, + "falseNegative": 72, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 18, + "recall": 0.2, + "total": 2970, + "trueNegative": 2880, + "truePositive": 18, + "wilsonPrecisionLower95": 0.8241 + }, + "complaint": { + "f1": 0.9565, + "falseNegative": 19, + "falsePositive": 12, + "precision": 0.966, + "predictedPositive": 353, + "recall": 0.9472, + "total": 2970, + "trueNegative": 2598, + "truePositive": 341, + "wilsonPrecisionLower95": 0.9415 + }, + "confirmationDecision": { + "f1": 0.8371, + "falseNegative": 49, + "falsePositive": 2, + "precision": 0.985, + "predictedPositive": 133, + "recall": 0.7278, + "total": 2970, + "trueNegative": 2788, + "truePositive": 131, + "wilsonPrecisionLower95": 0.9468 + }, + "followUpReminder": { + "f1": 0.5368, + "falseNegative": 225, + "falsePositive": 8, + "precision": 0.9441, + "predictedPositive": 143, + "recall": 0.375, + "total": 2970, + "trueNegative": 2602, + "truePositive": 135, + "wilsonPrecisionLower95": 0.8935 + }, + "invitation": { + "f1": 0.6061, + "falseNegative": 100, + "falsePositive": 4, + "precision": 0.9524, + "predictedPositive": 84, + "recall": 0.4444, + "total": 2970, + "trueNegative": 2786, + "truePositive": 80, + "wilsonPrecisionLower95": 0.8839 + }, + "question": { + "f1": 0.487, + "falseNegative": 424, + "falsePositive": 10, + "precision": 0.9537, + "predictedPositive": 216, + "recall": 0.327, + "total": 2970, + "trueNegative": 2330, + "truePositive": 206, + "wilsonPrecisionLower95": 0.9169 + }, + "replyableMessage": { + "f1": 0.8678, + "falseNegative": 384, + "falsePositive": 75, + "precision": 0.9526, + "predictedPositive": 1581, + "recall": 0.7968, + "total": 2970, + "trueNegative": 1005, + "truePositive": 1506, + "wilsonPrecisionLower95": 0.9409 + }, + "scheduleNegotiation": { + "f1": 0.9364, + "falseNegative": 9, + "falsePositive": 2, + "precision": 0.9759, + "predictedPositive": 83, + "recall": 0.9, + "total": 2970, + "trueNegative": 2878, + "truePositive": 81, + "wilsonPrecisionLower95": 0.9163 + }, + "task": { + "f1": 0.8384, + "falseNegative": 183, + "falsePositive": 24, + "precision": 0.9572, + "predictedPositive": 561, + "recall": 0.7458, + "total": 2970, + "trueNegative": 2226, + "truePositive": 537, + "wilsonPrecisionLower95": 0.9371 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 3e-06, + "augmentation": "numeric", + "char_max": 7, + "char_min": 2, + "external_weight": 0.8700000000000001, + "hard_example_weight": 2.3, + "l1_ratio": 0.15, + "max_char_features": 80000, + "max_word_features": 38000, + "min_df": 2, + "pseudo_weight": 0.39, + "round": 16, + "word_max": 3 + }, + "featureCount": 100252, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 3 + }, + "complaint": { + "negative": 250, + "positive": 11 + }, + "confirmationDecision": { + "negative": 250, + "positive": 25 + }, + "followUpReminder": { + "negative": 250, + "positive": 0 + }, + "invitation": { + "negative": 250, + "positive": 0 + }, + "question": { + "negative": 250, + "positive": 0 + }, + "replyableMessage": { + "negative": 76, + "positive": 250 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 0 + }, + "task": { + "negative": 250, + "positive": 0 + } + }, + "round": 16, + "selectionObjective": 0.8105, + "syntheticRegression": { + "intentsMeeting95Precision": 5, + "macroF1": 0.726, + "macroPrecision": 0.9485, + "macroRecall": 0.6388, + "minimumPrecision": 0.8195, + "perIntent": { + "blessing": { + "f1": 0.125, + "falseNegative": 84, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 6, + "recall": 0.0667, + "total": 2970, + "trueNegative": 2880, + "truePositive": 6, + "wilsonPrecisionLower95": 0.6097 + }, + "complaint": { + "f1": 0.9132, + "falseNegative": 55, + "falsePositive": 3, + "precision": 0.9903, + "predictedPositive": 308, + "recall": 0.8472, + "total": 2970, + "trueNegative": 2607, + "truePositive": 305, + "wilsonPrecisionLower95": 0.9718 + }, + "confirmationDecision": { + "f1": 0.6965, + "falseNegative": 71, + "falsePositive": 24, + "precision": 0.8195, + "predictedPositive": 133, + "recall": 0.6056, + "total": 2970, + "trueNegative": 2766, + "truePositive": 109, + "wilsonPrecisionLower95": 0.7455 + }, + "followUpReminder": { + "f1": 0.7409, + "falseNegative": 147, + "falsePositive": 2, + "precision": 0.9907, + "predictedPositive": 215, + "recall": 0.5917, + "total": 2970, + "trueNegative": 2608, + "truePositive": 213, + "wilsonPrecisionLower95": 0.9667 + }, + "invitation": { + "f1": 0.6264, + "falseNegative": 97, + "falsePositive": 2, + "precision": 0.9765, + "predictedPositive": 85, + "recall": 0.4611, + "total": 2970, + "trueNegative": 2788, + "truePositive": 83, + "wilsonPrecisionLower95": 0.9182 + }, + "question": { + "f1": 0.8457, + "falseNegative": 164, + "falsePositive": 6, + "precision": 0.9873, + "predictedPositive": 472, + "recall": 0.7397, + "total": 2970, + "trueNegative": 2334, + "truePositive": 466, + "wilsonPrecisionLower95": 0.9725 + }, + "replyableMessage": { + "f1": 0.8725, + "falseNegative": 371, + "falsePositive": 73, + "precision": 0.9541, + "predictedPositive": 1592, + "recall": 0.8037, + "total": 2970, + "trueNegative": 1007, + "truePositive": 1519, + "wilsonPrecisionLower95": 0.9427 + }, + "scheduleNegotiation": { + "f1": 0.9061, + "falseNegative": 8, + "falsePositive": 9, + "precision": 0.9011, + "predictedPositive": 91, + "recall": 0.9111, + "total": 2970, + "trueNegative": 2871, + "truePositive": 82, + "wilsonPrecisionLower95": 0.8226 + }, + "task": { + "f1": 0.8081, + "falseNegative": 200, + "falsePositive": 47, + "precision": 0.9171, + "predictedPositive": 567, + "recall": 0.7222, + "total": 2970, + "trueNegative": 2203, + "truePositive": 520, + "wilsonPrecisionLower95": 0.8915 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9371, + "falseNegative": 16, + "falsePositive": 6, + "precision": 0.9647, + "predictedPositive": 170, + "recall": 0.9111, + "total": 1485, + "trueNegative": 1299, + "truePositive": 164, + "wilsonPrecisionLower95": 0.9251 + }, + "threshold": 0.995 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8836, + "falseNegative": 32, + "falsePositive": 7, + "precision": 0.9548, + "predictedPositive": 155, + "recall": 0.8222, + "total": 1485, + "trueNegative": 1298, + "truePositive": 148, + "wilsonPrecisionLower95": 0.9097 + }, + "threshold": 0.99 + } + }, + "metrics": { + "f1": 0.8994, + "falseNegative": 56, + "falsePositive": 12, + "precision": 0.962, + "predictedPositive": 316, + "recall": 0.8444, + "total": 2970, + "trueNegative": 2598, + "truePositive": 304, + "wilsonPrecisionLower95": 0.9348 + }, + "threshold": 0.995 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8625, + "falseNegative": 21, + "falsePositive": 1, + "precision": 0.9857, + "predictedPositive": 70, + "recall": 0.7667, + "total": 1485, + "trueNegative": 1394, + "truePositive": 69, + "wilsonPrecisionLower95": 0.9234 + }, + "threshold": 0.915 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.7415, + "falseNegative": 71, + "falsePositive": 5, + "precision": 0.9561, + "predictedPositive": 114, + "recall": 0.6056, + "total": 2970, + "trueNegative": 2785, + "truePositive": 109, + "wilsonPrecisionLower95": 0.9014 + }, + "threshold": 0.96 + }, + "followUpReminder": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8667, + "falseNegative": 37, + "falsePositive": 7, + "precision": 0.9533, + "predictedPositive": 150, + "recall": 0.7944, + "total": 1485, + "trueNegative": 1298, + "truePositive": 143, + "wilsonPrecisionLower95": 0.9068 + }, + "threshold": 0.96 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6031, + "falseNegative": 101, + "falsePositive": 3, + "precision": 0.9634, + "predictedPositive": 82, + "recall": 0.4389, + "total": 1485, + "trueNegative": 1302, + "truePositive": 79, + "wilsonPrecisionLower95": 0.8979 + }, + "threshold": 0.97 + } + }, + "metrics": { + "f1": 0.7492, + "falseNegative": 139, + "falsePositive": 9, + "precision": 0.9609, + "predictedPositive": 230, + "recall": 0.6139, + "total": 2970, + "trueNegative": 2601, + "truePositive": 221, + "wilsonPrecisionLower95": 0.9273 + }, + "threshold": 0.97 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.3604, + "falseNegative": 70, + "falsePositive": 1, + "precision": 0.9524, + "predictedPositive": 21, + "recall": 0.2222, + "total": 1485, + "trueNegative": 1394, + "truePositive": 20, + "wilsonPrecisionLower95": 0.7733 + }, + "threshold": 0.965 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8273, + "falseNegative": 85, + "falsePositive": 11, + "precision": 0.9544, + "predictedPositive": 241, + "recall": 0.7302, + "total": 1485, + "trueNegative": 1159, + "truePositive": 230, + "wilsonPrecisionLower95": 0.9201 + }, + "threshold": 0.985 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.797, + "falseNegative": 99, + "falsePositive": 11, + "precision": 0.9515, + "predictedPositive": 227, + "recall": 0.6857, + "total": 1485, + "trueNegative": 1159, + "truePositive": 216, + "wilsonPrecisionLower95": 0.9153 + }, + "threshold": 0.835 + } + }, + "metrics": { + "f1": 0.8051, + "falseNegative": 190, + "falsePositive": 23, + "precision": 0.9503, + "predictedPositive": 463, + "recall": 0.6984, + "total": 2970, + "trueNegative": 2317, + "truePositive": 440, + "wilsonPrecisionLower95": 0.9266 + }, + "threshold": 0.94 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9343, + "falseNegative": 77, + "falsePositive": 45, + "precision": 0.9507, + "predictedPositive": 913, + "recall": 0.9185, + "total": 1485, + "trueNegative": 495, + "truePositive": 868, + "wilsonPrecisionLower95": 0.9347 + }, + "threshold": 0.91 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8207, + "falseNegative": 263, + "falsePositive": 35, + "precision": 0.9512, + "predictedPositive": 717, + "recall": 0.7217, + "total": 1485, + "trueNegative": 505, + "truePositive": 682, + "wilsonPrecisionLower95": 0.9329 + }, + "threshold": 0.98 + } + }, + "metrics": { + "f1": 0.8572, + "falseNegative": 419, + "falsePositive": 71, + "precision": 0.954, + "predictedPositive": 1542, + "recall": 0.7783, + "total": 2970, + "trueNegative": 1009, + "truePositive": 1471, + "wilsonPrecisionLower95": 0.9423 + }, + "threshold": 0.98 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9888, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 44, + "recall": 0.9778, + "total": 1485, + "trueNegative": 1440, + "truePositive": 44, + "wilsonPrecisionLower95": 0.9197 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.989, + "falseNegative": 0, + "falsePositive": 1, + "precision": 0.9783, + "predictedPositive": 46, + "recall": 1.0, + "total": 1485, + "trueNegative": 1439, + "truePositive": 45, + "wilsonPrecisionLower95": 0.8866 + }, + "threshold": 0.955 + } + }, + "metrics": { + "f1": 0.9778, + "falseNegative": 2, + "falsePositive": 2, + "precision": 0.9778, + "predictedPositive": 90, + "recall": 0.9778, + "total": 2970, + "trueNegative": 2878, + "truePositive": 88, + "wilsonPrecisionLower95": 0.9226 + }, + "threshold": 0.935 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8836, + "falseNegative": 64, + "falsePositive": 14, + "precision": 0.9548, + "predictedPositive": 310, + "recall": 0.8222, + "total": 1485, + "trueNegative": 1111, + "truePositive": 296, + "wilsonPrecisionLower95": 0.9256 + }, + "threshold": 0.975 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7896, + "falseNegative": 118, + "falsePositive": 11, + "precision": 0.9565, + "predictedPositive": 253, + "recall": 0.6722, + "total": 1485, + "trueNegative": 1114, + "truePositive": 242, + "wilsonPrecisionLower95": 0.9238 + }, + "threshold": 0.96 + } + }, + "metrics": { + "f1": 0.8316, + "falseNegative": 189, + "falsePositive": 26, + "precision": 0.9533, + "predictedPositive": 557, + "recall": 0.7375, + "total": 2970, + "trueNegative": 2224, + "truePositive": 531, + "wilsonPrecisionLower95": 0.9325 + }, + "threshold": 0.97 + } + }, + "trainingRecords": 16726, + "validation": { + "intentsMeeting95Precision": 9, + "macroF1": 0.7397, + "macroPrecision": 0.9707, + "macroRecall": 0.6388, + "minimumPrecision": 0.9509, + "perIntent": { + "blessing": { + "f1": 0.3636, + "falseNegative": 70, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 20, + "recall": 0.2222, + "total": 2970, + "trueNegative": 2880, + "truePositive": 20, + "wilsonPrecisionLower95": 0.8389 + }, + "complaint": { + "f1": 0.9109, + "falseNegative": 48, + "falsePositive": 13, + "precision": 0.96, + "predictedPositive": 325, + "recall": 0.8667, + "total": 2970, + "trueNegative": 2597, + "truePositive": 312, + "wilsonPrecisionLower95": 0.9328 + }, + "confirmationDecision": { + "f1": 0.6236, + "falseNegative": 98, + "falsePositive": 1, + "precision": 0.988, + "predictedPositive": 83, + "recall": 0.4556, + "total": 2970, + "trueNegative": 2789, + "truePositive": 82, + "wilsonPrecisionLower95": 0.9349 + }, + "followUpReminder": { + "f1": 0.75, + "falseNegative": 138, + "falsePositive": 10, + "precision": 0.9569, + "predictedPositive": 232, + "recall": 0.6167, + "total": 2970, + "trueNegative": 2600, + "truePositive": 222, + "wilsonPrecisionLower95": 0.9225 + }, + "invitation": { + "f1": 0.4917, + "falseNegative": 121, + "falsePositive": 1, + "precision": 0.9833, + "predictedPositive": 60, + "recall": 0.3278, + "total": 2970, + "trueNegative": 2789, + "truePositive": 59, + "wilsonPrecisionLower95": 0.9114 + }, + "question": { + "f1": 0.8124, + "falseNegative": 184, + "falsePositive": 22, + "precision": 0.953, + "predictedPositive": 468, + "recall": 0.7079, + "total": 2970, + "trueNegative": 2318, + "truePositive": 446, + "wilsonPrecisionLower95": 0.9299 + }, + "replyableMessage": { + "f1": 0.8807, + "falseNegative": 340, + "falsePositive": 80, + "precision": 0.9509, + "predictedPositive": 1630, + "recall": 0.8201, + "total": 2970, + "trueNegative": 1000, + "truePositive": 1550, + "wilsonPrecisionLower95": 0.9393 + }, + "scheduleNegotiation": { + "f1": 0.9889, + "falseNegative": 1, + "falsePositive": 1, + "precision": 0.9889, + "predictedPositive": 90, + "recall": 0.9889, + "total": 2970, + "trueNegative": 2879, + "truePositive": 89, + "wilsonPrecisionLower95": 0.9397 + }, + "task": { + "f1": 0.8359, + "falseNegative": 185, + "falsePositive": 25, + "precision": 0.9554, + "predictedPositive": 560, + "recall": 0.7431, + "total": 2970, + "trueNegative": 2225, + "truePositive": 535, + "wilsonPrecisionLower95": 0.9349 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 2e-06, + "augmentation": "numeric", + "char_max": 7, + "char_min": 3, + "external_weight": 0.8900000000000001, + "hard_example_weight": 2.5, + "l1_ratio": 0.2, + "max_char_features": 85000, + "max_word_features": 42000, + "min_df": 1, + "pseudo_weight": 0.405, + "round": 17, + "word_max": 3 + }, + "featureCount": 127000, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 1 + }, + "complaint": { + "negative": 250, + "positive": 4 + }, + "confirmationDecision": { + "negative": 250, + "positive": 23 + }, + "followUpReminder": { + "negative": 250, + "positive": 3 + }, + "invitation": { + "negative": 250, + "positive": 0 + }, + "question": { + "negative": 250, + "positive": 0 + }, + "replyableMessage": { + "negative": 70, + "positive": 250 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 0 + }, + "task": { + "negative": 250, + "positive": 0 + } + }, + "round": 17, + "selectionObjective": 0.6932, + "syntheticRegression": { + "intentsMeeting95Precision": 4, + "macroF1": 0.5086, + "macroPrecision": 0.9202, + "macroRecall": 0.4091, + "minimumPrecision": 0.7329, + "perIntent": { + "blessing": { + "f1": 0.1633, + "falseNegative": 82, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 8, + "recall": 0.0889, + "total": 2970, + "trueNegative": 2880, + "truePositive": 8, + "wilsonPrecisionLower95": 0.6756 + }, + "complaint": { + "f1": 0.914, + "falseNegative": 36, + "falsePositive": 25, + "precision": 0.9284, + "predictedPositive": 349, + "recall": 0.9, + "total": 2970, + "trueNegative": 2585, + "truePositive": 324, + "wilsonPrecisionLower95": 0.8964 + }, + "confirmationDecision": { + "f1": 0.6564, + "falseNegative": 73, + "falsePositive": 39, + "precision": 0.7329, + "predictedPositive": 146, + "recall": 0.5944, + "total": 2970, + "trueNegative": 2751, + "truePositive": 107, + "wilsonPrecisionLower95": 0.6558 + }, + "followUpReminder": { + "f1": 0.4, + "falseNegative": 270, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 90, + "recall": 0.25, + "total": 2970, + "trueNegative": 2610, + "truePositive": 90, + "wilsonPrecisionLower95": 0.9591 + }, + "invitation": { + "f1": 0.5954, + "falseNegative": 102, + "falsePositive": 4, + "precision": 0.9512, + "predictedPositive": 82, + "recall": 0.4333, + "total": 2970, + "trueNegative": 2786, + "truePositive": 78, + "wilsonPrecisionLower95": 0.8812 + }, + "question": { + "f1": 0.4976, + "falseNegative": 419, + "falsePositive": 7, + "precision": 0.9679, + "predictedPositive": 218, + "recall": 0.3349, + "total": 2970, + "trueNegative": 2333, + "truePositive": 211, + "wilsonPrecisionLower95": 0.9352 + }, + "replyableMessage": { + "f1": 0.2549, + "falseNegative": 1612, + "falsePositive": 13, + "precision": 0.9553, + "predictedPositive": 291, + "recall": 0.1471, + "total": 2970, + "trueNegative": 1067, + "truePositive": 278, + "wilsonPrecisionLower95": 0.9251 + }, + "scheduleNegotiation": { + "f1": 0.8333, + "falseNegative": 20, + "falsePositive": 8, + "precision": 0.8974, + "predictedPositive": 78, + "recall": 0.7778, + "total": 2970, + "trueNegative": 2872, + "truePositive": 70, + "wilsonPrecisionLower95": 0.8105 + }, + "task": { + "f1": 0.2629, + "falseNegative": 608, + "falsePositive": 20, + "precision": 0.8485, + "predictedPositive": 132, + "recall": 0.1556, + "total": 2970, + "trueNegative": 2230, + "truePositive": 112, + "wilsonPrecisionLower95": 0.7775 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.989, + "falseNegative": 0, + "falsePositive": 4, + "precision": 0.9783, + "predictedPositive": 184, + "recall": 1.0, + "total": 1485, + "trueNegative": 1301, + "truePositive": 180, + "wilsonPrecisionLower95": 0.9454 + }, + "threshold": 0.995 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8869, + "falseNegative": 31, + "falsePositive": 7, + "precision": 0.9551, + "predictedPositive": 156, + "recall": 0.8278, + "total": 1485, + "trueNegative": 1298, + "truePositive": 149, + "wilsonPrecisionLower95": 0.9103 + }, + "threshold": 0.995 + } + }, + "metrics": { + "f1": 0.944, + "falseNegative": 23, + "falsePositive": 17, + "precision": 0.952, + "predictedPositive": 354, + "recall": 0.9361, + "total": 2970, + "trueNegative": 2593, + "truePositive": 337, + "wilsonPrecisionLower95": 0.9244 + }, + "threshold": 0.975 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8447, + "falseNegative": 22, + "falsePositive": 3, + "precision": 0.9577, + "predictedPositive": 71, + "recall": 0.7556, + "total": 1485, + "trueNegative": 1392, + "truePositive": 68, + "wilsonPrecisionLower95": 0.883 + }, + "threshold": 0.825 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.7723, + "falseNegative": 63, + "falsePositive": 6, + "precision": 0.9512, + "predictedPositive": 123, + "recall": 0.65, + "total": 2970, + "trueNegative": 2784, + "truePositive": 117, + "wilsonPrecisionLower95": 0.8977 + }, + "threshold": 0.965 + }, + "followUpReminder": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6625, + "falseNegative": 155, + "falsePositive": 8, + "precision": 0.9524, + "predictedPositive": 168, + "recall": 0.5079, + "total": 1485, + "trueNegative": 1162, + "truePositive": 160, + "wilsonPrecisionLower95": 0.9089 + }, + "threshold": 0.97 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "replyableMessage": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9655, + "falseNegative": 3, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 42, + "recall": 0.9333, + "total": 1485, + "trueNegative": 1440, + "truePositive": 42, + "wilsonPrecisionLower95": 0.9162 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9425, + "falseNegative": 4, + "falsePositive": 1, + "precision": 0.9762, + "predictedPositive": 42, + "recall": 0.9111, + "total": 1485, + "trueNegative": 1439, + "truePositive": 41, + "wilsonPrecisionLower95": 0.8768 + }, + "threshold": 0.73 + } + }, + "metrics": { + "f1": 0.9486, + "falseNegative": 7, + "falsePositive": 2, + "precision": 0.9765, + "predictedPositive": 85, + "recall": 0.9222, + "total": 2970, + "trueNegative": 2878, + "truePositive": 83, + "wilsonPrecisionLower95": 0.9182 + }, + "threshold": 0.5 + }, + "task": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "trainingRecords": 16820, + "validation": { + "intentsMeeting95Precision": 8, + "macroF1": 0.5631, + "macroPrecision": 0.9693, + "macroRecall": 0.4421, + "minimumPrecision": 0.9474, + "perIntent": { + "blessing": { + "f1": 0.3019, + "falseNegative": 74, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 16, + "recall": 0.1778, + "total": 2970, + "trueNegative": 2880, + "truePositive": 16, + "wilsonPrecisionLower95": 0.8064 + }, + "complaint": { + "f1": 0.94, + "falseNegative": 31, + "falsePositive": 11, + "precision": 0.9676, + "predictedPositive": 340, + "recall": 0.9139, + "total": 2970, + "trueNegative": 2599, + "truePositive": 329, + "wilsonPrecisionLower95": 0.943 + }, + "confirmationDecision": { + "f1": 0.6545, + "falseNegative": 90, + "falsePositive": 5, + "precision": 0.9474, + "predictedPositive": 95, + "recall": 0.5, + "total": 2970, + "trueNegative": 2785, + "truePositive": 90, + "wilsonPrecisionLower95": 0.8827 + }, + "followUpReminder": { + "f1": 0.5282, + "falseNegative": 229, + "falsePositive": 5, + "precision": 0.9632, + "predictedPositive": 136, + "recall": 0.3639, + "total": 2970, + "trueNegative": 2605, + "truePositive": 131, + "wilsonPrecisionLower95": 0.9168 + }, + "invitation": { + "f1": 0.4378, + "falseNegative": 129, + "falsePositive": 2, + "precision": 0.9623, + "predictedPositive": 53, + "recall": 0.2833, + "total": 2970, + "trueNegative": 2788, + "truePositive": 51, + "wilsonPrecisionLower95": 0.8725 + }, + "question": { + "f1": 0.5428, + "falseNegative": 392, + "falsePositive": 9, + "precision": 0.9636, + "predictedPositive": 247, + "recall": 0.3778, + "total": 2970, + "trueNegative": 2331, + "truePositive": 238, + "wilsonPrecisionLower95": 0.9322 + }, + "replyableMessage": { + "f1": 0.2757, + "falseNegative": 1586, + "falsePositive": 11, + "precision": 0.9651, + "predictedPositive": 315, + "recall": 0.1608, + "total": 2970, + "trueNegative": 1069, + "truePositive": 304, + "wilsonPrecisionLower95": 0.9386 + }, + "scheduleNegotiation": { + "f1": 0.954, + "falseNegative": 7, + "falsePositive": 1, + "precision": 0.9881, + "predictedPositive": 84, + "recall": 0.9222, + "total": 2970, + "trueNegative": 2879, + "truePositive": 83, + "wilsonPrecisionLower95": 0.9356 + }, + "task": { + "f1": 0.4332, + "falseNegative": 519, + "falsePositive": 7, + "precision": 0.9663, + "predictedPositive": 208, + "recall": 0.2792, + "total": 2970, + "trueNegative": 2243, + "truePositive": 201, + "wilsonPrecisionLower95": 0.9322 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 1.5e-06, + "augmentation": "numeric", + "char_max": 8, + "char_min": 2, + "external_weight": 0.91, + "hard_example_weight": 2.5, + "l1_ratio": 0.2, + "max_char_features": 90000, + "max_word_features": 45000, + "min_df": 1, + "pseudo_weight": 0.42000000000000004, + "round": 18, + "word_max": 3 + }, + "featureCount": 135000, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 1 + }, + "complaint": { + "negative": 250, + "positive": 8 + }, + "confirmationDecision": { + "negative": 250, + "positive": 27 + }, + "followUpReminder": { + "negative": 250, + "positive": 0 + }, + "invitation": { + "negative": 250, + "positive": 0 + }, + "question": { + "negative": 250, + "positive": 0 + }, + "replyableMessage": { + "negative": 90, + "positive": 32 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 0 + }, + "task": { + "negative": 250, + "positive": 0 + } + }, + "round": 18, + "selectionObjective": 0.7725, + "syntheticRegression": { + "intentsMeeting95Precision": 4, + "macroF1": 0.6489, + "macroPrecision": 0.9452, + "macroRecall": 0.5429, + "minimumPrecision": 0.8205, + "perIntent": { + "blessing": { + "f1": 0.2178, + "falseNegative": 79, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 11, + "recall": 0.1222, + "total": 2970, + "trueNegative": 2880, + "truePositive": 11, + "wilsonPrecisionLower95": 0.7412 + }, + "complaint": { + "f1": 0.8661, + "falseNegative": 85, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 275, + "recall": 0.7639, + "total": 2970, + "trueNegative": 2610, + "truePositive": 275, + "wilsonPrecisionLower95": 0.9862 + }, + "confirmationDecision": { + "f1": 0.6465, + "falseNegative": 84, + "falsePositive": 21, + "precision": 0.8205, + "predictedPositive": 117, + "recall": 0.5333, + "total": 2970, + "trueNegative": 2769, + "truePositive": 96, + "wilsonPrecisionLower95": 0.7411 + }, + "followUpReminder": { + "f1": 0.5851, + "falseNegative": 207, + "falsePositive": 10, + "precision": 0.9387, + "predictedPositive": 163, + "recall": 0.425, + "total": 2970, + "trueNegative": 2600, + "truePositive": 153, + "wilsonPrecisionLower95": 0.8908 + }, + "invitation": { + "f1": 0.626, + "falseNegative": 98, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 82, + "recall": 0.4556, + "total": 2970, + "trueNegative": 2790, + "truePositive": 82, + "wilsonPrecisionLower95": 0.9552 + }, + "question": { + "f1": 0.7311, + "falseNegative": 267, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 363, + "recall": 0.5762, + "total": 2970, + "trueNegative": 2340, + "truePositive": 363, + "wilsonPrecisionLower95": 0.9895 + }, + "replyableMessage": { + "f1": 0.8939, + "falseNegative": 273, + "falsePositive": 111, + "precision": 0.9358, + "predictedPositive": 1728, + "recall": 0.8556, + "total": 2970, + "trueNegative": 969, + "truePositive": 1617, + "wilsonPrecisionLower95": 0.9232 + }, + "scheduleNegotiation": { + "f1": 0.9341, + "falseNegative": 5, + "falsePositive": 7, + "precision": 0.9239, + "predictedPositive": 92, + "recall": 0.9444, + "total": 2970, + "trueNegative": 2873, + "truePositive": 85, + "wilsonPrecisionLower95": 0.8512 + }, + "task": { + "f1": 0.3393, + "falseNegative": 569, + "falsePositive": 19, + "precision": 0.8882, + "predictedPositive": 170, + "recall": 0.2097, + "total": 2970, + "trueNegative": 2231, + "truePositive": 151, + "wilsonPrecisionLower95": 0.832 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9486, + "falseNegative": 7, + "falsePositive": 2, + "precision": 0.9765, + "predictedPositive": 85, + "recall": 0.9222, + "total": 1485, + "trueNegative": 1393, + "truePositive": 83, + "wilsonPrecisionLower95": 0.9182 + }, + "threshold": 0.9 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.8447, + "falseNegative": 44, + "falsePositive": 6, + "precision": 0.9577, + "predictedPositive": 142, + "recall": 0.7556, + "total": 2970, + "trueNegative": 2784, + "truePositive": 136, + "wilsonPrecisionLower95": 0.9109 + }, + "threshold": 0.955 + }, + "followUpReminder": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8822, + "falseNegative": 34, + "falsePositive": 5, + "precision": 0.9669, + "predictedPositive": 151, + "recall": 0.8111, + "total": 1485, + "trueNegative": 1300, + "truePositive": 146, + "wilsonPrecisionLower95": 0.9248 + }, + "threshold": 0.98 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.7641, + "falseNegative": 130, + "falsePositive": 12, + "precision": 0.9504, + "predictedPositive": 242, + "recall": 0.6389, + "total": 2970, + "trueNegative": 2598, + "truePositive": 230, + "wilsonPrecisionLower95": 0.9153 + }, + "threshold": 0.99 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8103, + "falseNegative": 95, + "falsePositive": 8, + "precision": 0.9649, + "predictedPositive": 228, + "recall": 0.6984, + "total": 1485, + "trueNegative": 1162, + "truePositive": 220, + "wilsonPrecisionLower95": 0.9323 + }, + "threshold": 0.915 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9575, + "falseNegative": 33, + "falsePositive": 48, + "precision": 0.95, + "predictedPositive": 960, + "recall": 0.9651, + "total": 1485, + "trueNegative": 492, + "truePositive": 912, + "wilsonPrecisionLower95": 0.9343 + }, + "threshold": 0.845 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8361, + "falseNegative": 241, + "falsePositive": 35, + "precision": 0.9526, + "predictedPositive": 739, + "recall": 0.745, + "total": 1485, + "trueNegative": 505, + "truePositive": 704, + "wilsonPrecisionLower95": 0.9348 + }, + "threshold": 0.98 + } + }, + "metrics": { + "f1": 0.901, + "falseNegative": 274, + "falsePositive": 81, + "precision": 0.9523, + "predictedPositive": 1697, + "recall": 0.855, + "total": 2970, + "trueNegative": 999, + "truePositive": 1616, + "wilsonPrecisionLower95": 0.9411 + }, + "threshold": 0.975 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9655, + "falseNegative": 3, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 42, + "recall": 0.9333, + "total": 1485, + "trueNegative": 1440, + "truePositive": 42, + "wilsonPrecisionLower95": 0.9162 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 45, + "recall": 1.0, + "total": 1485, + "trueNegative": 1440, + "truePositive": 45, + "wilsonPrecisionLower95": 0.9213 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.9831, + "falseNegative": 3, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 87, + "recall": 0.9667, + "total": 2970, + "trueNegative": 2880, + "truePositive": 87, + "wilsonPrecisionLower95": 0.9577 + }, + "threshold": 0.5 + }, + "task": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "trainingRecords": 16841, + "validation": { + "intentsMeeting95Precision": 8, + "macroF1": 0.6817, + "macroPrecision": 0.9748, + "macroRecall": 0.5634, + "minimumPrecision": 0.931, + "perIntent": { + "blessing": { + "f1": 0.3486, + "falseNegative": 71, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 19, + "recall": 0.2111, + "total": 2970, + "trueNegative": 2880, + "truePositive": 19, + "wilsonPrecisionLower95": 0.8318 + }, + "complaint": { + "f1": 0.88, + "falseNegative": 74, + "falsePositive": 4, + "precision": 0.9862, + "predictedPositive": 290, + "recall": 0.7944, + "total": 2970, + "trueNegative": 2606, + "truePositive": 286, + "wilsonPrecisionLower95": 0.9651 + }, + "confirmationDecision": { + "f1": 0.7183, + "falseNegative": 78, + "falsePositive": 2, + "precision": 0.9808, + "predictedPositive": 104, + "recall": 0.5667, + "total": 2970, + "trueNegative": 2788, + "truePositive": 102, + "wilsonPrecisionLower95": 0.9326 + }, + "followUpReminder": { + "f1": 0.6345, + "falseNegative": 189, + "falsePositive": 8, + "precision": 0.9553, + "predictedPositive": 179, + "recall": 0.475, + "total": 2970, + "trueNegative": 2602, + "truePositive": 171, + "wilsonPrecisionLower95": 0.9143 + }, + "invitation": { + "f1": 0.4538, + "falseNegative": 126, + "falsePositive": 4, + "precision": 0.931, + "predictedPositive": 58, + "recall": 0.3, + "total": 2970, + "trueNegative": 2786, + "truePositive": 54, + "wilsonPrecisionLower95": 0.8357 + }, + "question": { + "f1": 0.7187, + "falseNegative": 271, + "falsePositive": 10, + "precision": 0.9729, + "predictedPositive": 369, + "recall": 0.5698, + "total": 2970, + "trueNegative": 2330, + "truePositive": 359, + "wilsonPrecisionLower95": 0.9508 + }, + "replyableMessage": { + "f1": 0.9005, + "falseNegative": 274, + "falsePositive": 83, + "precision": 0.9511, + "predictedPositive": 1699, + "recall": 0.855, + "total": 2970, + "trueNegative": 997, + "truePositive": 1616, + "wilsonPrecisionLower95": 0.9398 + }, + "scheduleNegotiation": { + "f1": 0.9831, + "falseNegative": 3, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 87, + "recall": 0.9667, + "total": 2970, + "trueNegative": 2880, + "truePositive": 87, + "wilsonPrecisionLower95": 0.9577 + }, + "task": { + "f1": 0.4979, + "falseNegative": 481, + "falsePositive": 1, + "precision": 0.9958, + "predictedPositive": 240, + "recall": 0.3319, + "total": 2970, + "trueNegative": 2249, + "truePositive": 239, + "wilsonPrecisionLower95": 0.9768 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 1.2e-06, + "augmentation": "numeric", + "char_max": 8, + "char_min": 3, + "external_weight": 0.93, + "hard_example_weight": 2.7, + "l1_ratio": 0.25, + "max_char_features": 95000, + "max_word_features": 45000, + "min_df": 1, + "pseudo_weight": 0.43499999999999994, + "round": 19, + "word_max": 3 + }, + "featureCount": 140000, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 1 + }, + "complaint": { + "negative": 250, + "positive": 2 + }, + "confirmationDecision": { + "negative": 250, + "positive": 36 + }, + "followUpReminder": { + "negative": 250, + "positive": 0 + }, + "invitation": { + "negative": 250, + "positive": 0 + }, + "question": { + "negative": 250, + "positive": 4 + }, + "replyableMessage": { + "negative": 99, + "positive": 32 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 0 + }, + "task": { + "negative": 250, + "positive": 0 + } + }, + "round": 19, + "selectionObjective": 0.7336, + "syntheticRegression": { + "intentsMeeting95Precision": 6, + "macroF1": 0.6005, + "macroPrecision": 0.9664, + "macroRecall": 0.4678, + "minimumPrecision": 0.887, + "perIntent": { + "blessing": { + "f1": 0.2, + "falseNegative": 80, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 10, + "recall": 0.1111, + "total": 2970, + "trueNegative": 2880, + "truePositive": 10, + "wilsonPrecisionLower95": 0.7225 + }, + "complaint": { + "f1": 0.9205, + "falseNegative": 53, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 307, + "recall": 0.8528, + "total": 2970, + "trueNegative": 2610, + "truePositive": 307, + "wilsonPrecisionLower95": 0.9876 + }, + "confirmationDecision": { + "f1": 0.6915, + "falseNegative": 78, + "falsePositive": 13, + "precision": 0.887, + "predictedPositive": 115, + "recall": 0.5667, + "total": 2970, + "trueNegative": 2777, + "truePositive": 102, + "wilsonPrecisionLower95": 0.8162 + }, + "followUpReminder": { + "f1": 0.4778, + "falseNegative": 247, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 113, + "recall": 0.3139, + "total": 2970, + "trueNegative": 2610, + "truePositive": 113, + "wilsonPrecisionLower95": 0.9671 + }, + "invitation": { + "f1": 0.7092, + "falseNegative": 80, + "falsePositive": 2, + "precision": 0.9804, + "predictedPositive": 102, + "recall": 0.5556, + "total": 2970, + "trueNegative": 2788, + "truePositive": 100, + "wilsonPrecisionLower95": 0.9313 + }, + "question": { + "f1": 0.4703, + "falseNegative": 436, + "falsePositive": 1, + "precision": 0.9949, + "predictedPositive": 195, + "recall": 0.3079, + "total": 2970, + "trueNegative": 2339, + "truePositive": 194, + "wilsonPrecisionLower95": 0.9715 + }, + "replyableMessage": { + "f1": 0.6399, + "falseNegative": 957, + "falsePositive": 93, + "precision": 0.9094, + "predictedPositive": 1026, + "recall": 0.4937, + "total": 2970, + "trueNegative": 987, + "truePositive": 933, + "wilsonPrecisionLower95": 0.8902 + }, + "scheduleNegotiation": { + "f1": 0.8079, + "falseNegative": 29, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 61, + "recall": 0.6778, + "total": 2970, + "trueNegative": 2880, + "truePositive": 61, + "wilsonPrecisionLower95": 0.9408 + }, + "task": { + "f1": 0.4872, + "falseNegative": 482, + "falsePositive": 19, + "precision": 0.9261, + "predictedPositive": 257, + "recall": 0.3306, + "total": 2970, + "trueNegative": 2231, + "truePositive": 238, + "wilsonPrecisionLower95": 0.8874 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "confirmationDecision": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "followUpReminder": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "replyableMessage": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9309, + "falseNegative": 83, + "falsePositive": 45, + "precision": 0.9504, + "predictedPositive": 907, + "recall": 0.9122, + "total": 1485, + "trueNegative": 495, + "truePositive": 862, + "wilsonPrecisionLower95": 0.9343 + }, + "threshold": 0.98 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 45, + "recall": 1.0, + "total": 1485, + "trueNegative": 1440, + "truePositive": 45, + "wilsonPrecisionLower95": 0.9213 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9195, + "falseNegative": 5, + "falsePositive": 2, + "precision": 0.9524, + "predictedPositive": 42, + "recall": 0.8889, + "total": 1485, + "trueNegative": 1438, + "truePositive": 40, + "wilsonPrecisionLower95": 0.8421 + }, + "threshold": 0.945 + } + }, + "metrics": { + "f1": 0.9667, + "falseNegative": 3, + "falsePositive": 3, + "precision": 0.9667, + "predictedPositive": 90, + "recall": 0.9667, + "total": 2970, + "trueNegative": 2877, + "truePositive": 87, + "wilsonPrecisionLower95": 0.9065 + }, + "threshold": 0.83 + }, + "task": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "trainingRecords": 16801, + "validation": { + "intentsMeeting95Precision": 6, + "macroF1": 0.6316, + "macroPrecision": 0.9616, + "macroRecall": 0.4969, + "minimumPrecision": 0.9242, + "perIntent": { + "blessing": { + "f1": 0.4483, + "falseNegative": 64, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 26, + "recall": 0.2889, + "total": 2970, + "trueNegative": 2880, + "truePositive": 26, + "wilsonPrecisionLower95": 0.8713 + }, + "complaint": { + "f1": 0.8953, + "falseNegative": 65, + "falsePositive": 4, + "precision": 0.9866, + "predictedPositive": 299, + "recall": 0.8194, + "total": 2970, + "trueNegative": 2606, + "truePositive": 295, + "wilsonPrecisionLower95": 0.9661 + }, + "confirmationDecision": { + "f1": 0.5891, + "falseNegative": 104, + "falsePositive": 2, + "precision": 0.9744, + "predictedPositive": 78, + "recall": 0.4222, + "total": 2970, + "trueNegative": 2788, + "truePositive": 76, + "wilsonPrecisionLower95": 0.9112 + }, + "followUpReminder": { + "f1": 0.528, + "falseNegative": 228, + "falsePositive": 8, + "precision": 0.9429, + "predictedPositive": 140, + "recall": 0.3667, + "total": 2970, + "trueNegative": 2602, + "truePositive": 132, + "wilsonPrecisionLower95": 0.8913 + }, + "invitation": { + "f1": 0.6015, + "falseNegative": 100, + "falsePositive": 6, + "precision": 0.9302, + "predictedPositive": 86, + "recall": 0.4444, + "total": 2970, + "trueNegative": 2784, + "truePositive": 80, + "wilsonPrecisionLower95": 0.856 + }, + "question": { + "f1": 0.442, + "falseNegative": 447, + "falsePositive": 15, + "precision": 0.9242, + "predictedPositive": 198, + "recall": 0.2905, + "total": 2970, + "trueNegative": 2325, + "truePositive": 183, + "wilsonPrecisionLower95": 0.8788 + }, + "replyableMessage": { + "f1": 0.6347, + "falseNegative": 990, + "falsePositive": 46, + "precision": 0.9514, + "predictedPositive": 946, + "recall": 0.4762, + "total": 2970, + "trueNegative": 1034, + "truePositive": 900, + "wilsonPrecisionLower95": 0.9357 + }, + "scheduleNegotiation": { + "f1": 0.9605, + "falseNegative": 5, + "falsePositive": 2, + "precision": 0.977, + "predictedPositive": 87, + "recall": 0.9444, + "total": 2970, + "trueNegative": 2878, + "truePositive": 85, + "wilsonPrecisionLower95": 0.92 + }, + "task": { + "f1": 0.5853, + "falseNegative": 418, + "falsePositive": 10, + "precision": 0.9679, + "predictedPositive": 312, + "recall": 0.4194, + "total": 2970, + "trueNegative": 2240, + "truePositive": 302, + "wilsonPrecisionLower95": 0.942 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 1e-06, + "augmentation": "numeric", + "char_max": 8, + "char_min": 2, + "external_weight": 0.9500000000000001, + "hard_example_weight": 3.0, + "l1_ratio": 0.25, + "max_char_features": 100000, + "max_word_features": 50000, + "min_df": 1, + "pseudo_weight": 0.44999999999999996, + "round": 20, + "word_max": 3 + }, + "featureCount": 150000, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 1 + }, + "complaint": { + "negative": 250, + "positive": 2 + }, + "confirmationDecision": { + "negative": 250, + "positive": 25 + }, + "followUpReminder": { + "negative": 250, + "positive": 0 + }, + "invitation": { + "negative": 250, + "positive": 1 + }, + "question": { + "negative": 250, + "positive": 2 + }, + "replyableMessage": { + "negative": 104, + "positive": 32 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 1 + }, + "task": { + "negative": 250, + "positive": 0 + } + }, + "round": 20, + "selectionObjective": 0.7724, + "syntheticRegression": { + "intentsMeeting95Precision": 7, + "macroF1": 0.6637, + "macroPrecision": 0.9685, + "macroRecall": 0.5421, + "minimumPrecision": 0.9094, + "perIntent": { + "blessing": { + "f1": 0.2353, + "falseNegative": 78, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 12, + "recall": 0.1333, + "total": 2970, + "trueNegative": 2880, + "truePositive": 12, + "wilsonPrecisionLower95": 0.7575 + }, + "complaint": { + "f1": 0.8274, + "falseNegative": 106, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 254, + "recall": 0.7056, + "total": 2970, + "trueNegative": 2610, + "truePositive": 254, + "wilsonPrecisionLower95": 0.9851 + }, + "confirmationDecision": { + "f1": 0.8261, + "falseNegative": 47, + "falsePositive": 9, + "precision": 0.9366, + "predictedPositive": 142, + "recall": 0.7389, + "total": 2970, + "trueNegative": 2781, + "truePositive": 133, + "wilsonPrecisionLower95": 0.8839 + }, + "followUpReminder": { + "f1": 0.5155, + "falseNegative": 235, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 125, + "recall": 0.3472, + "total": 2970, + "trueNegative": 2610, + "truePositive": 125, + "wilsonPrecisionLower95": 0.9702 + }, + "invitation": { + "f1": 0.7241, + "falseNegative": 75, + "falsePositive": 5, + "precision": 0.9545, + "predictedPositive": 110, + "recall": 0.5833, + "total": 2970, + "trueNegative": 2785, + "truePositive": 105, + "wilsonPrecisionLower95": 0.898 + }, + "question": { + "f1": 0.5534, + "falseNegative": 389, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 241, + "recall": 0.3825, + "total": 2970, + "trueNegative": 2340, + "truePositive": 241, + "wilsonPrecisionLower95": 0.9843 + }, + "replyableMessage": { + "f1": 0.7688, + "falseNegative": 671, + "falsePositive": 62, + "precision": 0.9516, + "predictedPositive": 1281, + "recall": 0.645, + "total": 2970, + "trueNegative": 1018, + "truePositive": 1219, + "wilsonPrecisionLower95": 0.9384 + }, + "scheduleNegotiation": { + "f1": 0.9371, + "falseNegative": 8, + "falsePositive": 3, + "precision": 0.9647, + "predictedPositive": 85, + "recall": 0.9111, + "total": 2970, + "trueNegative": 2877, + "truePositive": 82, + "wilsonPrecisionLower95": 0.9013 + }, + "task": { + "f1": 0.5857, + "falseNegative": 409, + "falsePositive": 31, + "precision": 0.9094, + "predictedPositive": 342, + "recall": 0.4319, + "total": 2970, + "trueNegative": 2219, + "truePositive": 311, + "wilsonPrecisionLower95": 0.8742 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "confirmationDecision": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.5954, + "falseNegative": 51, + "falsePositive": 2, + "precision": 0.9512, + "predictedPositive": 41, + "recall": 0.4333, + "total": 1485, + "trueNegative": 1393, + "truePositive": 39, + "wilsonPrecisionLower95": 0.8386 + }, + "threshold": 0.995 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "followUpReminder": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8117, + "falseNegative": 279, + "falsePositive": 30, + "precision": 0.9569, + "predictedPositive": 696, + "recall": 0.7048, + "total": 1485, + "trueNegative": 510, + "truePositive": 666, + "wilsonPrecisionLower95": 0.9391 + }, + "threshold": 0.995 + } + }, + "metrics": { + "f1": 0.8736, + "falseNegative": 363, + "falsePositive": 79, + "precision": 0.9508, + "predictedPositive": 1606, + "recall": 0.8079, + "total": 2970, + "trueNegative": 1001, + "truePositive": 1527, + "wilsonPrecisionLower95": 0.9391 + }, + "threshold": 0.995 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 45, + "recall": 1.0, + "total": 1485, + "trueNegative": 1440, + "truePositive": 45, + "wilsonPrecisionLower95": 0.9213 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.967, + "falseNegative": 1, + "falsePositive": 2, + "precision": 0.9565, + "predictedPositive": 46, + "recall": 0.9778, + "total": 1485, + "trueNegative": 1438, + "truePositive": 44, + "wilsonPrecisionLower95": 0.8547 + }, + "threshold": 0.83 + } + }, + "metrics": { + "f1": 0.9834, + "falseNegative": 1, + "falsePositive": 2, + "precision": 0.978, + "predictedPositive": 91, + "recall": 0.9889, + "total": 2970, + "trueNegative": 2878, + "truePositive": 89, + "wilsonPrecisionLower95": 0.9234 + }, + "threshold": 0.83 + }, + "task": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "trainingRecords": 16755, + "validation": { + "intentsMeeting95Precision": 8, + "macroF1": 0.6867, + "macroPrecision": 0.9687, + "macroRecall": 0.5569, + "minimumPrecision": 0.9038, + "perIntent": { + "blessing": { + "f1": 0.3929, + "falseNegative": 68, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 22, + "recall": 0.2444, + "total": 2970, + "trueNegative": 2880, + "truePositive": 22, + "wilsonPrecisionLower95": 0.8513 + }, + "complaint": { + "f1": 0.8066, + "falseNegative": 114, + "falsePositive": 4, + "precision": 0.984, + "predictedPositive": 250, + "recall": 0.6833, + "total": 2970, + "trueNegative": 2606, + "truePositive": 246, + "wilsonPrecisionLower95": 0.9596 + }, + "confirmationDecision": { + "f1": 0.6618, + "falseNegative": 90, + "falsePositive": 2, + "precision": 0.9783, + "predictedPositive": 92, + "recall": 0.5, + "total": 2970, + "trueNegative": 2788, + "truePositive": 90, + "wilsonPrecisionLower95": 0.9242 + }, + "followUpReminder": { + "f1": 0.5977, + "falseNegative": 204, + "falsePositive": 6, + "precision": 0.963, + "predictedPositive": 162, + "recall": 0.4333, + "total": 2970, + "trueNegative": 2604, + "truePositive": 156, + "wilsonPrecisionLower95": 0.9216 + }, + "invitation": { + "f1": 0.662, + "falseNegative": 86, + "falsePositive": 10, + "precision": 0.9038, + "predictedPositive": 104, + "recall": 0.5222, + "total": 2970, + "trueNegative": 2780, + "truePositive": 94, + "wilsonPrecisionLower95": 0.832 + }, + "question": { + "f1": 0.5643, + "falseNegative": 380, + "falsePositive": 6, + "precision": 0.9766, + "predictedPositive": 256, + "recall": 0.3968, + "total": 2970, + "trueNegative": 2334, + "truePositive": 250, + "wilsonPrecisionLower95": 0.9498 + }, + "replyableMessage": { + "f1": 0.7934, + "falseNegative": 621, + "falsePositive": 40, + "precision": 0.9694, + "predictedPositive": 1309, + "recall": 0.6714, + "total": 2970, + "trueNegative": 1040, + "truePositive": 1269, + "wilsonPrecisionLower95": 0.9587 + }, + "scheduleNegotiation": { + "f1": 0.9834, + "falseNegative": 1, + "falsePositive": 2, + "precision": 0.978, + "predictedPositive": 91, + "recall": 0.9889, + "total": 2970, + "trueNegative": 2878, + "truePositive": 89, + "wilsonPrecisionLower95": 0.9234 + }, + "task": { + "f1": 0.7184, + "falseNegative": 308, + "falsePositive": 15, + "precision": 0.9649, + "predictedPositive": 427, + "recall": 0.5722, + "total": 2970, + "trueNegative": 2235, + "truePositive": 412, + "wilsonPrecisionLower95": 0.9429 + } + }, + "recordCount": 2970 + } + } +] diff --git a/ModelTraining/ClipboardSemantics/IterativeResearch/final-report.json b/ModelTraining/ClipboardSemantics/IterativeResearch/final-report.json new file mode 100644 index 0000000..96e85d0 --- /dev/null +++ b/ModelTraining/ClipboardSemantics/IterativeResearch/final-report.json @@ -0,0 +1,8411 @@ +{ + "baseTrainingRecords": 11500, + "deploymentDecision": "retain-current-deployed-models", + "elapsedSeconds": 170.1895, + "finalEvaluations": { + "golden": { + "byLanguage": { + "en": { + "intentsMeeting95Precision": 1, + "macroF1": 0.8445, + "macroPrecision": 0.9286, + "macroRecall": 0.8066, + "minimumPrecision": 0.7692, + "perIntent": { + "blessing": { + "f1": 0.8235, + "falseNegative": 3, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 7, + "recall": 0.7, + "total": 126, + "trueNegative": 116, + "truePositive": 7, + "wilsonPrecisionLower95": 0.6457 + }, + "complaint": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 6, + "recall": 1.0, + "total": 126, + "trueNegative": 120, + "truePositive": 6, + "wilsonPrecisionLower95": 0.6097 + }, + "confirmationDecision": { + "f1": 0.8696, + "falseNegative": 0, + "falsePositive": 3, + "precision": 0.7692, + "predictedPositive": 13, + "recall": 1.0, + "total": 126, + "trueNegative": 113, + "truePositive": 10, + "wilsonPrecisionLower95": 0.4974 + }, + "followUpReminder": { + "f1": 0.8696, + "falseNegative": 3, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 10, + "recall": 0.7692, + "total": 126, + "trueNegative": 113, + "truePositive": 10, + "wilsonPrecisionLower95": 0.7225 + }, + "invitation": { + "f1": 0.8571, + "falseNegative": 2, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 6, + "recall": 0.75, + "total": 126, + "trueNegative": 118, + "truePositive": 6, + "wilsonPrecisionLower95": 0.6097 + }, + "question": { + "f1": 0.8636, + "falseNegative": 4, + "falsePositive": 2, + "precision": 0.9048, + "predictedPositive": 21, + "recall": 0.8261, + "total": 126, + "trueNegative": 101, + "truePositive": 19, + "wilsonPrecisionLower95": 0.7109 + }, + "replyableMessage": { + "f1": 0.8889, + "falseNegative": 6, + "falsePositive": 10, + "precision": 0.8649, + "predictedPositive": 74, + "recall": 0.9143, + "total": 126, + "trueNegative": 46, + "truePositive": 64, + "wilsonPrecisionLower95": 0.7688 + }, + "scheduleNegotiation": { + "f1": 0.8571, + "falseNegative": 1, + "falsePositive": 2, + "precision": 0.8182, + "predictedPositive": 11, + "recall": 0.9, + "total": 126, + "trueNegative": 114, + "truePositive": 9, + "wilsonPrecisionLower95": 0.523 + }, + "task": { + "f1": 0.5714, + "falseNegative": 9, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 6, + "recall": 0.4, + "total": 126, + "trueNegative": 111, + "truePositive": 6, + "wilsonPrecisionLower95": 0.6097 + } + }, + "recordCount": 126 + }, + "zh-Hans": { + "intentsMeeting95Precision": 1, + "macroF1": 0.6673, + "macroPrecision": 0.9541, + "macroRecall": 0.5535, + "minimumPrecision": 0.7143, + "perIntent": { + "blessing": { + "f1": 0.4615, + "falseNegative": 7, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 3, + "recall": 0.3, + "total": 126, + "trueNegative": 116, + "truePositive": 3, + "wilsonPrecisionLower95": 0.4385 + }, + "complaint": { + "f1": 0.6667, + "falseNegative": 3, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 3, + "recall": 0.5, + "total": 126, + "trueNegative": 120, + "truePositive": 3, + "wilsonPrecisionLower95": 0.4385 + }, + "confirmationDecision": { + "f1": 0.5882, + "falseNegative": 5, + "falsePositive": 2, + "precision": 0.7143, + "predictedPositive": 7, + "recall": 0.5, + "total": 126, + "trueNegative": 114, + "truePositive": 5, + "wilsonPrecisionLower95": 0.3589 + }, + "followUpReminder": { + "f1": 0.8182, + "falseNegative": 4, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 9, + "recall": 0.6923, + "total": 126, + "trueNegative": 113, + "truePositive": 9, + "wilsonPrecisionLower95": 0.7009 + }, + "invitation": { + "f1": 0.5455, + "falseNegative": 5, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 3, + "recall": 0.375, + "total": 126, + "trueNegative": 118, + "truePositive": 3, + "wilsonPrecisionLower95": 0.4385 + }, + "question": { + "f1": 0.9787, + "falseNegative": 0, + "falsePositive": 1, + "precision": 0.9583, + "predictedPositive": 24, + "recall": 1.0, + "total": 126, + "trueNegative": 102, + "truePositive": 23, + "wilsonPrecisionLower95": 0.7976 + }, + "replyableMessage": { + "f1": 0.9143, + "falseNegative": 6, + "falsePositive": 6, + "precision": 0.9143, + "predictedPositive": 70, + "recall": 0.9143, + "total": 126, + "trueNegative": 50, + "truePositive": 64, + "wilsonPrecisionLower95": 0.8253 + }, + "scheduleNegotiation": { + "f1": 0.4615, + "falseNegative": 7, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 3, + "recall": 0.3, + "total": 126, + "trueNegative": 116, + "truePositive": 3, + "wilsonPrecisionLower95": 0.4385 + }, + "task": { + "f1": 0.5714, + "falseNegative": 9, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 6, + "recall": 0.4, + "total": 126, + "trueNegative": 111, + "truePositive": 6, + "wilsonPrecisionLower95": 0.6097 + } + }, + "recordCount": 126 + } + }, + "bySource": { + "acknowledgment": { + "intentsMeeting95Precision": 0, + "macroF1": 0.0, + "macroPrecision": 0.0, + "macroRecall": 0.0, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 1, + "precision": 0.0, + "predictedPositive": 1, + "recall": 0.0, + "total": 8, + "trueNegative": 7, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 8 + }, + "blessing_boundary": { + "intentsMeeting95Precision": 0, + "macroF1": 0.1482, + "macroPrecision": 0.1667, + "macroRecall": 0.1667, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.6667, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 1, + "recall": 0.5, + "total": 20, + "trueNegative": 18, + "truePositive": 1, + "wilsonPrecisionLower95": 0.2065 + }, + "replyableMessage": { + "f1": 0.6667, + "falseNegative": 0, + "falsePositive": 4, + "precision": 0.5, + "predictedPositive": 8, + "recall": 1.0, + "total": 20, + "trueNegative": 12, + "truePositive": 4, + "wilsonPrecisionLower95": 0.2152 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 2, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 18, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 20 + }, + "blessing_golden": { + "intentsMeeting95Precision": 2, + "macroF1": 0.1823, + "macroPrecision": 0.2222, + "macroRecall": 0.1611, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.6667, + "falseNegative": 10, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 10, + "recall": 0.5, + "total": 20, + "trueNegative": 0, + "truePositive": 10, + "wilsonPrecisionLower95": 0.7225 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.9744, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 19, + "recall": 0.95, + "total": 20, + "trueNegative": 0, + "truePositive": 19, + "wilsonPrecisionLower95": 0.8318 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 20 + }, + "complaint_request": { + "intentsMeeting95Precision": 0, + "macroF1": 0.2703, + "macroPrecision": 0.3333, + "macroRecall": 0.2315, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.6667, + "falseNegative": 3, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 3, + "recall": 0.5, + "total": 6, + "trueNegative": 0, + "truePositive": 3, + "wilsonPrecisionLower95": 0.4385 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.8571, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 3, + "recall": 0.75, + "total": 6, + "trueNegative": 2, + "truePositive": 3, + "wilsonPrecisionLower95": 0.4385 + }, + "replyableMessage": { + "f1": 0.9091, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 5, + "recall": 0.8333, + "total": 6, + "trueNegative": 0, + "truePositive": 5, + "wilsonPrecisionLower95": 0.5655 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 6 + }, + "complaint_statement": { + "intentsMeeting95Precision": 0, + "macroF1": 0.2222, + "macroPrecision": 0.2222, + "macroRecall": 0.2222, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 6, + "recall": 1.0, + "total": 6, + "trueNegative": 0, + "truePositive": 6, + "wilsonPrecisionLower95": 0.6097 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 6, + "recall": 1.0, + "total": 6, + "trueNegative": 0, + "truePositive": 6, + "wilsonPrecisionLower95": 0.6097 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 6 + }, + "confirmation_decision_boundary": { + "intentsMeeting95Precision": 0, + "macroF1": 0.0, + "macroPrecision": 0.0, + "macroRecall": 0.0, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 3, + "precision": 0.0, + "predictedPositive": 3, + "recall": 0.0, + "total": 20, + "trueNegative": 17, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 3, + "precision": 0.0, + "predictedPositive": 3, + "recall": 0.0, + "total": 20, + "trueNegative": 17, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 20 + }, + "confirmation_decision_golden": { + "intentsMeeting95Precision": 2, + "macroF1": 0.2063, + "macroPrecision": 0.2222, + "macroRecall": 0.1944, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.8571, + "falseNegative": 5, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 15, + "recall": 0.75, + "total": 20, + "trueNegative": 0, + "truePositive": 15, + "wilsonPrecisionLower95": 0.7961 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 20, + "recall": 1.0, + "total": 20, + "trueNegative": 0, + "truePositive": 20, + "wilsonPrecisionLower95": 0.8389 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 20 + }, + "conversational_message": { + "intentsMeeting95Precision": 1, + "macroF1": 0.1111, + "macroPrecision": 0.1111, + "macroRecall": 0.1111, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 10, + "trueNegative": 10, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 10, + "trueNegative": 10, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 10, + "trueNegative": 10, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 10, + "trueNegative": 10, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 10, + "trueNegative": 10, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 10, + "trueNegative": 10, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 10, + "recall": 1.0, + "total": 10, + "trueNegative": 0, + "truePositive": 10, + "wilsonPrecisionLower95": 0.7225 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 10, + "trueNegative": 10, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 10, + "trueNegative": 10, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 10 + }, + "event_statement": { + "intentsMeeting95Precision": 0, + "macroF1": 0.0, + "macroPrecision": 0.0, + "macroRecall": 0.0, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 4 + }, + "follow_up_reminder_boundary": { + "intentsMeeting95Precision": 0, + "macroF1": 0.0, + "macroPrecision": 0.0, + "macroRecall": 0.0, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 3, + "precision": 0.0, + "predictedPositive": 3, + "recall": 0.0, + "total": 20, + "trueNegative": 17, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 20 + }, + "follow_up_reminder_golden": { + "intentsMeeting95Precision": 1, + "macroF1": 0.2187, + "macroPrecision": 0.3333, + "macroRecall": 0.1706, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.8571, + "falseNegative": 5, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 15, + "recall": 0.75, + "total": 20, + "trueNegative": 0, + "truePositive": 15, + "wilsonPrecisionLower95": 0.7961 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.6667, + "falseNegative": 7, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 7, + "recall": 0.5, + "total": 20, + "trueNegative": 6, + "truePositive": 7, + "wilsonPrecisionLower95": 0.6457 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.4444, + "falseNegative": 10, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 4, + "recall": 0.2857, + "total": 20, + "trueNegative": 6, + "truePositive": 4, + "wilsonPrecisionLower95": 0.5101 + } + }, + "recordCount": 20 + }, + "information_question": { + "intentsMeeting95Precision": 0, + "macroF1": 0.1693, + "macroPrecision": 0.2222, + "macroRecall": 0.1389, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.6667, + "falseNegative": 2, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 2, + "recall": 0.5, + "total": 4, + "trueNegative": 0, + "truePositive": 2, + "wilsonPrecisionLower95": 0.3424 + }, + "replyableMessage": { + "f1": 0.8571, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 3, + "recall": 0.75, + "total": 4, + "trueNegative": 0, + "truePositive": 3, + "wilsonPrecisionLower95": 0.4385 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 4 + }, + "invitation_question": { + "intentsMeeting95Precision": 0, + "macroF1": 0.3175, + "macroPrecision": 0.3333, + "macroRecall": 0.3056, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.8571, + "falseNegative": 2, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 6, + "recall": 0.75, + "total": 8, + "trueNegative": 0, + "truePositive": 6, + "wilsonPrecisionLower95": 0.6097 + }, + "question": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 8, + "recall": 1.0, + "total": 8, + "trueNegative": 0, + "truePositive": 8, + "wilsonPrecisionLower95": 0.6756 + }, + "replyableMessage": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 8, + "recall": 1.0, + "total": 8, + "trueNegative": 0, + "truePositive": 8, + "wilsonPrecisionLower95": 0.6756 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 8 + }, + "negative_news": { + "intentsMeeting95Precision": 0, + "macroF1": 0.0, + "macroPrecision": 0.0, + "macroRecall": 0.0, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 1, + "precision": 0.0, + "predictedPositive": 1, + "recall": 0.0, + "total": 6, + "trueNegative": 5, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 6 + }, + "neutral_fact": { + "intentsMeeting95Precision": 0, + "macroF1": 0.0, + "macroPrecision": 0.0, + "macroRecall": 0.0, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 10, + "trueNegative": 10, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 10, + "trueNegative": 10, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 10, + "trueNegative": 10, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 10, + "trueNegative": 10, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 10, + "trueNegative": 10, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 10, + "trueNegative": 10, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 2, + "precision": 0.0, + "predictedPositive": 2, + "recall": 0.0, + "total": 10, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 1, + "precision": 0.0, + "predictedPositive": 1, + "recall": 0.0, + "total": 10, + "trueNegative": 9, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 10, + "trueNegative": 10, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 10 + }, + "positive_feedback": { + "intentsMeeting95Precision": 0, + "macroF1": 0.1111, + "macroPrecision": 0.1111, + "macroRecall": 0.1111, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 6, + "recall": 1.0, + "total": 6, + "trueNegative": 0, + "truePositive": 6, + "wilsonPrecisionLower95": 0.6097 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 6 + }, + "quoted_question": { + "intentsMeeting95Precision": 0, + "macroF1": 0.0, + "macroPrecision": 0.0, + "macroRecall": 0.0, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 4, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 4 + }, + "schedule_negotiation_boundary": { + "intentsMeeting95Precision": 0, + "macroF1": 0.2541, + "macroPrecision": 0.284, + "macroRecall": 0.2639, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 1, + "precision": 0.0, + "predictedPositive": 1, + "recall": 0.0, + "total": 20, + "trueNegative": 19, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.5455, + "falseNegative": 5, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 3, + "recall": 0.375, + "total": 20, + "trueNegative": 12, + "truePositive": 3, + "wilsonPrecisionLower95": 0.4385 + }, + "question": { + "f1": 0.8, + "falseNegative": 0, + "falsePositive": 2, + "precision": 0.6667, + "predictedPositive": 6, + "recall": 1.0, + "total": 20, + "trueNegative": 14, + "truePositive": 4, + "wilsonPrecisionLower95": 0.3 + }, + "replyableMessage": { + "f1": 0.9412, + "falseNegative": 0, + "falsePositive": 1, + "precision": 0.8889, + "predictedPositive": 9, + "recall": 1.0, + "total": 20, + "trueNegative": 11, + "truePositive": 8, + "wilsonPrecisionLower95": 0.565 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 1, + "precision": 0.0, + "predictedPositive": 1, + "recall": 0.0, + "total": 20, + "trueNegative": 19, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 20 + }, + "schedule_negotiation_golden": { + "intentsMeeting95Precision": 2, + "macroF1": 0.2997, + "macroPrecision": 0.3275, + "macroRecall": 0.2833, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.973, + "falseNegative": 0, + "falsePositive": 1, + "precision": 0.9474, + "predictedPositive": 19, + "recall": 1.0, + "total": 20, + "trueNegative": 1, + "truePositive": 18, + "wilsonPrecisionLower95": 0.7536 + }, + "replyableMessage": { + "f1": 0.9744, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 19, + "recall": 0.95, + "total": 20, + "trueNegative": 0, + "truePositive": 19, + "wilsonPrecisionLower95": 0.8318 + }, + "scheduleNegotiation": { + "f1": 0.75, + "falseNegative": 8, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 12, + "recall": 0.6, + "total": 20, + "trueNegative": 0, + "truePositive": 12, + "wilsonPrecisionLower95": 0.7575 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 20, + "trueNegative": 20, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 20 + }, + "self_plan": { + "intentsMeeting95Precision": 0, + "macroF1": 0.1111, + "macroPrecision": 0.1111, + "macroRecall": 0.1111, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 2, + "recall": 1.0, + "total": 6, + "trueNegative": 4, + "truePositive": 2, + "wilsonPrecisionLower95": 0.3424 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 2, + "precision": 0.0, + "predictedPositive": 2, + "recall": 0.0, + "total": 6, + "trueNegative": 4, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 6 + }, + "task_question": { + "intentsMeeting95Precision": 0, + "macroF1": 0.2963, + "macroPrecision": 0.3333, + "macroRecall": 0.2778, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 6, + "recall": 1.0, + "total": 6, + "trueNegative": 0, + "truePositive": 6, + "wilsonPrecisionLower95": 0.6097 + }, + "replyableMessage": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 6, + "recall": 1.0, + "total": 6, + "trueNegative": 0, + "truePositive": 6, + "wilsonPrecisionLower95": 0.6097 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 6, + "trueNegative": 6, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.6667, + "falseNegative": 3, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 3, + "recall": 0.5, + "total": 6, + "trueNegative": 0, + "truePositive": 3, + "wilsonPrecisionLower95": 0.4385 + } + }, + "recordCount": 6 + }, + "task_statement": { + "intentsMeeting95Precision": 0, + "macroF1": 0.2632, + "macroPrecision": 0.3333, + "macroRecall": 0.2222, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.6667, + "falseNegative": 2, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 2, + "recall": 0.5, + "total": 8, + "trueNegative": 4, + "truePositive": 2, + "wilsonPrecisionLower95": 0.3424 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.9333, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 7, + "recall": 0.875, + "total": 8, + "trueNegative": 0, + "truePositive": 7, + "wilsonPrecisionLower95": 0.6457 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 8, + "trueNegative": 8, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.7692, + "falseNegative": 3, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 5, + "recall": 0.625, + "total": 8, + "trueNegative": 0, + "truePositive": 5, + "wilsonPrecisionLower95": 0.5655 + } + }, + "recordCount": 8 + } + }, + "intentsMeeting95Precision": 3, + "macroF1": 0.7711, + "macroPrecision": 0.9366, + "macroRecall": 0.6801, + "minimumPrecision": 0.75, + "perIntent": { + "blessing": { + "f1": 0.6667, + "falseNegative": 10, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 10, + "recall": 0.5, + "total": 252, + "trueNegative": 232, + "truePositive": 10, + "wilsonPrecisionLower95": 0.7225 + }, + "complaint": { + "f1": 0.8571, + "falseNegative": 3, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 9, + "recall": 0.75, + "total": 252, + "trueNegative": 240, + "truePositive": 9, + "wilsonPrecisionLower95": 0.7009 + }, + "confirmationDecision": { + "f1": 0.75, + "falseNegative": 5, + "falsePositive": 5, + "precision": 0.75, + "predictedPositive": 20, + "recall": 0.75, + "total": 252, + "trueNegative": 227, + "truePositive": 15, + "wilsonPrecisionLower95": 0.5313 + }, + "followUpReminder": { + "f1": 0.8444, + "falseNegative": 7, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 19, + "recall": 0.7308, + "total": 252, + "trueNegative": 226, + "truePositive": 19, + "wilsonPrecisionLower95": 0.8318 + }, + "invitation": { + "f1": 0.72, + "falseNegative": 7, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 9, + "recall": 0.5625, + "total": 252, + "trueNegative": 236, + "truePositive": 9, + "wilsonPrecisionLower95": 0.7009 + }, + "question": { + "f1": 0.9231, + "falseNegative": 4, + "falsePositive": 3, + "precision": 0.9333, + "predictedPositive": 45, + "recall": 0.913, + "total": 252, + "trueNegative": 203, + "truePositive": 42, + "wilsonPrecisionLower95": 0.8214 + }, + "replyableMessage": { + "f1": 0.9014, + "falseNegative": 12, + "falsePositive": 16, + "precision": 0.8889, + "predictedPositive": 144, + "recall": 0.9143, + "total": 252, + "trueNegative": 96, + "truePositive": 128, + "wilsonPrecisionLower95": 0.8271 + }, + "scheduleNegotiation": { + "f1": 0.7059, + "falseNegative": 8, + "falsePositive": 2, + "precision": 0.8571, + "predictedPositive": 14, + "recall": 0.6, + "total": 252, + "trueNegative": 230, + "truePositive": 12, + "wilsonPrecisionLower95": 0.6006 + }, + "task": { + "f1": 0.5714, + "falseNegative": 18, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 12, + "recall": 0.4, + "total": 252, + "trueNegative": 222, + "truePositive": 12, + "wilsonPrecisionLower95": 0.7575 + } + }, + "recordCount": 252 + }, + "randomHoldout": { + "byLanguage": { + "en": { + "intentsMeeting95Precision": 4, + "macroF1": 0.6163, + "macroPrecision": 0.8521, + "macroRecall": 0.5281, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 340, + "trueNegative": 340, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.3333, + "falseNegative": 16, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 4, + "recall": 0.2, + "total": 340, + "trueNegative": 320, + "truePositive": 4, + "wilsonPrecisionLower95": 0.5101 + }, + "confirmationDecision": { + "f1": 0.9302, + "falseNegative": 0, + "falsePositive": 3, + "precision": 0.8696, + "predictedPositive": 23, + "recall": 1.0, + "total": 340, + "trueNegative": 317, + "truePositive": 20, + "wilsonPrecisionLower95": 0.6787 + }, + "followUpReminder": { + "f1": 0.9027, + "falseNegative": 9, + "falsePositive": 2, + "precision": 0.9623, + "predictedPositive": 53, + "recall": 0.85, + "total": 340, + "trueNegative": 278, + "truePositive": 51, + "wilsonPrecisionLower95": 0.8725 + }, + "invitation": { + "f1": 0.5714, + "falseNegative": 12, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 8, + "recall": 0.4, + "total": 340, + "trueNegative": 320, + "truePositive": 8, + "wilsonPrecisionLower95": 0.6756 + }, + "question": { + "f1": 0.6087, + "falseNegative": 45, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 35, + "recall": 0.4375, + "total": 340, + "trueNegative": 260, + "truePositive": 35, + "wilsonPrecisionLower95": 0.9011 + }, + "replyableMessage": { + "f1": 0.8859, + "falseNegative": 37, + "falsePositive": 5, + "precision": 0.9702, + "predictedPositive": 168, + "recall": 0.815, + "total": 340, + "trueNegative": 135, + "truePositive": 163, + "wilsonPrecisionLower95": 0.9322 + }, + "scheduleNegotiation": { + "f1": 0.7429, + "falseNegative": 7, + "falsePositive": 2, + "precision": 0.8667, + "predictedPositive": 15, + "recall": 0.65, + "total": 340, + "trueNegative": 318, + "truePositive": 13, + "wilsonPrecisionLower95": 0.6212 + }, + "task": { + "f1": 0.5714, + "falseNegative": 36, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 24, + "recall": 0.4, + "total": 340, + "trueNegative": 280, + "truePositive": 24, + "wilsonPrecisionLower95": 0.862 + } + }, + "recordCount": 340 + }, + "zh-Hans": { + "intentsMeeting95Precision": 4, + "macroF1": 0.4934, + "macroPrecision": 0.7719, + "macroRecall": 0.3925, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 340, + "trueNegative": 340, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.3333, + "falseNegative": 16, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 4, + "recall": 0.2, + "total": 340, + "trueNegative": 320, + "truePositive": 4, + "wilsonPrecisionLower95": 0.5101 + }, + "confirmationDecision": { + "f1": 0.6667, + "falseNegative": 10, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 10, + "recall": 0.5, + "total": 340, + "trueNegative": 320, + "truePositive": 10, + "wilsonPrecisionLower95": 0.7225 + }, + "followUpReminder": { + "f1": 0.7879, + "falseNegative": 21, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 39, + "recall": 0.65, + "total": 340, + "trueNegative": 280, + "truePositive": 39, + "wilsonPrecisionLower95": 0.9103 + }, + "invitation": { + "f1": 0.6207, + "falseNegative": 11, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 9, + "recall": 0.45, + "total": 340, + "trueNegative": 320, + "truePositive": 9, + "wilsonPrecisionLower95": 0.7009 + }, + "question": { + "f1": 0.7786, + "falseNegative": 29, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 51, + "recall": 0.6375, + "total": 340, + "trueNegative": 260, + "truePositive": 51, + "wilsonPrecisionLower95": 0.93 + }, + "replyableMessage": { + "f1": 0.9203, + "falseNegative": 21, + "falsePositive": 10, + "precision": 0.9471, + "predictedPositive": 189, + "recall": 0.895, + "total": 340, + "trueNegative": 130, + "truePositive": 179, + "wilsonPrecisionLower95": 0.9054 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 20, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 340, + "trueNegative": 320, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.3333, + "falseNegative": 48, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 12, + "recall": 0.2, + "total": 340, + "trueNegative": 280, + "truePositive": 12, + "wilsonPrecisionLower95": 0.7575 + } + }, + "recordCount": 340 + } + }, + "bySource": { + "acknowledgment_boundary": { + "intentsMeeting95Precision": 0, + "macroF1": 0.0, + "macroPrecision": 0.0, + "macroRecall": 0.0, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 40 + }, + "complaint_support": { + "intentsMeeting95Precision": 1, + "macroF1": 0.1783, + "macroPrecision": 0.3333, + "macroRecall": 0.1389, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.3333, + "falseNegative": 32, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 8, + "recall": 0.2, + "total": 40, + "trueNegative": 0, + "truePositive": 8, + "wilsonPrecisionLower95": 0.6756 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.3673, + "falseNegative": 31, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 9, + "recall": 0.225, + "total": 40, + "trueNegative": 0, + "truePositive": 9, + "wilsonPrecisionLower95": 0.7009 + }, + "replyableMessage": { + "f1": 0.9041, + "falseNegative": 7, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 33, + "recall": 0.825, + "total": 40, + "trueNegative": 0, + "truePositive": 33, + "wilsonPrecisionLower95": 0.8957 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 40 + }, + "confirmation_decision": { + "intentsMeeting95Precision": 2, + "macroF1": 0.1957, + "macroPrecision": 0.2222, + "macroRecall": 0.175, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.8571, + "falseNegative": 10, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 30, + "recall": 0.75, + "total": 40, + "trueNegative": 0, + "truePositive": 30, + "wilsonPrecisionLower95": 0.8865 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.9041, + "falseNegative": 7, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 33, + "recall": 0.825, + "total": 40, + "trueNegative": 0, + "truePositive": 33, + "wilsonPrecisionLower95": 0.8957 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 40 + }, + "conversation": { + "intentsMeeting95Precision": 1, + "macroF1": 0.1053, + "macroPrecision": 0.1111, + "macroRecall": 0.1, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.9474, + "falseNegative": 4, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 36, + "recall": 0.9, + "total": 40, + "trueNegative": 0, + "truePositive": 36, + "wilsonPrecisionLower95": 0.9036 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 40 + }, + "event_boundary": { + "intentsMeeting95Precision": 0, + "macroF1": 0.0, + "macroPrecision": 0.0, + "macroRecall": 0.0, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 2, + "precision": 0.0, + "predictedPositive": 2, + "recall": 0.0, + "total": 40, + "trueNegative": 38, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 40 + }, + "fixed_invitation": { + "intentsMeeting95Precision": 3, + "macroF1": 0.2797, + "macroPrecision": 0.3333, + "macroRecall": 0.2528, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.5965, + "falseNegative": 23, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 17, + "recall": 0.425, + "total": 40, + "trueNegative": 0, + "truePositive": 17, + "wilsonPrecisionLower95": 0.8157 + }, + "question": { + "f1": 0.9333, + "falseNegative": 5, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 35, + "recall": 0.875, + "total": 40, + "trueNegative": 0, + "truePositive": 35, + "wilsonPrecisionLower95": 0.9011 + }, + "replyableMessage": { + "f1": 0.9873, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 39, + "recall": 0.975, + "total": 40, + "trueNegative": 0, + "truePositive": 39, + "wilsonPrecisionLower95": 0.9103 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 40 + }, + "follow_up_instruction": { + "intentsMeeting95Precision": 2, + "macroF1": 0.2153, + "macroPrecision": 0.3333, + "macroRecall": 0.1694, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.7302, + "falseNegative": 17, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 23, + "recall": 0.575, + "total": 40, + "trueNegative": 0, + "truePositive": 23, + "wilsonPrecisionLower95": 0.8569 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.8406, + "falseNegative": 11, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 29, + "recall": 0.725, + "total": 40, + "trueNegative": 0, + "truePositive": 29, + "wilsonPrecisionLower95": 0.883 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.3673, + "falseNegative": 31, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 9, + "recall": 0.225, + "total": 40, + "trueNegative": 0, + "truePositive": 9, + "wilsonPrecisionLower95": 0.7009 + } + }, + "recordCount": 40 + }, + "follow_up_personal_reminder": { + "intentsMeeting95Precision": 1, + "macroF1": 0.097, + "macroPrecision": 0.1111, + "macroRecall": 0.0861, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.8732, + "falseNegative": 9, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 31, + "recall": 0.775, + "total": 40, + "trueNegative": 0, + "truePositive": 31, + "wilsonPrecisionLower95": 0.8897 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 40 + }, + "information_question": { + "intentsMeeting95Precision": 2, + "macroF1": 0.1421, + "macroPrecision": 0.2222, + "macroRecall": 0.1083, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 3, + "precision": 0.0, + "predictedPositive": 3, + "recall": 0.0, + "total": 40, + "trueNegative": 37, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 2, + "precision": 0.0, + "predictedPositive": 2, + "recall": 0.0, + "total": 40, + "trueNegative": 38, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.4906, + "falseNegative": 27, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 13, + "recall": 0.325, + "total": 40, + "trueNegative": 0, + "truePositive": 13, + "wilsonPrecisionLower95": 0.7719 + }, + "replyableMessage": { + "f1": 0.7879, + "falseNegative": 14, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 26, + "recall": 0.65, + "total": 40, + "trueNegative": 0, + "truePositive": 26, + "wilsonPrecisionLower95": 0.8713 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 40 + }, + "negative_news": { + "intentsMeeting95Precision": 0, + "macroF1": 0.0, + "macroPrecision": 0.0, + "macroRecall": 0.0, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 3, + "precision": 0.0, + "predictedPositive": 3, + "recall": 0.0, + "total": 40, + "trueNegative": 37, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 2, + "precision": 0.0, + "predictedPositive": 2, + "recall": 0.0, + "total": 40, + "trueNegative": 38, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 40 + }, + "neutral_fact": { + "intentsMeeting95Precision": 0, + "macroF1": 0.0, + "macroPrecision": 0.0, + "macroRecall": 0.0, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 5, + "precision": 0.0, + "predictedPositive": 5, + "recall": 0.0, + "total": 40, + "trueNegative": 35, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 40 + }, + "positive_feedback": { + "intentsMeeting95Precision": 1, + "macroF1": 0.1068, + "macroPrecision": 0.1111, + "macroRecall": 0.1028, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.961, + "falseNegative": 3, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 37, + "recall": 0.925, + "total": 40, + "trueNegative": 0, + "truePositive": 37, + "wilsonPrecisionLower95": 0.9059 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 40 + }, + "quoted_question_boundary": { + "intentsMeeting95Precision": 0, + "macroF1": 0.0, + "macroPrecision": 0.0, + "macroRecall": 0.0, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 1, + "precision": 0.0, + "predictedPositive": 1, + "recall": 0.0, + "total": 40, + "trueNegative": 39, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 40 + }, + "schedule_negotiation": { + "intentsMeeting95Precision": 3, + "macroF1": 0.259, + "macroPrecision": 0.3333, + "macroRecall": 0.2278, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.8406, + "falseNegative": 11, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 29, + "recall": 0.725, + "total": 40, + "trueNegative": 0, + "truePositive": 29, + "wilsonPrecisionLower95": 0.883 + }, + "replyableMessage": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 40, + "recall": 1.0, + "total": 40, + "trueNegative": 0, + "truePositive": 40, + "wilsonPrecisionLower95": 0.9124 + }, + "scheduleNegotiation": { + "f1": 0.4906, + "falseNegative": 27, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 13, + "recall": 0.325, + "total": 40, + "trueNegative": 0, + "truePositive": 13, + "wilsonPrecisionLower95": 0.7719 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 40 + }, + "task_next_action": { + "intentsMeeting95Precision": 3, + "macroF1": 0.257, + "macroPrecision": 0.3333, + "macroRecall": 0.225, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.9474, + "falseNegative": 4, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 36, + "recall": 0.9, + "total": 40, + "trueNegative": 0, + "truePositive": 36, + "wilsonPrecisionLower95": 0.9036 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.9041, + "falseNegative": 7, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 33, + "recall": 0.825, + "total": 40, + "trueNegative": 0, + "truePositive": 33, + "wilsonPrecisionLower95": 0.8957 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.4615, + "falseNegative": 28, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 12, + "recall": 0.3, + "total": 40, + "trueNegative": 0, + "truePositive": 12, + "wilsonPrecisionLower95": 0.7575 + } + }, + "recordCount": 40 + }, + "task_request": { + "intentsMeeting95Precision": 2, + "macroF1": 0.1659, + "macroPrecision": 0.2222, + "macroRecall": 0.1417, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.9474, + "falseNegative": 4, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 36, + "recall": 0.9, + "total": 40, + "trueNegative": 0, + "truePositive": 36, + "wilsonPrecisionLower95": 0.9036 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.5455, + "falseNegative": 25, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 15, + "recall": 0.375, + "total": 40, + "trueNegative": 0, + "truePositive": 15, + "wilsonPrecisionLower95": 0.7961 + } + }, + "recordCount": 40 + }, + "vague_future_boundary": { + "intentsMeeting95Precision": 0, + "macroF1": 0.0, + "macroPrecision": 0.0, + "macroRecall": 0.0, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 4, + "precision": 0.0, + "predictedPositive": 4, + "recall": 0.0, + "total": 40, + "trueNegative": 36, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 40, + "trueNegative": 40, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 40 + } + }, + "intentsMeeting95Precision": 5, + "macroF1": 0.5709, + "macroPrecision": 0.8569, + "macroRecall": 0.4603, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 680, + "trueNegative": 680, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.3333, + "falseNegative": 32, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 8, + "recall": 0.2, + "total": 680, + "trueNegative": 640, + "truePositive": 8, + "wilsonPrecisionLower95": 0.6756 + }, + "confirmationDecision": { + "f1": 0.8219, + "falseNegative": 10, + "falsePositive": 3, + "precision": 0.9091, + "predictedPositive": 33, + "recall": 0.75, + "total": 680, + "trueNegative": 637, + "truePositive": 30, + "wilsonPrecisionLower95": 0.7643 + }, + "followUpReminder": { + "f1": 0.8491, + "falseNegative": 30, + "falsePositive": 2, + "precision": 0.9783, + "predictedPositive": 92, + "recall": 0.75, + "total": 680, + "trueNegative": 558, + "truePositive": 90, + "wilsonPrecisionLower95": 0.9242 + }, + "invitation": { + "f1": 0.5965, + "falseNegative": 23, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 17, + "recall": 0.425, + "total": 680, + "trueNegative": 640, + "truePositive": 17, + "wilsonPrecisionLower95": 0.8157 + }, + "question": { + "f1": 0.6992, + "falseNegative": 74, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 86, + "recall": 0.5375, + "total": 680, + "trueNegative": 520, + "truePositive": 86, + "wilsonPrecisionLower95": 0.9572 + }, + "replyableMessage": { + "f1": 0.9036, + "falseNegative": 58, + "falsePositive": 15, + "precision": 0.958, + "predictedPositive": 357, + "recall": 0.855, + "total": 680, + "trueNegative": 265, + "truePositive": 342, + "wilsonPrecisionLower95": 0.9318 + }, + "scheduleNegotiation": { + "f1": 0.4727, + "falseNegative": 27, + "falsePositive": 2, + "precision": 0.8667, + "predictedPositive": 15, + "recall": 0.325, + "total": 680, + "trueNegative": 638, + "truePositive": 13, + "wilsonPrecisionLower95": 0.6212 + }, + "task": { + "f1": 0.4615, + "falseNegative": 84, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 36, + "recall": 0.3, + "total": 680, + "trueNegative": 560, + "truePositive": 36, + "wilsonPrecisionLower95": 0.9036 + } + }, + "recordCount": 680 + }, + "researchOnlyComprehensive": { + "byLanguage": { + "en": { + "intentsMeeting95Precision": 0, + "macroF1": 0.2616, + "macroPrecision": 0.3966, + "macroRecall": 0.2659, + "minimumPrecision": 0.0012, + "perIntent": { + "blessing": { + "f1": 0.2308, + "falseNegative": 60, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 9, + "recall": 0.1304, + "total": 13176, + "trueNegative": 13107, + "truePositive": 9, + "wilsonPrecisionLower95": 0.7009 + }, + "complaint": { + "f1": 0.048, + "falseNegative": 421, + "falsePositive": 95, + "precision": 0.1204, + "predictedPositive": 108, + "recall": 0.03, + "total": 13176, + "trueNegative": 12647, + "truePositive": 13, + "wilsonPrecisionLower95": 0.0717 + }, + "confirmationDecision": { + "f1": 0.0024, + "falseNegative": 10, + "falsePositive": 1668, + "precision": 0.0012, + "predictedPositive": 1670, + "recall": 0.1667, + "total": 13176, + "trueNegative": 11496, + "truePositive": 2, + "wilsonPrecisionLower95": 0.0003 + }, + "followUpReminder": { + "f1": 0.1134, + "falseNegative": 114, + "falsePositive": 58, + "precision": 0.1594, + "predictedPositive": 69, + "recall": 0.088, + "total": 13176, + "trueNegative": 12993, + "truePositive": 11, + "wilsonPrecisionLower95": 0.0914 + }, + "invitation": { + "f1": 0.336, + "falseNegative": 40, + "falsePositive": 43, + "precision": 0.3281, + "predictedPositive": 64, + "recall": 0.3443, + "total": 13176, + "trueNegative": 13072, + "truePositive": 21, + "wilsonPrecisionLower95": 0.2257 + }, + "question": { + "f1": 0.4288, + "falseNegative": 1828, + "falsePositive": 378, + "precision": 0.6866, + "predictedPositive": 1206, + "recall": 0.3117, + "total": 13176, + "trueNegative": 10142, + "truePositive": 828, + "wilsonPrecisionLower95": 0.6598 + }, + "replyableMessage": { + "f1": 0.7642, + "falseNegative": 2967, + "falsePositive": 1917, + "precision": 0.805, + "predictedPositive": 9831, + "recall": 0.7273, + "total": 13176, + "trueNegative": 378, + "truePositive": 7914, + "wilsonPrecisionLower95": 0.7971 + }, + "scheduleNegotiation": { + "f1": 0.1683, + "falseNegative": 41, + "falsePositive": 216, + "precision": 0.1074, + "predictedPositive": 242, + "recall": 0.3881, + "total": 13176, + "trueNegative": 12893, + "truePositive": 26, + "wilsonPrecisionLower95": 0.0744 + }, + "task": { + "f1": 0.2628, + "falseNegative": 888, + "falsePositive": 408, + "precision": 0.3615, + "predictedPositive": 639, + "recall": 0.2064, + "total": 13176, + "trueNegative": 11649, + "truePositive": 231, + "wilsonPrecisionLower95": 0.3252 + } + }, + "recordCount": 13176 + }, + "zh-Hans": { + "intentsMeeting95Precision": 0, + "macroF1": 0.1774, + "macroPrecision": 0.3011, + "macroRecall": 0.149, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 17, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 28019, + "trueNegative": 28002, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.4077, + "falseNegative": 154, + "falsePositive": 183, + "precision": 0.388, + "predictedPositive": 299, + "recall": 0.4296, + "total": 28019, + "trueNegative": 27566, + "truePositive": 116, + "wilsonPrecisionLower95": 0.3345 + }, + "confirmationDecision": { + "f1": 0.0909, + "falseNegative": 53, + "falsePositive": 27, + "precision": 0.129, + "predictedPositive": 31, + "recall": 0.0702, + "total": 28019, + "trueNegative": 27935, + "truePositive": 4, + "wilsonPrecisionLower95": 0.0513 + }, + "followUpReminder": { + "f1": 0.1856, + "falseNegative": 234, + "falsePositive": 38, + "precision": 0.4493, + "predictedPositive": 69, + "recall": 0.117, + "total": 28019, + "trueNegative": 27716, + "truePositive": 31, + "wilsonPrecisionLower95": 0.3377 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 73, + "falsePositive": 9, + "precision": 0.0, + "predictedPositive": 9, + "recall": 0.0, + "total": 28019, + "trueNegative": 27937, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "question": { + "f1": 0.1419, + "falseNegative": 6619, + "falsePositive": 216, + "precision": 0.7234, + "predictedPositive": 781, + "recall": 0.0786, + "total": 28019, + "trueNegative": 20619, + "truePositive": 565, + "wilsonPrecisionLower95": 0.691 + }, + "replyableMessage": { + "f1": 0.7622, + "falseNegative": 9480, + "falsePositive": 1096, + "precision": 0.9393, + "predictedPositive": 18042, + "recall": 0.6413, + "total": 28019, + "trueNegative": 497, + "truePositive": 16946, + "wilsonPrecisionLower95": 0.9357 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 4, + "falsePositive": 2, + "precision": 0.0, + "predictedPositive": 2, + "recall": 0.0, + "total": 28019, + "trueNegative": 28013, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "task": { + "f1": 0.0086, + "falseNegative": 656, + "falsePositive": 34, + "precision": 0.0811, + "predictedPositive": 37, + "recall": 0.0046, + "total": 28019, + "trueNegative": 27326, + "truePositive": 3, + "wilsonPrecisionLower95": 0.028 + } + }, + "recordCount": 28019 + } + }, + "bySource": { + "AmazonScience/MASSIVE en-US test": { + "intentsMeeting95Precision": 2, + "macroF1": 0.1614, + "macroPrecision": 0.2661, + "macroRecall": 0.1233, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 2967, + "trueNegative": 2967, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 17, + "precision": 0.0, + "predictedPositive": 17, + "recall": 0.0, + "total": 2967, + "trueNegative": 2950, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 347, + "precision": 0.0, + "predictedPositive": 347, + "recall": 0.0, + "total": 2967, + "trueNegative": 2620, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "followUpReminder": { + "f1": 0.1447, + "falseNegative": 86, + "falsePositive": 44, + "precision": 0.2, + "predictedPositive": 55, + "recall": 0.1134, + "total": 2967, + "trueNegative": 2826, + "truePositive": 11, + "wilsonPrecisionLower95": 0.1155 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 7, + "precision": 0.0, + "predictedPositive": 7, + "recall": 0.0, + "total": 2967, + "trueNegative": 2960, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "question": { + "f1": 0.3532, + "falseNegative": 1266, + "falsePositive": 16, + "precision": 0.9563, + "predictedPositive": 366, + "recall": 0.2166, + "total": 2967, + "trueNegative": 1335, + "truePositive": 350, + "wilsonPrecisionLower95": 0.9302 + }, + "replyableMessage": { + "f1": 0.7951, + "falseNegative": 1009, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 1958, + "recall": 0.6599, + "total": 2967, + "trueNegative": 0, + "truePositive": 1958, + "wilsonPrecisionLower95": 0.998 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 101, + "precision": 0.0, + "predictedPositive": 101, + "recall": 0.0, + "total": 2967, + "trueNegative": 2866, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "task": { + "f1": 0.1599, + "falseNegative": 476, + "falsePositive": 207, + "precision": 0.239, + "predictedPositive": 272, + "recall": 0.1201, + "total": 2967, + "trueNegative": 2219, + "truePositive": 65, + "wilsonPrecisionLower95": 0.1921 + } + }, + "recordCount": 2967 + }, + "AmazonScience/MASSIVE zh-CN test": { + "intentsMeeting95Precision": 1, + "macroF1": 0.1368, + "macroPrecision": 0.2943, + "macroRecall": 0.0998, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 2850, + "trueNegative": 2850, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 2, + "precision": 0.0, + "predictedPositive": 2, + "recall": 0.0, + "total": 2850, + "trueNegative": 2848, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 4, + "precision": 0.0, + "predictedPositive": 4, + "recall": 0.0, + "total": 2850, + "trueNegative": 2846, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "followUpReminder": { + "f1": 0.2537, + "falseNegative": 140, + "falsePositive": 13, + "precision": 0.6667, + "predictedPositive": 39, + "recall": 0.1566, + "total": 2850, + "trueNegative": 2671, + "truePositive": 26, + "wilsonPrecisionLower95": 0.5098 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 7, + "precision": 0.0, + "predictedPositive": 7, + "recall": 0.0, + "total": 2850, + "trueNegative": 2843, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "question": { + "f1": 0.2054, + "falseNegative": 1381, + "falsePositive": 19, + "precision": 0.905, + "predictedPositive": 200, + "recall": 0.1159, + "total": 2850, + "trueNegative": 1269, + "truePositive": 181, + "wilsonPrecisionLower95": 0.8564 + }, + "replyableMessage": { + "f1": 0.7684, + "falseNegative": 1072, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 1778, + "recall": 0.6239, + "total": 2850, + "trueNegative": 0, + "truePositive": 1778, + "wilsonPrecisionLower95": 0.9978 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 2, + "precision": 0.0, + "predictedPositive": 2, + "recall": 0.0, + "total": 2850, + "trueNegative": 2848, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "task": { + "f1": 0.0041, + "falseNegative": 477, + "falsePositive": 12, + "precision": 0.0769, + "predictedPositive": 13, + "recall": 0.0021, + "total": 2850, + "trueNegative": 2360, + "truePositive": 1, + "wilsonPrecisionLower95": 0.0137 + } + }, + "recordCount": 2850 + }, + "CFPB via hpe-ai/customer-complaints": { + "intentsMeeting95Precision": 1, + "macroF1": 0.0837, + "macroPrecision": 0.2593, + "macroRecall": 0.063, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 250, + "trueNegative": 250, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0159, + "falseNegative": 248, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 2, + "recall": 0.008, + "total": 250, + "trueNegative": 0, + "truePositive": 2, + "wilsonPrecisionLower95": 0.3424 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 7, + "precision": 0.0, + "predictedPositive": 7, + "recall": 0.0, + "total": 250, + "trueNegative": 243, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 250, + "trueNegative": 250, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 250, + "trueNegative": 250, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 17, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 250, + "trueNegative": 233, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.7013, + "falseNegative": 115, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 135, + "recall": 0.54, + "total": 250, + "trueNegative": 0, + "truePositive": 135, + "wilsonPrecisionLower95": 0.9723 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 250, + "trueNegative": 250, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0357, + "falseNegative": 52, + "falsePositive": 2, + "precision": 0.3333, + "predictedPositive": 3, + "recall": 0.0189, + "total": 250, + "trueNegative": 195, + "truePositive": 1, + "wilsonPrecisionLower95": 0.0615 + } + }, + "recordCount": 250 + }, + "CPED test split": { + "intentsMeeting95Precision": 0, + "macroF1": 0.1219, + "macroPrecision": 0.2231, + "macroRecall": 0.096, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 17, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 25049, + "trueNegative": 25032, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.018, + "falseNegative": 147, + "falsePositive": 181, + "precision": 0.0163, + "predictedPositive": 184, + "recall": 0.02, + "total": 25049, + "trueNegative": 24718, + "truePositive": 3, + "wilsonPrecisionLower95": 0.0056 + }, + "confirmationDecision": { + "f1": 0.0952, + "falseNegative": 53, + "falsePositive": 23, + "precision": 0.1481, + "predictedPositive": 27, + "recall": 0.0702, + "total": 25049, + "trueNegative": 24969, + "truePositive": 4, + "wilsonPrecisionLower95": 0.0592 + }, + "followUpReminder": { + "f1": 0.0775, + "falseNegative": 94, + "falsePositive": 25, + "precision": 0.1667, + "predictedPositive": 30, + "recall": 0.0505, + "total": 25049, + "trueNegative": 24925, + "truePositive": 5, + "wilsonPrecisionLower95": 0.0734 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 73, + "falsePositive": 2, + "precision": 0.0, + "predictedPositive": 2, + "recall": 0.0, + "total": 25049, + "trueNegative": 24974, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "question": { + "f1": 0.1245, + "falseNegative": 5202, + "falsePositive": 197, + "precision": 0.6609, + "predictedPositive": 581, + "recall": 0.0687, + "total": 25049, + "trueNegative": 19266, + "truePositive": 384, + "wilsonPrecisionLower95": 0.6215 + }, + "replyableMessage": { + "f1": 0.7609, + "falseNegative": 8378, + "falsePositive": 1096, + "precision": 0.9322, + "predictedPositive": 16174, + "recall": 0.6428, + "total": 25049, + "trueNegative": 497, + "truePositive": 15078, + "wilsonPrecisionLower95": 0.9283 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 4, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 25049, + "trueNegative": 25045, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0208, + "falseNegative": 166, + "falsePositive": 22, + "precision": 0.0833, + "predictedPositive": 24, + "recall": 0.0119, + "total": 25049, + "trueNegative": 24859, + "truePositive": 2, + "wilsonPrecisionLower95": 0.0232 + } + }, + "recordCount": 25049 + }, + "Google Research GoEmotions test": { + "intentsMeeting95Precision": 1, + "macroF1": 0.1942, + "macroPrecision": 0.34, + "macroRecall": 0.1531, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.1918, + "falseNegative": 59, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 7, + "recall": 0.1061, + "total": 5418, + "trueNegative": 5352, + "truePositive": 7, + "wilsonPrecisionLower95": 0.6457 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 20, + "falsePositive": 27, + "precision": 0.0, + "predictedPositive": 27, + "recall": 0.0, + "total": 5418, + "trueNegative": 5371, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 881, + "precision": 0.0, + "predictedPositive": 881, + "recall": 0.0, + "total": 5418, + "trueNegative": 4537, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 6, + "precision": 0.0, + "predictedPositive": 6, + "recall": 0.0, + "total": 5418, + "trueNegative": 5412, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 8, + "precision": 0.0, + "predictedPositive": 8, + "recall": 0.0, + "total": 5418, + "trueNegative": 5410, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "question": { + "f1": 0.3991, + "falseNegative": 363, + "falsePositive": 146, + "precision": 0.5365, + "predictedPositive": 315, + "recall": 0.3177, + "total": 5418, + "trueNegative": 4740, + "truePositive": 169, + "wilsonPrecisionLower95": 0.4813 + }, + "replyableMessage": { + "f1": 0.8467, + "falseNegative": 1440, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 3978, + "recall": 0.7342, + "total": 5418, + "trueNegative": 0, + "truePositive": 3978, + "wilsonPrecisionLower95": 0.999 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 33, + "precision": 0.0, + "predictedPositive": 33, + "recall": 0.0, + "total": 5418, + "trueNegative": 5385, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "task": { + "f1": 0.3099, + "falseNegative": 39, + "falsePositive": 10, + "precision": 0.5238, + "predictedPositive": 21, + "recall": 0.22, + "total": 5418, + "trueNegative": 5358, + "truePositive": 11, + "wilsonPrecisionLower95": 0.3237 + } + }, + "recordCount": 5418 + }, + "Meituan-Dianping/ASAP test": { + "intentsMeeting95Precision": 2, + "macroF1": 0.203, + "macroPrecision": 0.2222, + "macroRecall": 0.188, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 120, + "trueNegative": 120, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.97, + "falseNegative": 7, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 113, + "recall": 0.9417, + "total": 120, + "trueNegative": 0, + "truePositive": 113, + "wilsonPrecisionLower95": 0.9671 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 120, + "trueNegative": 120, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 120, + "trueNegative": 120, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 120, + "trueNegative": 120, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 36, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 120, + "trueNegative": 84, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.8571, + "falseNegative": 30, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 90, + "recall": 0.75, + "total": 120, + "trueNegative": 0, + "truePositive": 90, + "wilsonPrecisionLower95": 0.9591 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 120, + "trueNegative": 120, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 13, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 120, + "trueNegative": 107, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 120 + }, + "Sulak2020 GitHub issues multi-repo dataset": { + "intentsMeeting95Precision": 1, + "macroF1": 0.28, + "macroPrecision": 0.415, + "macroRecall": 0.216, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.6667, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 1, + "recall": 0.5, + "total": 1004, + "trueNegative": 1002, + "truePositive": 1, + "wilsonPrecisionLower95": 0.2065 + }, + "complaint": { + "f1": 0.0995, + "falseNegative": 153, + "falsePositive": 46, + "precision": 0.193, + "predictedPositive": 57, + "recall": 0.0671, + "total": 1004, + "trueNegative": 794, + "truePositive": 11, + "wilsonPrecisionLower95": 0.1113 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 2, + "falsePositive": 61, + "precision": 0.0, + "predictedPositive": 61, + "recall": 0.0, + "total": 1004, + "trueNegative": 941, + "truePositive": 0, + "wilsonPrecisionLower95": -0.0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 2, + "falsePositive": 1, + "precision": 0.0, + "predictedPositive": 1, + "recall": 0.0, + "total": 1004, + "trueNegative": 1001, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 2, + "precision": 0.0, + "predictedPositive": 2, + "recall": 0.0, + "total": 1004, + "trueNegative": 1002, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "question": { + "f1": 0.4474, + "falseNegative": 39, + "falsePositive": 3, + "precision": 0.85, + "predictedPositive": 20, + "recall": 0.3036, + "total": 1004, + "trueNegative": 945, + "truePositive": 17, + "wilsonPrecisionLower95": 0.6396 + }, + "replyableMessage": { + "f1": 0.8326, + "falseNegative": 288, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 716, + "recall": 0.7131, + "total": 1004, + "trueNegative": 0, + "truePositive": 716, + "wilsonPrecisionLower95": 0.9947 + }, + "scheduleNegotiation": { + "f1": 0.4737, + "falseNegative": 16, + "falsePositive": 4, + "precision": 0.6923, + "predictedPositive": 13, + "recall": 0.36, + "total": 1004, + "trueNegative": 975, + "truePositive": 9, + "wilsonPrecisionLower95": 0.4237 + }, + "task": { + "f1": 0.0, + "falseNegative": 90, + "falsePositive": 3, + "precision": 0.0, + "predictedPositive": 3, + "recall": 0.0, + "total": 1004, + "trueNegative": 911, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + } + }, + "recordCount": 1004 + }, + "Verified Enron Intent Dataset": { + "intentsMeeting95Precision": 0, + "macroF1": 0.3577, + "macroPrecision": 0.3369, + "macroRecall": 0.4364, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 1, + "recall": 1.0, + "total": 3537, + "trueNegative": 3536, + "truePositive": 1, + "wilsonPrecisionLower95": 0.2065 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 5, + "precision": 0.0, + "predictedPositive": 5, + "recall": 0.0, + "total": 3537, + "trueNegative": 3532, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "confirmationDecision": { + "f1": 0.0104, + "falseNegative": 8, + "falsePositive": 372, + "precision": 0.0053, + "predictedPositive": 374, + "recall": 0.2, + "total": 3537, + "trueNegative": 3155, + "truePositive": 2, + "wilsonPrecisionLower95": 0.0015 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 26, + "falsePositive": 7, + "precision": 0.0, + "predictedPositive": 7, + "recall": 0.0, + "total": 3537, + "trueNegative": 3504, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "invitation": { + "f1": 0.3889, + "falseNegative": 40, + "falsePositive": 26, + "precision": 0.4468, + "predictedPositive": 47, + "recall": 0.3443, + "total": 3537, + "trueNegative": 3450, + "truePositive": 21, + "wilsonPrecisionLower95": 0.3141 + }, + "question": { + "f1": 0.6213, + "falseNegative": 143, + "falsePositive": 213, + "precision": 0.5782, + "predictedPositive": 505, + "recall": 0.6713, + "total": 3537, + "trueNegative": 2889, + "truePositive": 292, + "wilsonPrecisionLower95": 0.5347 + }, + "replyableMessage": { + "f1": 0.5259, + "falseNegative": 115, + "falsePositive": 1917, + "precision": 0.3702, + "predictedPositive": 3044, + "recall": 0.9074, + "total": 3537, + "trueNegative": 378, + "truePositive": 1127, + "wilsonPrecisionLower95": 0.3533 + }, + "scheduleNegotiation": { + "f1": 0.2482, + "falseNegative": 25, + "falsePositive": 78, + "precision": 0.1789, + "predictedPositive": 95, + "recall": 0.4048, + "total": 3537, + "trueNegative": 3417, + "truePositive": 17, + "wilsonPrecisionLower95": 0.1148 + }, + "task": { + "f1": 0.4248, + "falseNegative": 231, + "falsePositive": 186, + "precision": 0.4529, + "predictedPositive": 340, + "recall": 0.4, + "total": 3537, + "trueNegative": 2966, + "truePositive": 154, + "wilsonPrecisionLower95": 0.4008 + } + }, + "recordCount": 3537 + } + }, + "intentsMeeting95Precision": 0, + "macroF1": 0.2383, + "macroPrecision": 0.4398, + "macroRecall": 0.2161, + "minimumPrecision": 0.0035, + "perIntent": { + "blessing": { + "f1": 0.1895, + "falseNegative": 77, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 9, + "recall": 0.1047, + "total": 41195, + "trueNegative": 41109, + "truePositive": 9, + "wilsonPrecisionLower95": 0.7009 + }, + "complaint": { + "f1": 0.2322, + "falseNegative": 575, + "falsePositive": 278, + "precision": 0.317, + "predictedPositive": 407, + "recall": 0.1832, + "total": 41195, + "trueNegative": 40213, + "truePositive": 129, + "wilsonPrecisionLower95": 0.2736 + }, + "confirmationDecision": { + "f1": 0.0068, + "falseNegative": 63, + "falsePositive": 1695, + "precision": 0.0035, + "predictedPositive": 1701, + "recall": 0.087, + "total": 41195, + "trueNegative": 39431, + "truePositive": 6, + "wilsonPrecisionLower95": 0.0016 + }, + "followUpReminder": { + "f1": 0.1591, + "falseNegative": 348, + "falsePositive": 96, + "precision": 0.3043, + "predictedPositive": 138, + "recall": 0.1077, + "total": 41195, + "trueNegative": 40709, + "truePositive": 42, + "wilsonPrecisionLower95": 0.2337 + }, + "invitation": { + "f1": 0.2029, + "falseNegative": 113, + "falsePositive": 52, + "precision": 0.2877, + "predictedPositive": 73, + "recall": 0.1567, + "total": 41195, + "trueNegative": 41009, + "truePositive": 21, + "wilsonPrecisionLower95": 0.1965 + }, + "question": { + "f1": 0.2356, + "falseNegative": 8447, + "falsePositive": 594, + "precision": 0.7011, + "predictedPositive": 1987, + "recall": 0.1416, + "total": 41195, + "trueNegative": 30761, + "truePositive": 1393, + "wilsonPrecisionLower95": 0.6806 + }, + "replyableMessage": { + "f1": 0.7628, + "falseNegative": 12447, + "falsePositive": 3013, + "precision": 0.8919, + "predictedPositive": 27873, + "recall": 0.6664, + "total": 41195, + "trueNegative": 875, + "truePositive": 24860, + "wilsonPrecisionLower95": 0.8882 + }, + "scheduleNegotiation": { + "f1": 0.1651, + "falseNegative": 45, + "falsePositive": 218, + "precision": 0.1066, + "predictedPositive": 244, + "recall": 0.3662, + "total": 41195, + "trueNegative": 40906, + "truePositive": 26, + "wilsonPrecisionLower95": 0.0738 + }, + "task": { + "f1": 0.1907, + "falseNegative": 1544, + "falsePositive": 442, + "precision": 0.3462, + "predictedPositive": 676, + "recall": 0.1316, + "total": 41195, + "trueNegative": 38975, + "truePositive": 234, + "wilsonPrecisionLower95": 0.3113 + } + }, + "recordCount": 41195 + }, + "targetedRelease": { + "byLanguage": { + "en": { + "intentsMeeting95Precision": 4, + "macroF1": 0.4138, + "macroPrecision": 0.5476, + "macroRecall": 0.3512, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 320, + "trueNegative": 320, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.6957, + "falseNegative": 56, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 64, + "recall": 0.5333, + "total": 320, + "trueNegative": 200, + "truePositive": 64, + "wilsonPrecisionLower95": 0.9434 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 4, + "precision": 0.0, + "predictedPositive": 4, + "recall": 0.0, + "total": 320, + "trueNegative": 316, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "followUpReminder": { + "f1": 0.961, + "falseNegative": 3, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 37, + "recall": 0.925, + "total": 320, + "trueNegative": 280, + "truePositive": 37, + "wilsonPrecisionLower95": 0.9059 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 320, + "trueNegative": 320, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.4762, + "falseNegative": 55, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 25, + "recall": 0.3125, + "total": 320, + "trueNegative": 240, + "truePositive": 25, + "wilsonPrecisionLower95": 0.8668 + }, + "replyableMessage": { + "f1": 0.8819, + "falseNegative": 32, + "falsePositive": 13, + "precision": 0.9282, + "predictedPositive": 181, + "recall": 0.84, + "total": 320, + "trueNegative": 107, + "truePositive": 168, + "wilsonPrecisionLower95": 0.881 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 320, + "trueNegative": 320, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.7097, + "falseNegative": 36, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 44, + "recall": 0.55, + "total": 320, + "trueNegative": 240, + "truePositive": 44, + "wilsonPrecisionLower95": 0.9197 + } + }, + "recordCount": 320 + }, + "zh-Hans": { + "intentsMeeting95Precision": 4, + "macroF1": 0.3345, + "macroPrecision": 0.5501, + "macroRecall": 0.2766, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 320, + "trueNegative": 320, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.4516, + "falseNegative": 85, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 35, + "recall": 0.2917, + "total": 320, + "trueNegative": 200, + "truePositive": 35, + "wilsonPrecisionLower95": 0.9011 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 320, + "trueNegative": 320, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.2609, + "falseNegative": 34, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 6, + "recall": 0.15, + "total": 320, + "trueNegative": 280, + "truePositive": 6, + "wilsonPrecisionLower95": 0.6097 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 320, + "trueNegative": 320, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.8811, + "falseNegative": 17, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 63, + "recall": 0.7875, + "total": 320, + "trueNegative": 240, + "truePositive": 63, + "wilsonPrecisionLower95": 0.9425 + }, + "replyableMessage": { + "f1": 0.9552, + "falseNegative": 8, + "falsePositive": 10, + "precision": 0.9505, + "predictedPositive": 202, + "recall": 0.96, + "total": 320, + "trueNegative": 110, + "truePositive": 192, + "wilsonPrecisionLower95": 0.9113 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 320, + "trueNegative": 320, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.4615, + "falseNegative": 56, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 24, + "recall": 0.3, + "total": 320, + "trueNegative": 240, + "truePositive": 24, + "wilsonPrecisionLower95": 0.862 + } + }, + "recordCount": 320 + } + }, + "bySource": { + "release_complaint_brief": { + "intentsMeeting95Precision": 2, + "macroF1": 0.1893, + "macroPrecision": 0.2222, + "macroRecall": 0.1708, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.7097, + "falseNegative": 36, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 44, + "recall": 0.55, + "total": 80, + "trueNegative": 0, + "truePositive": 44, + "wilsonPrecisionLower95": 0.9197 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.9937, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 79, + "recall": 0.9875, + "total": 80, + "trueNegative": 0, + "truePositive": 79, + "wilsonPrecisionLower95": 0.9536 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 80 + }, + "release_complaint_question": { + "intentsMeeting95Precision": 3, + "macroF1": 0.2152, + "macroPrecision": 0.3333, + "macroRecall": 0.1667, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.5586, + "falseNegative": 49, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 31, + "recall": 0.3875, + "total": 80, + "trueNegative": 0, + "truePositive": 31, + "wilsonPrecisionLower95": 0.8897 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.5047, + "falseNegative": 53, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 27, + "recall": 0.3375, + "total": 80, + "trueNegative": 0, + "truePositive": 27, + "wilsonPrecisionLower95": 0.8754 + }, + "replyableMessage": { + "f1": 0.8732, + "falseNegative": 18, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 62, + "recall": 0.775, + "total": 80, + "trueNegative": 0, + "truePositive": 62, + "wilsonPrecisionLower95": 0.9417 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 80 + }, + "release_complaint_unspoken": { + "intentsMeeting95Precision": 2, + "macroF1": 0.1565, + "macroPrecision": 0.2222, + "macroRecall": 0.1333, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.4615, + "falseNegative": 56, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 24, + "recall": 0.3, + "total": 80, + "trueNegative": 0, + "truePositive": 24, + "wilsonPrecisionLower95": 0.862 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.9474, + "falseNegative": 8, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 72, + "recall": 0.9, + "total": 80, + "trueNegative": 0, + "truePositive": 72, + "wilsonPrecisionLower95": 0.9493 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 80 + }, + "release_negative_boundary": { + "intentsMeeting95Precision": 0, + "macroF1": 0.0, + "macroPrecision": 0.0, + "macroRecall": 0.0, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 80 + }, + "release_personal_boundary": { + "intentsMeeting95Precision": 1, + "macroF1": 0.0777, + "macroPrecision": 0.1111, + "macroRecall": 0.0597, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.6992, + "falseNegative": 37, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 43, + "recall": 0.5375, + "total": 80, + "trueNegative": 0, + "truePositive": 43, + "wilsonPrecisionLower95": 0.918 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 4, + "precision": 0.0, + "predictedPositive": 4, + "recall": 0.0, + "total": 80, + "trueNegative": 76, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 80 + }, + "release_resolved_boundary": { + "intentsMeeting95Precision": 0, + "macroF1": 0.0, + "macroPrecision": 0.0, + "macroRecall": 0.0, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 19, + "precision": 0.0, + "predictedPositive": 19, + "recall": 0.0, + "total": 80, + "trueNegative": 61, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + } + }, + "recordCount": 80 + }, + "release_task_owner": { + "intentsMeeting95Precision": 2, + "macroF1": 0.171, + "macroPrecision": 0.2222, + "macroRecall": 0.1514, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 4, + "precision": 0.0, + "predictedPositive": 4, + "recall": 0.0, + "total": 80, + "trueNegative": 76, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "replyableMessage": { + "f1": 0.9937, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 79, + "recall": 0.9875, + "total": 80, + "trueNegative": 0, + "truePositive": 79, + "wilsonPrecisionLower95": 0.9536 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.5455, + "falseNegative": 50, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 30, + "recall": 0.375, + "total": 80, + "trueNegative": 0, + "truePositive": 30, + "wilsonPrecisionLower95": 0.8865 + } + }, + "recordCount": 80 + }, + "release_task_polite_request": { + "intentsMeeting95Precision": 3, + "macroF1": 0.2698, + "macroPrecision": 0.3333, + "macroRecall": 0.2319, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "followUpReminder": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.8652, + "falseNegative": 19, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 61, + "recall": 0.7625, + "total": 80, + "trueNegative": 0, + "truePositive": 61, + "wilsonPrecisionLower95": 0.9408 + }, + "replyableMessage": { + "f1": 0.9189, + "falseNegative": 12, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 68, + "recall": 0.85, + "total": 80, + "trueNegative": 0, + "truePositive": 68, + "wilsonPrecisionLower95": 0.9465 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 80, + "trueNegative": 80, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.6441, + "falseNegative": 42, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 38, + "recall": 0.475, + "total": 80, + "trueNegative": 0, + "truePositive": 38, + "wilsonPrecisionLower95": 0.9082 + } + }, + "recordCount": 80 + } + }, + "intentsMeeting95Precision": 4, + "macroF1": 0.3899, + "macroPrecision": 0.5489, + "macroRecall": 0.3139, + "minimumPrecision": 0.0, + "perIntent": { + "blessing": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 640, + "trueNegative": 640, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "complaint": { + "f1": 0.5841, + "falseNegative": 141, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 99, + "recall": 0.4125, + "total": 640, + "trueNegative": 400, + "truePositive": 99, + "wilsonPrecisionLower95": 0.9626 + }, + "confirmationDecision": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 4, + "precision": 0.0, + "predictedPositive": 4, + "recall": 0.0, + "total": 640, + "trueNegative": 636, + "truePositive": 0, + "wilsonPrecisionLower95": 0.0 + }, + "followUpReminder": { + "f1": 0.6992, + "falseNegative": 37, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 43, + "recall": 0.5375, + "total": 640, + "trueNegative": 560, + "truePositive": 43, + "wilsonPrecisionLower95": 0.918 + }, + "invitation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 640, + "trueNegative": 640, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "question": { + "f1": 0.7097, + "falseNegative": 72, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 88, + "recall": 0.55, + "total": 640, + "trueNegative": 480, + "truePositive": 88, + "wilsonPrecisionLower95": 0.9582 + }, + "replyableMessage": { + "f1": 0.9195, + "falseNegative": 40, + "falsePositive": 23, + "precision": 0.9399, + "predictedPositive": 383, + "recall": 0.9, + "total": 640, + "trueNegative": 217, + "truePositive": 360, + "wilsonPrecisionLower95": 0.9115 + }, + "scheduleNegotiation": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 640, + "trueNegative": 640, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "task": { + "f1": 0.5965, + "falseNegative": 92, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 68, + "recall": 0.425, + "total": 640, + "trueNegative": 480, + "truePositive": 68, + "wilsonPrecisionLower95": 0.9465 + } + }, + "recordCount": 640 + } + }, + "generatedAtUnix": 1787825530, + "openTrainingRecords": 3845, + "overlapChecks": { + "ModelTraining/ClipboardSemantics/comprehensive-online-holdout-corpus.jsonl": 0, + "ModelTraining/ClipboardSemantics/random-holdout-corpus.jsonl": 0, + "ModelTraining/ClipboardSemantics/targeted-release-holdout-corpus.jsonl": 0 + }, + "platformLimitation": "Linux surrogate research only. Apple Create ML/Core ML artifacts must be retrained and tested on macOS before deployment.", + "releaseGatePassed": false, + "roundCount": 20, + "schemaVersion": 1, + "seed": 20260827, + "selectedConfiguration": { + "alpha": 2e-05, + "augmentation": "none", + "char_max": 5, + "char_min": 2, + "external_weight": 0.5700000000000001, + "hard_example_weight": 1.0, + "l1_ratio": 0.0, + "max_char_features": 45000, + "max_word_features": 22000, + "min_df": 2, + "pseudo_weight": 0.16499999999999998, + "round": 1, + "word_max": 2 + }, + "selectedRound": 1, + "selectedThresholds": { + "blessing": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9318, + "falseNegative": 4, + "falsePositive": 2, + "precision": 0.9535, + "predictedPositive": 43, + "recall": 0.9111, + "total": 1485, + "trueNegative": 1438, + "truePositive": 41, + "wilsonPrecisionLower95": 0.8454 + }, + "threshold": 0.915 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7568, + "falseNegative": 17, + "falsePositive": 1, + "precision": 0.9655, + "predictedPositive": 29, + "recall": 0.6222, + "total": 1485, + "trueNegative": 1439, + "truePositive": 28, + "wilsonPrecisionLower95": 0.8282 + }, + "threshold": 0.9 + } + }, + "metrics": { + "f1": 0.8375, + "falseNegative": 23, + "falsePositive": 3, + "precision": 0.9571, + "predictedPositive": 70, + "recall": 0.7444, + "total": 2970, + "trueNegative": 2877, + "truePositive": 67, + "wilsonPrecisionLower95": 0.8814 + }, + "threshold": 0.915 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 180, + "recall": 1.0, + "total": 1485, + "trueNegative": 1305, + "truePositive": 180, + "wilsonPrecisionLower95": 0.9791 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9526, + "falseNegative": 9, + "falsePositive": 8, + "precision": 0.9553, + "predictedPositive": 179, + "recall": 0.95, + "total": 1485, + "trueNegative": 1297, + "truePositive": 171, + "wilsonPrecisionLower95": 0.9143 + }, + "threshold": 0.54 + } + }, + "metrics": { + "f1": 0.9764, + "falseNegative": 9, + "falsePositive": 8, + "precision": 0.9777, + "predictedPositive": 359, + "recall": 0.975, + "total": 2970, + "trueNegative": 2602, + "truePositive": 351, + "wilsonPrecisionLower95": 0.9567 + }, + "threshold": 0.54 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8848, + "falseNegative": 17, + "falsePositive": 2, + "precision": 0.9733, + "predictedPositive": 75, + "recall": 0.8111, + "total": 1485, + "trueNegative": 1393, + "truePositive": 73, + "wilsonPrecisionLower95": 0.9079 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7947, + "falseNegative": 30, + "falsePositive": 1, + "precision": 0.9836, + "predictedPositive": 61, + "recall": 0.6667, + "total": 1485, + "trueNegative": 1394, + "truePositive": 60, + "wilsonPrecisionLower95": 0.9128 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.8418, + "falseNegative": 47, + "falsePositive": 3, + "precision": 0.9779, + "predictedPositive": 136, + "recall": 0.7389, + "total": 2970, + "trueNegative": 2787, + "truePositive": 133, + "wilsonPrecisionLower95": 0.9372 + }, + "threshold": 0.5 + }, + "followUpReminder": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8554, + "falseNegative": 41, + "falsePositive": 6, + "precision": 0.9586, + "predictedPositive": 145, + "recall": 0.7722, + "total": 1485, + "trueNegative": 1299, + "truePositive": 139, + "wilsonPrecisionLower95": 0.9127 + }, + "threshold": 0.83 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8354, + "falseNegative": 48, + "falsePositive": 4, + "precision": 0.9706, + "predictedPositive": 136, + "recall": 0.7333, + "total": 1485, + "trueNegative": 1301, + "truePositive": 132, + "wilsonPrecisionLower95": 0.9268 + }, + "threshold": 0.505 + } + }, + "metrics": { + "f1": 0.7994, + "falseNegative": 113, + "falsePositive": 11, + "precision": 0.9574, + "predictedPositive": 258, + "recall": 0.6861, + "total": 2970, + "trueNegative": 2599, + "truePositive": 247, + "wilsonPrecisionLower95": 0.9253 + }, + "threshold": 0.815 + }, + "invitation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8302, + "falseNegative": 24, + "falsePositive": 3, + "precision": 0.9565, + "predictedPositive": 69, + "recall": 0.7333, + "total": 1485, + "trueNegative": 1392, + "truePositive": 66, + "wilsonPrecisionLower95": 0.8798 + }, + "threshold": 0.935 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6765, + "falseNegative": 44, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 46, + "recall": 0.5111, + "total": 1485, + "trueNegative": 1395, + "truePositive": 46, + "wilsonPrecisionLower95": 0.9229 + }, + "threshold": 0.505 + } + }, + "metrics": { + "f1": 0.5476, + "falseNegative": 111, + "falsePositive": 3, + "precision": 0.9583, + "predictedPositive": 72, + "recall": 0.3833, + "total": 2970, + "trueNegative": 2787, + "truePositive": 69, + "wilsonPrecisionLower95": 0.8845 + }, + "threshold": 0.935 + }, + "question": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8445, + "falseNegative": 76, + "falsePositive": 12, + "precision": 0.9522, + "predictedPositive": 251, + "recall": 0.7587, + "total": 1485, + "trueNegative": 1158, + "truePositive": 239, + "wilsonPrecisionLower95": 0.9183 + }, + "threshold": 0.725 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8536, + "falseNegative": 76, + "falsePositive": 6, + "precision": 0.9755, + "predictedPositive": 245, + "recall": 0.7587, + "total": 1485, + "trueNegative": 1164, + "truePositive": 239, + "wilsonPrecisionLower95": 0.9476 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.8729, + "falseNegative": 122, + "falsePositive": 26, + "precision": 0.9513, + "predictedPositive": 534, + "recall": 0.8063, + "total": 2970, + "trueNegative": 2314, + "truePositive": 508, + "wilsonPrecisionLower95": 0.9296 + }, + "threshold": 0.515 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.948, + "falseNegative": 52, + "falsePositive": 46, + "precision": 0.951, + "predictedPositive": 939, + "recall": 0.945, + "total": 1485, + "trueNegative": 494, + "truePositive": 893, + "wilsonPrecisionLower95": 0.9353 + }, + "threshold": 0.7 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9424, + "falseNegative": 62, + "falsePositive": 46, + "precision": 0.9505, + "predictedPositive": 929, + "recall": 0.9344, + "total": 1485, + "trueNegative": 494, + "truePositive": 883, + "wilsonPrecisionLower95": 0.9346 + }, + "threshold": 0.695 + } + }, + "metrics": { + "f1": 0.9454, + "falseNegative": 115, + "falsePositive": 90, + "precision": 0.9517, + "predictedPositive": 1865, + "recall": 0.9392, + "total": 2970, + "trueNegative": 990, + "truePositive": 1775, + "wilsonPrecisionLower95": 0.9411 + }, + "threshold": 0.7 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 45, + "recall": 1.0, + "total": 1485, + "trueNegative": 1440, + "truePositive": 45, + "wilsonPrecisionLower95": 0.9213 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.967, + "falseNegative": 1, + "falsePositive": 2, + "precision": 0.9565, + "predictedPositive": 46, + "recall": 0.9778, + "total": 1485, + "trueNegative": 1438, + "truePositive": 44, + "wilsonPrecisionLower95": 0.8547 + }, + "threshold": 0.93 + } + }, + "metrics": { + "f1": 0.978, + "falseNegative": 1, + "falsePositive": 3, + "precision": 0.9674, + "predictedPositive": 92, + "recall": 0.9889, + "total": 2970, + "trueNegative": 2877, + "truePositive": 89, + "wilsonPrecisionLower95": 0.9085 + }, + "threshold": 0.91 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8403, + "falseNegative": 89, + "falsePositive": 14, + "precision": 0.9509, + "predictedPositive": 285, + "recall": 0.7528, + "total": 1485, + "trueNegative": 1111, + "truePositive": 271, + "wilsonPrecisionLower95": 0.9192 + }, + "threshold": 0.835 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8278, + "falseNegative": 98, + "falsePositive": 11, + "precision": 0.9597, + "predictedPositive": 273, + "recall": 0.7278, + "total": 1485, + "trueNegative": 1114, + "truePositive": 262, + "wilsonPrecisionLower95": 0.9293 + }, + "threshold": 0.745 + } + }, + "metrics": { + "f1": 0.8637, + "falseNegative": 153, + "falsePositive": 26, + "precision": 0.9562, + "predictedPositive": 593, + "recall": 0.7875, + "total": 2970, + "trueNegative": 2224, + "truePositive": 567, + "wilsonPrecisionLower95": 0.9365 + }, + "threshold": 0.755 + } + }, + "selectionObjective": 0.8669, + "trainingPolicy": "No user clipboard data. Official partial labels plus deterministic label-preserving augmentation and three-round high-confidence self-training consensus. Frozen holdouts never enter training or threshold calibration." +} diff --git a/ModelTraining/ClipboardSemantics/IterativeResearch/macos-replay.json b/ModelTraining/ClipboardSemantics/IterativeResearch/macos-replay.json new file mode 100644 index 0000000..7172b98 --- /dev/null +++ b/ModelTraining/ClipboardSemantics/IterativeResearch/macos-replay.json @@ -0,0 +1,481 @@ +{ + "automaticPromotionAllowed": false, + "baseCorpus": "ModelTraining/ClipboardSemantics/clipboard_semantic_corpus.jsonl", + "configuration": { + "alpha": 2e-05, + "augmentation": "none", + "char_max": 5, + "char_min": 2, + "external_weight": 0.5700000000000001, + "hard_example_weight": 1.0, + "l1_ratio": 0.0, + "max_char_features": 45000, + "max_word_features": 22000, + "min_df": 2, + "pseudo_weight": 0.16499999999999998, + "round": 1, + "word_max": 2 + }, + "openCorpus": "ModelTraining/ClipboardSemantics/open-training-corpus.jsonl", + "reason": "Surrogate feature weights are not deployable NLModel assets and cannot bypass the existing macOS acceptance policy.", + "requiredCommands": [ + "python3 Scripts/clipboard_semantics/generate_open_training_corpus.py", + "xcrun swift Scripts/clipboard_semantics/train_models.swift --algorithms maxEnt --corpus ModelTraining/ClipboardSemantics/combined-training-corpus.jsonl", + "xcrun swift Scripts/clipboard_semantics/evaluate_random_holdout.swift --corpus ModelTraining/ClipboardSemantics/random-holdout-corpus.jsonl" + ], + "schemaVersion": 1, + "seed": 20260827, + "selectedRound": 1, + "thresholds": { + "blessing": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9318, + "falseNegative": 4, + "falsePositive": 2, + "precision": 0.9535, + "predictedPositive": 43, + "recall": 0.9111, + "total": 1485, + "trueNegative": 1438, + "truePositive": 41, + "wilsonPrecisionLower95": 0.8454 + }, + "threshold": 0.915 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7568, + "falseNegative": 17, + "falsePositive": 1, + "precision": 0.9655, + "predictedPositive": 29, + "recall": 0.6222, + "total": 1485, + "trueNegative": 1439, + "truePositive": 28, + "wilsonPrecisionLower95": 0.8282 + }, + "threshold": 0.9 + } + }, + "metrics": { + "f1": 0.8375, + "falseNegative": 23, + "falsePositive": 3, + "precision": 0.9571, + "predictedPositive": 70, + "recall": 0.7444, + "total": 2970, + "trueNegative": 2877, + "truePositive": 67, + "wilsonPrecisionLower95": 0.8814 + }, + "threshold": 0.915 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 180, + "recall": 1.0, + "total": 1485, + "trueNegative": 1305, + "truePositive": 180, + "wilsonPrecisionLower95": 0.9791 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9526, + "falseNegative": 9, + "falsePositive": 8, + "precision": 0.9553, + "predictedPositive": 179, + "recall": 0.95, + "total": 1485, + "trueNegative": 1297, + "truePositive": 171, + "wilsonPrecisionLower95": 0.9143 + }, + "threshold": 0.54 + } + }, + "metrics": { + "f1": 0.9764, + "falseNegative": 9, + "falsePositive": 8, + "precision": 0.9777, + "predictedPositive": 359, + "recall": 0.975, + "total": 2970, + "trueNegative": 2602, + "truePositive": 351, + "wilsonPrecisionLower95": 0.9567 + }, + "threshold": 0.54 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8848, + "falseNegative": 17, + "falsePositive": 2, + "precision": 0.9733, + "predictedPositive": 75, + "recall": 0.8111, + "total": 1485, + "trueNegative": 1393, + "truePositive": 73, + "wilsonPrecisionLower95": 0.9079 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7947, + "falseNegative": 30, + "falsePositive": 1, + "precision": 0.9836, + "predictedPositive": 61, + "recall": 0.6667, + "total": 1485, + "trueNegative": 1394, + "truePositive": 60, + "wilsonPrecisionLower95": 0.9128 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.8418, + "falseNegative": 47, + "falsePositive": 3, + "precision": 0.9779, + "predictedPositive": 136, + "recall": 0.7389, + "total": 2970, + "trueNegative": 2787, + "truePositive": 133, + "wilsonPrecisionLower95": 0.9372 + }, + "threshold": 0.5 + }, + "followUpReminder": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8554, + "falseNegative": 41, + "falsePositive": 6, + "precision": 0.9586, + "predictedPositive": 145, + "recall": 0.7722, + "total": 1485, + "trueNegative": 1299, + "truePositive": 139, + "wilsonPrecisionLower95": 0.9127 + }, + "threshold": 0.83 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8354, + "falseNegative": 48, + "falsePositive": 4, + "precision": 0.9706, + "predictedPositive": 136, + "recall": 0.7333, + "total": 1485, + "trueNegative": 1301, + "truePositive": 132, + "wilsonPrecisionLower95": 0.9268 + }, + "threshold": 0.505 + } + }, + "metrics": { + "f1": 0.7994, + "falseNegative": 113, + "falsePositive": 11, + "precision": 0.9574, + "predictedPositive": 258, + "recall": 0.6861, + "total": 2970, + "trueNegative": 2599, + "truePositive": 247, + "wilsonPrecisionLower95": 0.9253 + }, + "threshold": 0.815 + }, + "invitation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8302, + "falseNegative": 24, + "falsePositive": 3, + "precision": 0.9565, + "predictedPositive": 69, + "recall": 0.7333, + "total": 1485, + "trueNegative": 1392, + "truePositive": 66, + "wilsonPrecisionLower95": 0.8798 + }, + "threshold": 0.935 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6765, + "falseNegative": 44, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 46, + "recall": 0.5111, + "total": 1485, + "trueNegative": 1395, + "truePositive": 46, + "wilsonPrecisionLower95": 0.9229 + }, + "threshold": 0.505 + } + }, + "metrics": { + "f1": 0.5476, + "falseNegative": 111, + "falsePositive": 3, + "precision": 0.9583, + "predictedPositive": 72, + "recall": 0.3833, + "total": 2970, + "trueNegative": 2787, + "truePositive": 69, + "wilsonPrecisionLower95": 0.8845 + }, + "threshold": 0.935 + }, + "question": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8445, + "falseNegative": 76, + "falsePositive": 12, + "precision": 0.9522, + "predictedPositive": 251, + "recall": 0.7587, + "total": 1485, + "trueNegative": 1158, + "truePositive": 239, + "wilsonPrecisionLower95": 0.9183 + }, + "threshold": 0.725 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8536, + "falseNegative": 76, + "falsePositive": 6, + "precision": 0.9755, + "predictedPositive": 245, + "recall": 0.7587, + "total": 1485, + "trueNegative": 1164, + "truePositive": 239, + "wilsonPrecisionLower95": 0.9476 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.8729, + "falseNegative": 122, + "falsePositive": 26, + "precision": 0.9513, + "predictedPositive": 534, + "recall": 0.8063, + "total": 2970, + "trueNegative": 2314, + "truePositive": 508, + "wilsonPrecisionLower95": 0.9296 + }, + "threshold": 0.515 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.948, + "falseNegative": 52, + "falsePositive": 46, + "precision": 0.951, + "predictedPositive": 939, + "recall": 0.945, + "total": 1485, + "trueNegative": 494, + "truePositive": 893, + "wilsonPrecisionLower95": 0.9353 + }, + "threshold": 0.7 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9424, + "falseNegative": 62, + "falsePositive": 46, + "precision": 0.9505, + "predictedPositive": 929, + "recall": 0.9344, + "total": 1485, + "trueNegative": 494, + "truePositive": 883, + "wilsonPrecisionLower95": 0.9346 + }, + "threshold": 0.695 + } + }, + "metrics": { + "f1": 0.9454, + "falseNegative": 115, + "falsePositive": 90, + "precision": 0.9517, + "predictedPositive": 1865, + "recall": 0.9392, + "total": 2970, + "trueNegative": 990, + "truePositive": 1775, + "wilsonPrecisionLower95": 0.9411 + }, + "threshold": 0.7 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 45, + "recall": 1.0, + "total": 1485, + "trueNegative": 1440, + "truePositive": 45, + "wilsonPrecisionLower95": 0.9213 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.967, + "falseNegative": 1, + "falsePositive": 2, + "precision": 0.9565, + "predictedPositive": 46, + "recall": 0.9778, + "total": 1485, + "trueNegative": 1438, + "truePositive": 44, + "wilsonPrecisionLower95": 0.8547 + }, + "threshold": 0.93 + } + }, + "metrics": { + "f1": 0.978, + "falseNegative": 1, + "falsePositive": 3, + "precision": 0.9674, + "predictedPositive": 92, + "recall": 0.9889, + "total": 2970, + "trueNegative": 2877, + "truePositive": 89, + "wilsonPrecisionLower95": 0.9085 + }, + "threshold": 0.91 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8403, + "falseNegative": 89, + "falsePositive": 14, + "precision": 0.9509, + "predictedPositive": 285, + "recall": 0.7528, + "total": 1485, + "trueNegative": 1111, + "truePositive": 271, + "wilsonPrecisionLower95": 0.9192 + }, + "threshold": 0.835 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8278, + "falseNegative": 98, + "falsePositive": 11, + "precision": 0.9597, + "predictedPositive": 273, + "recall": 0.7278, + "total": 1485, + "trueNegative": 1114, + "truePositive": 262, + "wilsonPrecisionLower95": 0.9293 + }, + "threshold": 0.745 + } + }, + "metrics": { + "f1": 0.8637, + "falseNegative": 153, + "falsePositive": 26, + "precision": 0.9562, + "predictedPositive": 593, + "recall": 0.7875, + "total": 2970, + "trueNegative": 2224, + "truePositive": 567, + "wilsonPrecisionLower95": 0.9365 + }, + "threshold": 0.755 + } + } +} diff --git a/ModelTraining/ClipboardSemantics/IterativeResearch/research-summary.json b/ModelTraining/ClipboardSemantics/IterativeResearch/research-summary.json new file mode 100644 index 0000000..c5b9c42 --- /dev/null +++ b/ModelTraining/ClipboardSemantics/IterativeResearch/research-summary.json @@ -0,0 +1,129 @@ +{ + "schemaVersion": 1, + "completedResearchRounds": 60, + "requestedCoreRounds": 20, + "seed": 20260827, + "data": { + "baseTrainingRecords": 11500, + "reachableLicensedOpenRecords": 3845, + "holdoutOverlapCounts": { + "comprehensiveOnline": 0, + "random": 0, + "targetedRelease": 0 + }, + "unavailableSources": [ + "MASSIVE", + "CLINC150", + "CFPB" + ] + }, + "phases": [ + { + "name": "shared-configuration-search", + "rounds": 20, + "selectedRound": 1, + "golden": { + "macroPrecision": 0.9366, + "macroRecall": 0.6801, + "macroF1": 0.7711 + }, + "randomHoldout": { + "macroPrecision": 0.8569, + "macroRecall": 0.4603, + "macroF1": 0.5709 + }, + "researchOnlyComprehensive": { + "macroPrecision": 0.4398, + "macroRecall": 0.2161, + "macroF1": 0.2383 + }, + "targetedRelease": { + "macroPrecision": 0.5489, + "macroRecall": 0.3139, + "macroF1": 0.3899 + } + }, + { + "name": "per-intent-fine-tuning", + "rounds": 20, + "selectedRoundsByIntent": { + "blessing": 19, + "complaint": 1, + "confirmationDecision": 3, + "followUpReminder": 1, + "invitation": 1, + "question": 1, + "replyableMessage": 1, + "scheduleNegotiation": 7, + "task": 15 + }, + "golden": { + "macroPrecision": 0.9377, + "macroRecall": 0.6819, + "macroF1": 0.7666 + }, + "randomHoldout": { + "macroPrecision": 0.8701, + "macroRecall": 0.4705, + "macroF1": 0.5770 + }, + "researchOnlyComprehensive": { + "macroPrecision": 0.4292, + "macroRecall": 0.2250, + "macroF1": 0.2645 + }, + "targetedRelease": { + "macroPrecision": 0.5467, + "macroRecall": 0.3375, + "macroF1": 0.4095 + } + }, + { + "name": "explicit-evidence-fine-tuning", + "rounds": 20, + "selectedRound": 1, + "golden": { + "macroPrecision": 0.9877, + "macroRecall": 0.4354, + "macroF1": 0.5476 + }, + "randomHoldout": { + "macroPrecision": 0.7731, + "macroRecall": 0.3029, + "macroF1": 0.3905 + }, + "researchOnlyComprehensive": { + "macroPrecision": 0.6097, + "macroRecall": 0.1647, + "macroF1": 0.2325 + }, + "targetedRelease": { + "macroPrecision": 0.5489, + "macroRecall": 0.2537, + "macroF1": 0.3209 + } + } + ], + "currentDeployedReference": { + "randomHoldout": { + "macroPrecision": 0.9832, + "macroRecall": 0.6557, + "macroF1": 0.7669 + }, + "researchOnlyComprehensive": { + "macroPrecision": 0.2948, + "macroRecall": 0.3320, + "macroF1": 0.2783 + } + }, + "findings": [ + "The first low-complexity round generalized better than later self-training rounds.", + "Three-round high-confidence pseudo-labeling amplified source and template bias instead of improving frozen regressions.", + "Per-intent configuration selection improved the research-only comprehensive F1 from 0.2383 to 0.2645 but remained below the deployed reference F1 of 0.2783.", + "Explicit evidence gates raised golden precision to 0.9877 and research-only precision to 0.6097, but recall collapsed and the release gate still failed.", + "No surrogate candidate is eligible to replace or augment the deployed Apple models." + ], + "releaseGatePassed": false, + "deploymentDecision": "retain-current-deployed-models", + "platformLimitation": "Linux cannot train, compile, run, or benchmark the deployable Apple Create ML models. These rounds are surrogate research and produce a macOS replay specification, not app-ready model assets." +} diff --git a/ModelTraining/ClipboardSemantics/IterativeResearch/rounds.json b/ModelTraining/ClipboardSemantics/IterativeResearch/rounds.json new file mode 100644 index 0000000..f64dde3 --- /dev/null +++ b/ModelTraining/ClipboardSemantics/IterativeResearch/rounds.json @@ -0,0 +1,14831 @@ +[ + { + "configuration": { + "alpha": 2e-05, + "augmentation": "none", + "char_max": 5, + "char_min": 2, + "external_weight": 0.5700000000000001, + "hard_example_weight": 1.0, + "l1_ratio": 0.0, + "max_char_features": 45000, + "max_word_features": 22000, + "min_df": 2, + "pseudo_weight": 0.16499999999999998, + "round": 1, + "word_max": 2 + }, + "featureCount": 56451, + "pseudoLabelCounts": {}, + "round": 1, + "selectionObjective": 0.8669, + "syntheticRegression": { + "intentsMeeting95Precision": 6, + "macroF1": 0.8088, + "macroPrecision": 0.9661, + "macroRecall": 0.7541, + "minimumPrecision": 0.8589, + "perIntent": { + "blessing": { + "f1": 0.125, + "falseNegative": 84, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 6, + "recall": 0.0667, + "total": 2970, + "trueNegative": 2880, + "truePositive": 6, + "wilsonPrecisionLower95": 0.6097 + }, + "complaint": { + "f1": 0.9773, + "falseNegative": 16, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 344, + "recall": 0.9556, + "total": 2970, + "trueNegative": 2610, + "truePositive": 344, + "wilsonPrecisionLower95": 0.989 + }, + "confirmationDecision": { + "f1": 0.8163, + "falseNegative": 40, + "falsePositive": 23, + "precision": 0.8589, + "predictedPositive": 163, + "recall": 0.7778, + "total": 2970, + "trueNegative": 2767, + "truePositive": 140, + "wilsonPrecisionLower95": 0.7972 + }, + "followUpReminder": { + "f1": 0.8705, + "falseNegative": 81, + "falsePositive": 2, + "precision": 0.9929, + "predictedPositive": 281, + "recall": 0.775, + "total": 2970, + "trueNegative": 2608, + "truePositive": 279, + "wilsonPrecisionLower95": 0.9744 + }, + "invitation": { + "f1": 0.8545, + "falseNegative": 42, + "falsePositive": 5, + "precision": 0.965, + "predictedPositive": 143, + "recall": 0.7667, + "total": 2970, + "trueNegative": 2785, + "truePositive": 138, + "wilsonPrecisionLower95": 0.9208 + }, + "question": { + "f1": 0.9092, + "falseNegative": 104, + "falsePositive": 1, + "precision": 0.9981, + "predictedPositive": 527, + "recall": 0.8349, + "total": 2970, + "trueNegative": 2339, + "truePositive": 526, + "wilsonPrecisionLower95": 0.9893 + }, + "replyableMessage": { + "f1": 0.9465, + "falseNegative": 156, + "falsePositive": 40, + "precision": 0.9775, + "predictedPositive": 1774, + "recall": 0.9175, + "total": 2970, + "trueNegative": 1040, + "truePositive": 1734, + "wilsonPrecisionLower95": 0.9694 + }, + "scheduleNegotiation": { + "f1": 0.9727, + "falseNegative": 1, + "falsePositive": 4, + "precision": 0.957, + "predictedPositive": 93, + "recall": 0.9889, + "total": 2970, + "trueNegative": 2876, + "truePositive": 89, + "wilsonPrecisionLower95": 0.8946 + }, + "task": { + "f1": 0.8073, + "falseNegative": 213, + "falsePositive": 29, + "precision": 0.9459, + "predictedPositive": 536, + "recall": 0.7042, + "total": 2970, + "trueNegative": 2221, + "truePositive": 507, + "wilsonPrecisionLower95": 0.9234 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9318, + "falseNegative": 4, + "falsePositive": 2, + "precision": 0.9535, + "predictedPositive": 43, + "recall": 0.9111, + "total": 1485, + "trueNegative": 1438, + "truePositive": 41, + "wilsonPrecisionLower95": 0.8454 + }, + "threshold": 0.915 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7568, + "falseNegative": 17, + "falsePositive": 1, + "precision": 0.9655, + "predictedPositive": 29, + "recall": 0.6222, + "total": 1485, + "trueNegative": 1439, + "truePositive": 28, + "wilsonPrecisionLower95": 0.8282 + }, + "threshold": 0.9 + } + }, + "metrics": { + "f1": 0.8375, + "falseNegative": 23, + "falsePositive": 3, + "precision": 0.9571, + "predictedPositive": 70, + "recall": 0.7444, + "total": 2970, + "trueNegative": 2877, + "truePositive": 67, + "wilsonPrecisionLower95": 0.8814 + }, + "threshold": 0.915 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 180, + "recall": 1.0, + "total": 1485, + "trueNegative": 1305, + "truePositive": 180, + "wilsonPrecisionLower95": 0.9791 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9526, + "falseNegative": 9, + "falsePositive": 8, + "precision": 0.9553, + "predictedPositive": 179, + "recall": 0.95, + "total": 1485, + "trueNegative": 1297, + "truePositive": 171, + "wilsonPrecisionLower95": 0.9143 + }, + "threshold": 0.54 + } + }, + "metrics": { + "f1": 0.9764, + "falseNegative": 9, + "falsePositive": 8, + "precision": 0.9777, + "predictedPositive": 359, + "recall": 0.975, + "total": 2970, + "trueNegative": 2602, + "truePositive": 351, + "wilsonPrecisionLower95": 0.9567 + }, + "threshold": 0.54 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8848, + "falseNegative": 17, + "falsePositive": 2, + "precision": 0.9733, + "predictedPositive": 75, + "recall": 0.8111, + "total": 1485, + "trueNegative": 1393, + "truePositive": 73, + "wilsonPrecisionLower95": 0.9079 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7947, + "falseNegative": 30, + "falsePositive": 1, + "precision": 0.9836, + "predictedPositive": 61, + "recall": 0.6667, + "total": 1485, + "trueNegative": 1394, + "truePositive": 60, + "wilsonPrecisionLower95": 0.9128 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.8418, + "falseNegative": 47, + "falsePositive": 3, + "precision": 0.9779, + "predictedPositive": 136, + "recall": 0.7389, + "total": 2970, + "trueNegative": 2787, + "truePositive": 133, + "wilsonPrecisionLower95": 0.9372 + }, + "threshold": 0.5 + }, + "followUpReminder": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8554, + "falseNegative": 41, + "falsePositive": 6, + "precision": 0.9586, + "predictedPositive": 145, + "recall": 0.7722, + "total": 1485, + "trueNegative": 1299, + "truePositive": 139, + "wilsonPrecisionLower95": 0.9127 + }, + "threshold": 0.83 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8354, + "falseNegative": 48, + "falsePositive": 4, + "precision": 0.9706, + "predictedPositive": 136, + "recall": 0.7333, + "total": 1485, + "trueNegative": 1301, + "truePositive": 132, + "wilsonPrecisionLower95": 0.9268 + }, + "threshold": 0.505 + } + }, + "metrics": { + "f1": 0.7994, + "falseNegative": 113, + "falsePositive": 11, + "precision": 0.9574, + "predictedPositive": 258, + "recall": 0.6861, + "total": 2970, + "trueNegative": 2599, + "truePositive": 247, + "wilsonPrecisionLower95": 0.9253 + }, + "threshold": 0.815 + }, + "invitation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8302, + "falseNegative": 24, + "falsePositive": 3, + "precision": 0.9565, + "predictedPositive": 69, + "recall": 0.7333, + "total": 1485, + "trueNegative": 1392, + "truePositive": 66, + "wilsonPrecisionLower95": 0.8798 + }, + "threshold": 0.935 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6765, + "falseNegative": 44, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 46, + "recall": 0.5111, + "total": 1485, + "trueNegative": 1395, + "truePositive": 46, + "wilsonPrecisionLower95": 0.9229 + }, + "threshold": 0.505 + } + }, + "metrics": { + "f1": 0.5476, + "falseNegative": 111, + "falsePositive": 3, + "precision": 0.9583, + "predictedPositive": 72, + "recall": 0.3833, + "total": 2970, + "trueNegative": 2787, + "truePositive": 69, + "wilsonPrecisionLower95": 0.8845 + }, + "threshold": 0.935 + }, + "question": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8445, + "falseNegative": 76, + "falsePositive": 12, + "precision": 0.9522, + "predictedPositive": 251, + "recall": 0.7587, + "total": 1485, + "trueNegative": 1158, + "truePositive": 239, + "wilsonPrecisionLower95": 0.9183 + }, + "threshold": 0.725 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8536, + "falseNegative": 76, + "falsePositive": 6, + "precision": 0.9755, + "predictedPositive": 245, + "recall": 0.7587, + "total": 1485, + "trueNegative": 1164, + "truePositive": 239, + "wilsonPrecisionLower95": 0.9476 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.8729, + "falseNegative": 122, + "falsePositive": 26, + "precision": 0.9513, + "predictedPositive": 534, + "recall": 0.8063, + "total": 2970, + "trueNegative": 2314, + "truePositive": 508, + "wilsonPrecisionLower95": 0.9296 + }, + "threshold": 0.515 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.948, + "falseNegative": 52, + "falsePositive": 46, + "precision": 0.951, + "predictedPositive": 939, + "recall": 0.945, + "total": 1485, + "trueNegative": 494, + "truePositive": 893, + "wilsonPrecisionLower95": 0.9353 + }, + "threshold": 0.7 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9424, + "falseNegative": 62, + "falsePositive": 46, + "precision": 0.9505, + "predictedPositive": 929, + "recall": 0.9344, + "total": 1485, + "trueNegative": 494, + "truePositive": 883, + "wilsonPrecisionLower95": 0.9346 + }, + "threshold": 0.695 + } + }, + "metrics": { + "f1": 0.9454, + "falseNegative": 115, + "falsePositive": 90, + "precision": 0.9517, + "predictedPositive": 1865, + "recall": 0.9392, + "total": 2970, + "trueNegative": 990, + "truePositive": 1775, + "wilsonPrecisionLower95": 0.9411 + }, + "threshold": 0.7 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 45, + "recall": 1.0, + "total": 1485, + "trueNegative": 1440, + "truePositive": 45, + "wilsonPrecisionLower95": 0.9213 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.967, + "falseNegative": 1, + "falsePositive": 2, + "precision": 0.9565, + "predictedPositive": 46, + "recall": 0.9778, + "total": 1485, + "trueNegative": 1438, + "truePositive": 44, + "wilsonPrecisionLower95": 0.8547 + }, + "threshold": 0.93 + } + }, + "metrics": { + "f1": 0.978, + "falseNegative": 1, + "falsePositive": 3, + "precision": 0.9674, + "predictedPositive": 92, + "recall": 0.9889, + "total": 2970, + "trueNegative": 2877, + "truePositive": 89, + "wilsonPrecisionLower95": 0.9085 + }, + "threshold": 0.91 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8403, + "falseNegative": 89, + "falsePositive": 14, + "precision": 0.9509, + "predictedPositive": 285, + "recall": 0.7528, + "total": 1485, + "trueNegative": 1111, + "truePositive": 271, + "wilsonPrecisionLower95": 0.9192 + }, + "threshold": 0.835 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8278, + "falseNegative": 98, + "falsePositive": 11, + "precision": 0.9597, + "predictedPositive": 273, + "recall": 0.7278, + "total": 1485, + "trueNegative": 1114, + "truePositive": 262, + "wilsonPrecisionLower95": 0.9293 + }, + "threshold": 0.745 + } + }, + "metrics": { + "f1": 0.8637, + "falseNegative": 153, + "falsePositive": 26, + "precision": 0.9562, + "predictedPositive": 593, + "recall": 0.7875, + "total": 2970, + "trueNegative": 2224, + "truePositive": 567, + "wilsonPrecisionLower95": 0.9365 + }, + "threshold": 0.755 + } + }, + "trainingRecords": 15345, + "validation": { + "intentsMeeting95Precision": 9, + "macroF1": 0.822, + "macroPrecision": 0.9713, + "macroRecall": 0.7487, + "minimumPrecision": 0.9507, + "perIntent": { + "blessing": { + "f1": 0.3636, + "falseNegative": 70, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 20, + "recall": 0.2222, + "total": 2970, + "trueNegative": 2880, + "truePositive": 20, + "wilsonPrecisionLower95": 0.8389 + }, + "complaint": { + "f1": 0.9764, + "falseNegative": 9, + "falsePositive": 8, + "precision": 0.9777, + "predictedPositive": 359, + "recall": 0.975, + "total": 2970, + "trueNegative": 2602, + "truePositive": 351, + "wilsonPrecisionLower95": 0.9567 + }, + "confirmationDecision": { + "f1": 0.8418, + "falseNegative": 47, + "falsePositive": 3, + "precision": 0.9779, + "predictedPositive": 136, + "recall": 0.7389, + "total": 2970, + "trueNegative": 2787, + "truePositive": 133, + "wilsonPrecisionLower95": 0.9372 + }, + "followUpReminder": { + "f1": 0.8456, + "falseNegative": 89, + "falsePositive": 10, + "precision": 0.9644, + "predictedPositive": 281, + "recall": 0.7528, + "total": 2970, + "trueNegative": 2600, + "truePositive": 271, + "wilsonPrecisionLower95": 0.9357 + }, + "invitation": { + "f1": 0.7593, + "falseNegative": 68, + "falsePositive": 3, + "precision": 0.9739, + "predictedPositive": 115, + "recall": 0.6222, + "total": 2970, + "trueNegative": 2787, + "truePositive": 112, + "wilsonPrecisionLower95": 0.9261 + }, + "question": { + "f1": 0.849, + "falseNegative": 152, + "falsePositive": 18, + "precision": 0.9637, + "predictedPositive": 496, + "recall": 0.7587, + "total": 2970, + "trueNegative": 2322, + "truePositive": 478, + "wilsonPrecisionLower95": 0.9434 + }, + "replyableMessage": { + "f1": 0.9452, + "falseNegative": 114, + "falsePositive": 92, + "precision": 0.9507, + "predictedPositive": 1868, + "recall": 0.9397, + "total": 2970, + "trueNegative": 988, + "truePositive": 1776, + "wilsonPrecisionLower95": 0.94 + }, + "scheduleNegotiation": { + "f1": 0.9834, + "falseNegative": 1, + "falsePositive": 2, + "precision": 0.978, + "predictedPositive": 91, + "recall": 0.9889, + "total": 2970, + "trueNegative": 2878, + "truePositive": 89, + "wilsonPrecisionLower95": 0.9234 + }, + "task": { + "f1": 0.8341, + "falseNegative": 187, + "falsePositive": 25, + "precision": 0.9552, + "predictedPositive": 558, + "recall": 0.7403, + "total": 2970, + "trueNegative": 2225, + "truePositive": 533, + "wilsonPrecisionLower95": 0.9347 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 1.5e-05, + "augmentation": "none", + "char_max": 5, + "char_min": 3, + "external_weight": 0.5900000000000001, + "hard_example_weight": 1.0, + "l1_ratio": 0.0, + "max_char_features": 50000, + "max_word_features": 25000, + "min_df": 2, + "pseudo_weight": 0.18, + "round": 2, + "word_max": 2 + }, + "featureCount": 61451, + "pseudoLabelCounts": {}, + "round": 2, + "selectionObjective": 0.8228, + "syntheticRegression": { + "intentsMeeting95Precision": 7, + "macroF1": 0.751, + "macroPrecision": 0.9667, + "macroRecall": 0.6602, + "minimumPrecision": 0.8606, + "perIntent": { + "blessing": { + "f1": 0.2178, + "falseNegative": 79, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 11, + "recall": 0.1222, + "total": 2970, + "trueNegative": 2880, + "truePositive": 11, + "wilsonPrecisionLower95": 0.7412 + }, + "complaint": { + "f1": 0.9595, + "falseNegative": 28, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 332, + "recall": 0.9222, + "total": 2970, + "trueNegative": 2610, + "truePositive": 332, + "wilsonPrecisionLower95": 0.9886 + }, + "confirmationDecision": { + "f1": 0.8232, + "falseNegative": 38, + "falsePositive": 23, + "precision": 0.8606, + "predictedPositive": 165, + "recall": 0.7889, + "total": 2970, + "trueNegative": 2767, + "truePositive": 142, + "wilsonPrecisionLower95": 0.7995 + }, + "followUpReminder": { + "f1": 0.7565, + "falseNegative": 141, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 219, + "recall": 0.6083, + "total": 2970, + "trueNegative": 2610, + "truePositive": 219, + "wilsonPrecisionLower95": 0.9828 + }, + "invitation": { + "f1": 0.6394, + "falseNegative": 94, + "falsePositive": 3, + "precision": 0.9663, + "predictedPositive": 89, + "recall": 0.4778, + "total": 2970, + "trueNegative": 2787, + "truePositive": 86, + "wilsonPrecisionLower95": 0.9055 + }, + "question": { + "f1": 0.86, + "falseNegative": 151, + "falsePositive": 5, + "precision": 0.9897, + "predictedPositive": 484, + "recall": 0.7603, + "total": 2970, + "trueNegative": 2335, + "truePositive": 479, + "wilsonPrecisionLower95": 0.976 + }, + "replyableMessage": { + "f1": 0.9038, + "falseNegative": 293, + "falsePositive": 47, + "precision": 0.9714, + "predictedPositive": 1644, + "recall": 0.845, + "total": 2970, + "trueNegative": 1033, + "truePositive": 1597, + "wilsonPrecisionLower95": 0.9622 + }, + "scheduleNegotiation": { + "f1": 0.9425, + "falseNegative": 8, + "falsePositive": 2, + "precision": 0.9762, + "predictedPositive": 84, + "recall": 0.9111, + "total": 2970, + "trueNegative": 2878, + "truePositive": 82, + "wilsonPrecisionLower95": 0.9173 + }, + "task": { + "f1": 0.6564, + "falseNegative": 356, + "falsePositive": 25, + "precision": 0.9357, + "predictedPositive": 389, + "recall": 0.5056, + "total": 2970, + "trueNegative": 2225, + "truePositive": 364, + "wilsonPrecisionLower95": 0.9069 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9048, + "falseNegative": 7, + "falsePositive": 1, + "precision": 0.9744, + "predictedPositive": 39, + "recall": 0.8444, + "total": 1485, + "trueNegative": 1439, + "truePositive": 38, + "wilsonPrecisionLower95": 0.8682 + }, + "threshold": 0.59 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 180, + "recall": 1.0, + "total": 1485, + "trueNegative": 1305, + "truePositive": 180, + "wilsonPrecisionLower95": 0.9791 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9226, + "falseNegative": 19, + "falsePositive": 8, + "precision": 0.9527, + "predictedPositive": 169, + "recall": 0.8944, + "total": 1485, + "trueNegative": 1297, + "truePositive": 161, + "wilsonPrecisionLower95": 0.9094 + }, + "threshold": 0.555 + } + }, + "metrics": { + "f1": 0.9649, + "falseNegative": 16, + "falsePositive": 9, + "precision": 0.9745, + "predictedPositive": 353, + "recall": 0.9556, + "total": 2970, + "trueNegative": 2601, + "truePositive": 344, + "wilsonPrecisionLower95": 0.9523 + }, + "threshold": 0.5 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.954, + "falseNegative": 7, + "falsePositive": 1, + "precision": 0.9881, + "predictedPositive": 84, + "recall": 0.9222, + "total": 1485, + "trueNegative": 1394, + "truePositive": 83, + "wilsonPrecisionLower95": 0.9356 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7183, + "falseNegative": 39, + "falsePositive": 1, + "precision": 0.9808, + "predictedPositive": 52, + "recall": 0.5667, + "total": 1485, + "trueNegative": 1394, + "truePositive": 51, + "wilsonPrecisionLower95": 0.8988 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.8481, + "falseNegative": 46, + "falsePositive": 2, + "precision": 0.9853, + "predictedPositive": 136, + "recall": 0.7444, + "total": 2970, + "trueNegative": 2788, + "truePositive": 134, + "wilsonPrecisionLower95": 0.948 + }, + "threshold": 0.5 + }, + "followUpReminder": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8328, + "falseNegative": 48, + "falsePositive": 5, + "precision": 0.9635, + "predictedPositive": 137, + "recall": 0.7333, + "total": 1485, + "trueNegative": 1300, + "truePositive": 132, + "wilsonPrecisionLower95": 0.9174 + }, + "threshold": 0.91 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6879, + "falseNegative": 83, + "falsePositive": 5, + "precision": 0.951, + "predictedPositive": 102, + "recall": 0.5389, + "total": 1485, + "trueNegative": 1300, + "truePositive": 97, + "wilsonPrecisionLower95": 0.8903 + }, + "threshold": 0.765 + } + }, + "metrics": { + "f1": 0.6243, + "falseNegative": 193, + "falsePositive": 8, + "precision": 0.9543, + "predictedPositive": 175, + "recall": 0.4639, + "total": 2970, + "trueNegative": 2602, + "truePositive": 167, + "wilsonPrecisionLower95": 0.9124 + }, + "threshold": 0.905 + }, + "invitation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9059, + "falseNegative": 13, + "falsePositive": 3, + "precision": 0.9625, + "predictedPositive": 80, + "recall": 0.8556, + "total": 1485, + "trueNegative": 1392, + "truePositive": 77, + "wilsonPrecisionLower95": 0.8955 + }, + "threshold": 0.86 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.6113, + "falseNegative": 99, + "falsePositive": 4, + "precision": 0.9529, + "predictedPositive": 85, + "recall": 0.45, + "total": 2970, + "trueNegative": 2786, + "truePositive": 81, + "wilsonPrecisionLower95": 0.8852 + }, + "threshold": 0.855 + }, + "question": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8505, + "falseNegative": 76, + "falsePositive": 8, + "precision": 0.9676, + "predictedPositive": 247, + "recall": 0.7587, + "total": 1485, + "trueNegative": 1162, + "truePositive": 239, + "wilsonPrecisionLower95": 0.9374 + }, + "threshold": 0.705 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7308, + "falseNegative": 129, + "falsePositive": 8, + "precision": 0.9588, + "predictedPositive": 194, + "recall": 0.5905, + "total": 1485, + "trueNegative": 1162, + "truePositive": 186, + "wilsonPrecisionLower95": 0.9208 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.7813, + "falseNegative": 212, + "falsePositive": 22, + "precision": 0.95, + "predictedPositive": 440, + "recall": 0.6635, + "total": 2970, + "trueNegative": 2318, + "truePositive": 418, + "wilsonPrecisionLower95": 0.9255 + }, + "threshold": 0.57 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9629, + "falseNegative": 23, + "falsePositive": 48, + "precision": 0.9505, + "predictedPositive": 970, + "recall": 0.9757, + "total": 1485, + "trueNegative": 492, + "truePositive": 922, + "wilsonPrecisionLower95": 0.935 + }, + "threshold": 0.62 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8526, + "falseNegative": 216, + "falsePositive": 36, + "precision": 0.9529, + "predictedPositive": 765, + "recall": 0.7714, + "total": 1485, + "trueNegative": 504, + "truePositive": 729, + "wilsonPrecisionLower95": 0.9355 + }, + "threshold": 0.84 + } + }, + "metrics": { + "f1": 0.9224, + "falseNegative": 197, + "falsePositive": 88, + "precision": 0.9506, + "predictedPositive": 1781, + "recall": 0.8958, + "total": 2970, + "trueNegative": 992, + "truePositive": 1693, + "wilsonPrecisionLower95": 0.9395 + }, + "threshold": 0.775 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9783, + "falseNegative": 0, + "falsePositive": 2, + "precision": 0.9574, + "predictedPositive": 47, + "recall": 1.0, + "total": 1485, + "trueNegative": 1438, + "truePositive": 45, + "wilsonPrecisionLower95": 0.8575 + }, + "threshold": 0.765 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9778, + "falseNegative": 1, + "falsePositive": 1, + "precision": 0.9778, + "predictedPositive": 45, + "recall": 0.9778, + "total": 1485, + "trueNegative": 1439, + "truePositive": 44, + "wilsonPrecisionLower95": 0.8843 + }, + "threshold": 0.855 + } + }, + "metrics": { + "f1": 0.978, + "falseNegative": 1, + "falsePositive": 3, + "precision": 0.9674, + "predictedPositive": 92, + "recall": 0.9889, + "total": 2970, + "trueNegative": 2877, + "truePositive": 89, + "wilsonPrecisionLower95": 0.9085 + }, + "threshold": 0.855 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8541, + "falseNegative": 82, + "falsePositive": 13, + "precision": 0.9553, + "predictedPositive": 291, + "recall": 0.7722, + "total": 1485, + "trueNegative": 1112, + "truePositive": 278, + "wilsonPrecisionLower95": 0.9251 + }, + "threshold": 0.84 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.5418, + "falseNegative": 224, + "falsePositive": 6, + "precision": 0.9577, + "predictedPositive": 142, + "recall": 0.3778, + "total": 1485, + "trueNegative": 1119, + "truePositive": 136, + "wilsonPrecisionLower95": 0.9109 + }, + "threshold": 0.91 + } + }, + "metrics": { + "f1": 0.7246, + "falseNegative": 299, + "falsePositive": 21, + "precision": 0.9525, + "predictedPositive": 442, + "recall": 0.5847, + "total": 2970, + "trueNegative": 2229, + "truePositive": 421, + "wilsonPrecisionLower95": 0.9285 + }, + "threshold": 0.88 + } + }, + "trainingRecords": 15345, + "validation": { + "intentsMeeting95Precision": 9, + "macroF1": 0.7578, + "macroPrecision": 0.9691, + "macroRecall": 0.668, + "minimumPrecision": 0.9516, + "perIntent": { + "blessing": { + "f1": 0.2524, + "falseNegative": 77, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 13, + "recall": 0.1444, + "total": 2970, + "trueNegative": 2880, + "truePositive": 13, + "wilsonPrecisionLower95": 0.7719 + }, + "complaint": { + "f1": 0.9619, + "falseNegative": 19, + "falsePositive": 8, + "precision": 0.9771, + "predictedPositive": 349, + "recall": 0.9472, + "total": 2970, + "trueNegative": 2602, + "truePositive": 341, + "wilsonPrecisionLower95": 0.9554 + }, + "confirmationDecision": { + "f1": 0.8481, + "falseNegative": 46, + "falsePositive": 2, + "precision": 0.9853, + "predictedPositive": 136, + "recall": 0.7444, + "total": 2970, + "trueNegative": 2788, + "truePositive": 134, + "wilsonPrecisionLower95": 0.948 + }, + "followUpReminder": { + "f1": 0.7646, + "falseNegative": 131, + "falsePositive": 10, + "precision": 0.9582, + "predictedPositive": 239, + "recall": 0.6361, + "total": 2970, + "trueNegative": 2600, + "truePositive": 229, + "wilsonPrecisionLower95": 0.9247 + }, + "invitation": { + "f1": 0.5923, + "falseNegative": 103, + "falsePositive": 3, + "precision": 0.9625, + "predictedPositive": 80, + "recall": 0.4278, + "total": 2970, + "trueNegative": 2787, + "truePositive": 77, + "wilsonPrecisionLower95": 0.8955 + }, + "question": { + "f1": 0.7937, + "falseNegative": 205, + "falsePositive": 16, + "precision": 0.9637, + "predictedPositive": 441, + "recall": 0.6746, + "total": 2970, + "trueNegative": 2324, + "truePositive": 425, + "wilsonPrecisionLower95": 0.9419 + }, + "replyableMessage": { + "f1": 0.9109, + "falseNegative": 239, + "falsePositive": 84, + "precision": 0.9516, + "predictedPositive": 1735, + "recall": 0.8735, + "total": 2970, + "trueNegative": 996, + "truePositive": 1651, + "wilsonPrecisionLower95": 0.9404 + }, + "scheduleNegotiation": { + "f1": 0.978, + "falseNegative": 1, + "falsePositive": 3, + "precision": 0.9674, + "predictedPositive": 92, + "recall": 0.9889, + "total": 2970, + "trueNegative": 2877, + "truePositive": 89, + "wilsonPrecisionLower95": 0.9085 + }, + "task": { + "f1": 0.7181, + "falseNegative": 306, + "falsePositive": 19, + "precision": 0.9561, + "predictedPositive": 433, + "recall": 0.575, + "total": 2970, + "trueNegative": 2231, + "truePositive": 414, + "wilsonPrecisionLower95": 0.9325 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 1e-05, + "augmentation": "none", + "char_max": 6, + "char_min": 2, + "external_weight": 0.6100000000000001, + "hard_example_weight": 1.2, + "l1_ratio": 0.05, + "max_char_features": 55000, + "max_word_features": 25000, + "min_df": 2, + "pseudo_weight": 0.195, + "round": 3, + "word_max": 2 + }, + "featureCount": 66451, + "pseudoLabelCounts": {}, + "round": 3, + "selectionObjective": 0.8348, + "syntheticRegression": { + "intentsMeeting95Precision": 4, + "macroF1": 0.7343, + "macroPrecision": 0.9403, + "macroRecall": 0.6672, + "minimumPrecision": 0.809, + "perIntent": { + "blessing": { + "f1": 0.0645, + "falseNegative": 87, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 3, + "recall": 0.0333, + "total": 2970, + "trueNegative": 2880, + "truePositive": 3, + "wilsonPrecisionLower95": 0.4385 + }, + "complaint": { + "f1": 0.9816, + "falseNegative": 13, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 347, + "recall": 0.9639, + "total": 2970, + "trueNegative": 2610, + "truePositive": 347, + "wilsonPrecisionLower95": 0.9891 + }, + "confirmationDecision": { + "f1": 0.8045, + "falseNegative": 36, + "falsePositive": 34, + "precision": 0.809, + "predictedPositive": 178, + "recall": 0.8, + "total": 2970, + "trueNegative": 2756, + "truePositive": 144, + "wilsonPrecisionLower95": 0.745 + }, + "followUpReminder": { + "f1": 0.7967, + "falseNegative": 121, + "falsePositive": 1, + "precision": 0.9958, + "predictedPositive": 240, + "recall": 0.6639, + "total": 2970, + "trueNegative": 2609, + "truePositive": 239, + "wilsonPrecisionLower95": 0.9768 + }, + "invitation": { + "f1": 0.5985, + "falseNegative": 101, + "falsePositive": 5, + "precision": 0.9405, + "predictedPositive": 84, + "recall": 0.4389, + "total": 2970, + "trueNegative": 2785, + "truePositive": 79, + "wilsonPrecisionLower95": 0.8681 + }, + "question": { + "f1": 0.8643, + "falseNegative": 149, + "falsePositive": 2, + "precision": 0.9959, + "predictedPositive": 483, + "recall": 0.7635, + "total": 2970, + "trueNegative": 2338, + "truePositive": 481, + "wilsonPrecisionLower95": 0.985 + }, + "replyableMessage": { + "f1": 0.931, + "falseNegative": 189, + "falsePositive": 63, + "precision": 0.9643, + "predictedPositive": 1764, + "recall": 0.9, + "total": 2970, + "trueNegative": 1017, + "truePositive": 1701, + "wilsonPrecisionLower95": 0.9546 + }, + "scheduleNegotiation": { + "f1": 0.8222, + "falseNegative": 16, + "falsePositive": 16, + "precision": 0.8222, + "predictedPositive": 90, + "recall": 0.8222, + "total": 2970, + "trueNegative": 2864, + "truePositive": 74, + "wilsonPrecisionLower95": 0.7306 + }, + "task": { + "f1": 0.7452, + "falseNegative": 274, + "falsePositive": 31, + "precision": 0.935, + "predictedPositive": 477, + "recall": 0.6194, + "total": 2970, + "trueNegative": 2219, + "truePositive": 446, + "wilsonPrecisionLower95": 0.9092 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 180, + "recall": 1.0, + "total": 1485, + "trueNegative": 1305, + "truePositive": 180, + "wilsonPrecisionLower95": 0.9791 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9371, + "falseNegative": 16, + "falsePositive": 6, + "precision": 0.9647, + "predictedPositive": 170, + "recall": 0.9111, + "total": 1485, + "trueNegative": 1299, + "truePositive": 164, + "wilsonPrecisionLower95": 0.9251 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.969, + "falseNegative": 16, + "falsePositive": 6, + "precision": 0.9829, + "predictedPositive": 350, + "recall": 0.9556, + "total": 2970, + "trueNegative": 2604, + "truePositive": 344, + "wilsonPrecisionLower95": 0.9631 + }, + "threshold": 0.5 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9727, + "falseNegative": 1, + "falsePositive": 4, + "precision": 0.957, + "predictedPositive": 93, + "recall": 0.9889, + "total": 1485, + "trueNegative": 1391, + "truePositive": 89, + "wilsonPrecisionLower95": 0.8946 + }, + "threshold": 0.585 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7815, + "falseNegative": 31, + "falsePositive": 2, + "precision": 0.9672, + "predictedPositive": 61, + "recall": 0.6556, + "total": 1485, + "trueNegative": 1393, + "truePositive": 59, + "wilsonPrecisionLower95": 0.8881 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.8701, + "falseNegative": 36, + "falsePositive": 7, + "precision": 0.9536, + "predictedPositive": 151, + "recall": 0.8, + "total": 2970, + "trueNegative": 2783, + "truePositive": 144, + "wilsonPrecisionLower95": 0.9074 + }, + "threshold": 0.555 + }, + "followUpReminder": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8454, + "falseNegative": 46, + "falsePositive": 3, + "precision": 0.9781, + "predictedPositive": 137, + "recall": 0.7444, + "total": 1485, + "trueNegative": 1302, + "truePositive": 134, + "wilsonPrecisionLower95": 0.9376 + }, + "threshold": 0.97 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7525, + "falseNegative": 69, + "falsePositive": 4, + "precision": 0.9652, + "predictedPositive": 115, + "recall": 0.6167, + "total": 1485, + "trueNegative": 1301, + "truePositive": 111, + "wilsonPrecisionLower95": 0.914 + }, + "threshold": 0.785 + } + }, + "metrics": { + "f1": 0.703, + "falseNegative": 160, + "falsePositive": 9, + "precision": 0.9569, + "predictedPositive": 209, + "recall": 0.5556, + "total": 2970, + "trueNegative": 2601, + "truePositive": 200, + "wilsonPrecisionLower95": 0.9202 + }, + "threshold": 0.955 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7974, + "falseNegative": 29, + "falsePositive": 2, + "precision": 0.9683, + "predictedPositive": 63, + "recall": 0.6778, + "total": 1485, + "trueNegative": 1393, + "truePositive": 61, + "wilsonPrecisionLower95": 0.8914 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8259, + "falseNegative": 85, + "falsePositive": 12, + "precision": 0.9504, + "predictedPositive": 242, + "recall": 0.7302, + "total": 1485, + "trueNegative": 1158, + "truePositive": 230, + "wilsonPrecisionLower95": 0.9153 + }, + "threshold": 0.865 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8194, + "falseNegative": 95, + "falsePositive": 2, + "precision": 0.991, + "predictedPositive": 222, + "recall": 0.6984, + "total": 1485, + "trueNegative": 1168, + "truePositive": 220, + "wilsonPrecisionLower95": 0.9678 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.7959, + "falseNegative": 199, + "falsePositive": 22, + "precision": 0.9514, + "predictedPositive": 453, + "recall": 0.6841, + "total": 2970, + "trueNegative": 2318, + "truePositive": 431, + "wilsonPrecisionLower95": 0.9276 + }, + "threshold": 0.74 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9496, + "falseNegative": 50, + "falsePositive": 45, + "precision": 0.9521, + "predictedPositive": 940, + "recall": 0.9471, + "total": 1485, + "trueNegative": 495, + "truePositive": 895, + "wilsonPrecisionLower95": 0.9365 + }, + "threshold": 0.705 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9209, + "falseNegative": 101, + "falsePositive": 44, + "precision": 0.9505, + "predictedPositive": 888, + "recall": 0.8931, + "total": 1485, + "trueNegative": 496, + "truePositive": 844, + "wilsonPrecisionLower95": 0.9341 + }, + "threshold": 0.78 + } + }, + "metrics": { + "f1": 0.9318, + "falseNegative": 163, + "falsePositive": 90, + "precision": 0.9505, + "predictedPositive": 1817, + "recall": 0.9138, + "total": 2970, + "trueNegative": 990, + "truePositive": 1727, + "wilsonPrecisionLower95": 0.9395 + }, + "threshold": 0.77 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 45, + "recall": 1.0, + "total": 1485, + "trueNegative": 1440, + "truePositive": 45, + "wilsonPrecisionLower95": 0.9213 + }, + "threshold": 0.745 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9783, + "falseNegative": 0, + "falsePositive": 2, + "precision": 0.9574, + "predictedPositive": 47, + "recall": 1.0, + "total": 1485, + "trueNegative": 1438, + "truePositive": 45, + "wilsonPrecisionLower95": 0.8575 + }, + "threshold": 0.93 + } + }, + "metrics": { + "f1": 0.9778, + "falseNegative": 2, + "falsePositive": 2, + "precision": 0.9778, + "predictedPositive": 90, + "recall": 0.9778, + "total": 2970, + "trueNegative": 2878, + "truePositive": 88, + "wilsonPrecisionLower95": 0.9226 + }, + "threshold": 0.93 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8488, + "falseNegative": 85, + "falsePositive": 13, + "precision": 0.9549, + "predictedPositive": 288, + "recall": 0.7639, + "total": 1485, + "trueNegative": 1112, + "truePositive": 275, + "wilsonPrecisionLower95": 0.9243 + }, + "threshold": 0.865 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8006, + "falseNegative": 111, + "falsePositive": 13, + "precision": 0.9504, + "predictedPositive": 262, + "recall": 0.6917, + "total": 1485, + "trueNegative": 1112, + "truePositive": 249, + "wilsonPrecisionLower95": 0.917 + }, + "threshold": 0.8 + } + }, + "metrics": { + "f1": 0.8301, + "falseNegative": 190, + "falsePositive": 27, + "precision": 0.9515, + "predictedPositive": 557, + "recall": 0.7361, + "total": 2970, + "trueNegative": 2223, + "truePositive": 530, + "wilsonPrecisionLower95": 0.9304 + }, + "threshold": 0.82 + } + }, + "trainingRecords": 15345, + "validation": { + "intentsMeeting95Precision": 8, + "macroF1": 0.7723, + "macroPrecision": 0.9713, + "macroRecall": 0.7011, + "minimumPrecision": 0.9513, + "perIntent": { + "blessing": { + "f1": 0.1443, + "falseNegative": 83, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 7, + "recall": 0.0778, + "total": 2970, + "trueNegative": 2880, + "truePositive": 7, + "wilsonPrecisionLower95": 0.6457 + }, + "complaint": { + "f1": 0.969, + "falseNegative": 16, + "falsePositive": 6, + "precision": 0.9829, + "predictedPositive": 350, + "recall": 0.9556, + "total": 2970, + "trueNegative": 2604, + "truePositive": 344, + "wilsonPrecisionLower95": 0.9631 + }, + "confirmationDecision": { + "f1": 0.8862, + "falseNegative": 32, + "falsePositive": 6, + "precision": 0.961, + "predictedPositive": 154, + "recall": 0.8222, + "total": 2970, + "trueNegative": 2784, + "truePositive": 148, + "wilsonPrecisionLower95": 0.9176 + }, + "followUpReminder": { + "f1": 0.8007, + "falseNegative": 115, + "falsePositive": 7, + "precision": 0.9722, + "predictedPositive": 252, + "recall": 0.6806, + "total": 2970, + "trueNegative": 2603, + "truePositive": 245, + "wilsonPrecisionLower95": 0.9438 + }, + "invitation": { + "f1": 0.5781, + "falseNegative": 106, + "falsePositive": 2, + "precision": 0.9737, + "predictedPositive": 76, + "recall": 0.4111, + "total": 2970, + "trueNegative": 2788, + "truePositive": 74, + "wilsonPrecisionLower95": 0.909 + }, + "question": { + "f1": 0.8227, + "falseNegative": 180, + "falsePositive": 14, + "precision": 0.9698, + "predictedPositive": 464, + "recall": 0.7143, + "total": 2970, + "trueNegative": 2326, + "truePositive": 450, + "wilsonPrecisionLower95": 0.95 + }, + "replyableMessage": { + "f1": 0.9354, + "falseNegative": 151, + "falsePositive": 89, + "precision": 0.9513, + "predictedPositive": 1828, + "recall": 0.9201, + "total": 2970, + "trueNegative": 991, + "truePositive": 1739, + "wilsonPrecisionLower95": 0.9405 + }, + "scheduleNegotiation": { + "f1": 0.989, + "falseNegative": 0, + "falsePositive": 2, + "precision": 0.9783, + "predictedPositive": 92, + "recall": 1.0, + "total": 2970, + "trueNegative": 2878, + "truePositive": 90, + "wilsonPrecisionLower95": 0.9242 + }, + "task": { + "f1": 0.8252, + "falseNegative": 196, + "falsePositive": 26, + "precision": 0.9527, + "predictedPositive": 550, + "recall": 0.7278, + "total": 2970, + "trueNegative": 2224, + "truePositive": 524, + "wilsonPrecisionLower95": 0.9316 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 8e-06, + "augmentation": "none", + "char_max": 6, + "char_min": 3, + "external_weight": 0.63, + "hard_example_weight": 1.2, + "l1_ratio": 0.05, + "max_char_features": 60000, + "max_word_features": 30000, + "min_df": 2, + "pseudo_weight": 0.21, + "round": 4, + "word_max": 3 + }, + "featureCount": 79458, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 0 + }, + "complaint": { + "negative": 97, + "positive": 0 + }, + "confirmationDecision": { + "negative": 51, + "positive": 2 + }, + "followUpReminder": { + "negative": 250, + "positive": 0 + }, + "invitation": { + "negative": 250, + "positive": 0 + }, + "question": { + "negative": 36, + "positive": 1 + }, + "replyableMessage": { + "negative": 3, + "positive": 130 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 0 + }, + "task": { + "negative": 85, + "positive": 0 + } + }, + "round": 4, + "selectionObjective": 0.796, + "syntheticRegression": { + "intentsMeeting95Precision": 6, + "macroF1": 0.6978, + "macroPrecision": 0.9611, + "macroRecall": 0.5994, + "minimumPrecision": 0.8074, + "perIntent": { + "blessing": { + "f1": 0.0645, + "falseNegative": 87, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 3, + "recall": 0.0333, + "total": 2970, + "trueNegative": 2880, + "truePositive": 3, + "wilsonPrecisionLower95": 0.4385 + }, + "complaint": { + "f1": 0.967, + "falseNegative": 23, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 337, + "recall": 0.9361, + "total": 2970, + "trueNegative": 2610, + "truePositive": 337, + "wilsonPrecisionLower95": 0.9887 + }, + "confirmationDecision": { + "f1": 0.6921, + "falseNegative": 71, + "falsePositive": 26, + "precision": 0.8074, + "predictedPositive": 135, + "recall": 0.6056, + "total": 2970, + "trueNegative": 2764, + "truePositive": 109, + "wilsonPrecisionLower95": 0.7328 + }, + "followUpReminder": { + "f1": 0.7734, + "falseNegative": 133, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 227, + "recall": 0.6306, + "total": 2970, + "trueNegative": 2610, + "truePositive": 227, + "wilsonPrecisionLower95": 0.9834 + }, + "invitation": { + "f1": 0.5837, + "falseNegative": 105, + "falsePositive": 2, + "precision": 0.974, + "predictedPositive": 77, + "recall": 0.4167, + "total": 2970, + "trueNegative": 2788, + "truePositive": 75, + "wilsonPrecisionLower95": 0.9102 + }, + "question": { + "f1": 0.7613, + "falseNegative": 241, + "falsePositive": 3, + "precision": 0.9923, + "predictedPositive": 392, + "recall": 0.6175, + "total": 2970, + "trueNegative": 2337, + "truePositive": 389, + "wilsonPrecisionLower95": 0.9777 + }, + "replyableMessage": { + "f1": 0.8816, + "falseNegative": 353, + "falsePositive": 60, + "precision": 0.9624, + "predictedPositive": 1597, + "recall": 0.8132, + "total": 2970, + "trueNegative": 1020, + "truePositive": 1537, + "wilsonPrecisionLower95": 0.9519 + }, + "scheduleNegotiation": { + "f1": 0.9222, + "falseNegative": 13, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 77, + "recall": 0.8556, + "total": 2970, + "trueNegative": 2880, + "truePositive": 77, + "wilsonPrecisionLower95": 0.9525 + }, + "task": { + "f1": 0.6346, + "falseNegative": 370, + "falsePositive": 33, + "precision": 0.9138, + "predictedPositive": 383, + "recall": 0.4861, + "total": 2970, + "trueNegative": 2217, + "truePositive": 350, + "wilsonPrecisionLower95": 0.8815 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9972, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 179, + "recall": 0.9944, + "total": 1485, + "trueNegative": 1305, + "truePositive": 179, + "wilsonPrecisionLower95": 0.979 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9253, + "falseNegative": 19, + "falsePositive": 7, + "precision": 0.9583, + "predictedPositive": 168, + "recall": 0.8944, + "total": 1485, + "trueNegative": 1298, + "truePositive": 161, + "wilsonPrecisionLower95": 0.9165 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.9618, + "falseNegative": 20, + "falsePositive": 7, + "precision": 0.9798, + "predictedPositive": 347, + "recall": 0.9444, + "total": 2970, + "trueNegative": 2603, + "truePositive": 340, + "wilsonPrecisionLower95": 0.959 + }, + "threshold": 0.5 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8765, + "falseNegative": 19, + "falsePositive": 1, + "precision": 0.9861, + "predictedPositive": 72, + "recall": 0.7889, + "total": 1485, + "trueNegative": 1394, + "truePositive": 71, + "wilsonPrecisionLower95": 0.9254 + }, + "threshold": 0.945 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6569, + "falseNegative": 45, + "falsePositive": 2, + "precision": 0.9574, + "predictedPositive": 47, + "recall": 0.5, + "total": 1485, + "trueNegative": 1393, + "truePositive": 45, + "wilsonPrecisionLower95": 0.8575 + }, + "threshold": 0.64 + } + }, + "metrics": { + "f1": 0.7108, + "falseNegative": 78, + "falsePositive": 5, + "precision": 0.9533, + "predictedPositive": 107, + "recall": 0.5667, + "total": 2970, + "trueNegative": 2785, + "truePositive": 102, + "wilsonPrecisionLower95": 0.8952 + }, + "threshold": 0.91 + }, + "followUpReminder": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8, + "falseNegative": 56, + "falsePositive": 6, + "precision": 0.9538, + "predictedPositive": 130, + "recall": 0.6889, + "total": 1485, + "trueNegative": 1299, + "truePositive": 124, + "wilsonPrecisionLower95": 0.903 + }, + "threshold": 0.98 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6165, + "falseNegative": 98, + "falsePositive": 4, + "precision": 0.9535, + "predictedPositive": 86, + "recall": 0.4556, + "total": 1485, + "trueNegative": 1301, + "truePositive": 82, + "wilsonPrecisionLower95": 0.8864 + }, + "threshold": 0.825 + } + }, + "metrics": { + "f1": 0.5912, + "falseNegative": 206, + "falsePositive": 7, + "precision": 0.9565, + "predictedPositive": 161, + "recall": 0.4278, + "total": 2970, + "trueNegative": 2603, + "truePositive": 154, + "wilsonPrecisionLower95": 0.913 + }, + "threshold": 0.98 + }, + "invitation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8302, + "falseNegative": 24, + "falsePositive": 3, + "precision": 0.9565, + "predictedPositive": 69, + "recall": 0.7333, + "total": 1485, + "trueNegative": 1392, + "truePositive": 66, + "wilsonPrecisionLower95": 0.8798 + }, + "threshold": 0.98 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.536, + "falseNegative": 113, + "falsePositive": 3, + "precision": 0.9571, + "predictedPositive": 70, + "recall": 0.3722, + "total": 2970, + "trueNegative": 2787, + "truePositive": 67, + "wilsonPrecisionLower95": 0.8814 + }, + "threshold": 0.995 + }, + "question": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.7813, + "falseNegative": 106, + "falsePositive": 11, + "precision": 0.95, + "predictedPositive": 220, + "recall": 0.6635, + "total": 1485, + "trueNegative": 1159, + "truePositive": 209, + "wilsonPrecisionLower95": 0.9127 + }, + "threshold": 0.865 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6844, + "falseNegative": 148, + "falsePositive": 6, + "precision": 0.9653, + "predictedPositive": 173, + "recall": 0.5302, + "total": 1485, + "trueNegative": 1164, + "truePositive": 167, + "wilsonPrecisionLower95": 0.9264 + }, + "threshold": 0.63 + } + }, + "metrics": { + "f1": 0.75, + "falseNegative": 240, + "falsePositive": 20, + "precision": 0.9512, + "predictedPositive": 410, + "recall": 0.619, + "total": 2970, + "trueNegative": 2320, + "truePositive": 390, + "wilsonPrecisionLower95": 0.9259 + }, + "threshold": 0.74 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9441, + "falseNegative": 59, + "falsePositive": 46, + "precision": 0.9506, + "predictedPositive": 932, + "recall": 0.9376, + "total": 1485, + "trueNegative": 494, + "truePositive": 886, + "wilsonPrecisionLower95": 0.9348 + }, + "threshold": 0.76 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8342, + "falseNegative": 243, + "falsePositive": 36, + "precision": 0.9512, + "predictedPositive": 738, + "recall": 0.7429, + "total": 1485, + "trueNegative": 504, + "truePositive": 702, + "wilsonPrecisionLower95": 0.9332 + }, + "threshold": 0.885 + } + }, + "metrics": { + "f1": 0.9056, + "falseNegative": 255, + "falsePositive": 86, + "precision": 0.95, + "predictedPositive": 1721, + "recall": 0.8651, + "total": 2970, + "trueNegative": 994, + "truePositive": 1635, + "wilsonPrecisionLower95": 0.9387 + }, + "threshold": 0.855 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 45, + "recall": 1.0, + "total": 1485, + "trueNegative": 1440, + "truePositive": 45, + "wilsonPrecisionLower95": 0.9213 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9318, + "falseNegative": 4, + "falsePositive": 2, + "precision": 0.9535, + "predictedPositive": 43, + "recall": 0.9111, + "total": 1485, + "trueNegative": 1438, + "truePositive": 41, + "wilsonPrecisionLower95": 0.8454 + }, + "threshold": 0.945 + } + }, + "metrics": { + "f1": 0.9727, + "falseNegative": 1, + "falsePositive": 4, + "precision": 0.957, + "predictedPositive": 93, + "recall": 0.9889, + "total": 2970, + "trueNegative": 2876, + "truePositive": 89, + "wilsonPrecisionLower95": 0.8946 + }, + "threshold": 0.62 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.851, + "falseNegative": 83, + "falsePositive": 14, + "precision": 0.9519, + "predictedPositive": 291, + "recall": 0.7694, + "total": 1485, + "trueNegative": 1111, + "truePositive": 277, + "wilsonPrecisionLower95": 0.9209 + }, + "threshold": 0.87 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.4886, + "falseNegative": 242, + "falsePositive": 5, + "precision": 0.9593, + "predictedPositive": 123, + "recall": 0.3278, + "total": 1485, + "trueNegative": 1120, + "truePositive": 118, + "wilsonPrecisionLower95": 0.9084 + }, + "threshold": 0.935 + } + }, + "metrics": { + "f1": 0.6978, + "falseNegative": 324, + "falsePositive": 19, + "precision": 0.9542, + "predictedPositive": 415, + "recall": 0.55, + "total": 2970, + "trueNegative": 2231, + "truePositive": 396, + "wilsonPrecisionLower95": 0.9296 + }, + "threshold": 0.915 + } + }, + "trainingRecords": 15345, + "validation": { + "intentsMeeting95Precision": 9, + "macroF1": 0.7192, + "macroPrecision": 0.9671, + "macroRecall": 0.62, + "minimumPrecision": 0.9509, + "perIntent": { + "blessing": { + "f1": 0.2, + "falseNegative": 80, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 10, + "recall": 0.1111, + "total": 2970, + "trueNegative": 2880, + "truePositive": 10, + "wilsonPrecisionLower95": 0.7225 + }, + "complaint": { + "f1": 0.9618, + "falseNegative": 20, + "falsePositive": 7, + "precision": 0.9798, + "predictedPositive": 347, + "recall": 0.9444, + "total": 2970, + "trueNegative": 2603, + "truePositive": 340, + "wilsonPrecisionLower95": 0.959 + }, + "confirmationDecision": { + "f1": 0.7759, + "falseNegative": 64, + "falsePositive": 3, + "precision": 0.9748, + "predictedPositive": 119, + "recall": 0.6444, + "total": 2970, + "trueNegative": 2787, + "truePositive": 116, + "wilsonPrecisionLower95": 0.9285 + }, + "followUpReminder": { + "f1": 0.7153, + "falseNegative": 154, + "falsePositive": 10, + "precision": 0.9537, + "predictedPositive": 216, + "recall": 0.5722, + "total": 2970, + "trueNegative": 2600, + "truePositive": 206, + "wilsonPrecisionLower95": 0.9169 + }, + "invitation": { + "f1": 0.5301, + "falseNegative": 114, + "falsePositive": 3, + "precision": 0.9565, + "predictedPositive": 69, + "recall": 0.3667, + "total": 2970, + "trueNegative": 2787, + "truePositive": 66, + "wilsonPrecisionLower95": 0.8798 + }, + "question": { + "f1": 0.7351, + "falseNegative": 254, + "falsePositive": 17, + "precision": 0.9567, + "predictedPositive": 393, + "recall": 0.5968, + "total": 2970, + "trueNegative": 2323, + "truePositive": 376, + "wilsonPrecisionLower95": 0.9318 + }, + "replyableMessage": { + "f1": 0.8921, + "falseNegative": 302, + "falsePositive": 82, + "precision": 0.9509, + "predictedPositive": 1670, + "recall": 0.8402, + "total": 2970, + "trueNegative": 998, + "truePositive": 1588, + "wilsonPrecisionLower95": 0.9395 + }, + "scheduleNegotiation": { + "f1": 0.9663, + "falseNegative": 4, + "falsePositive": 2, + "precision": 0.9773, + "predictedPositive": 88, + "recall": 0.9556, + "total": 2970, + "trueNegative": 2878, + "truePositive": 86, + "wilsonPrecisionLower95": 0.9209 + }, + "task": { + "f1": 0.6966, + "falseNegative": 325, + "falsePositive": 19, + "precision": 0.9541, + "predictedPositive": 414, + "recall": 0.5486, + "total": 2970, + "trueNegative": 2231, + "truePositive": 395, + "wilsonPrecisionLower95": 0.9294 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 1.2e-05, + "augmentation": "none", + "char_max": 5, + "char_min": 2, + "external_weight": 0.65, + "hard_example_weight": 1.4, + "l1_ratio": 0.1, + "max_char_features": 60000, + "max_word_features": 32000, + "min_df": 1, + "pseudo_weight": 0.22499999999999998, + "round": 5, + "word_max": 3 + }, + "featureCount": 92000, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 0 + }, + "complaint": { + "negative": 88, + "positive": 0 + }, + "confirmationDecision": { + "negative": 96, + "positive": 3 + }, + "followUpReminder": { + "negative": 250, + "positive": 0 + }, + "invitation": { + "negative": 250, + "positive": 0 + }, + "question": { + "negative": 40, + "positive": 1 + }, + "replyableMessage": { + "negative": 3, + "positive": 150 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 0 + }, + "task": { + "negative": 97, + "positive": 0 + } + }, + "round": 5, + "selectionObjective": 0.7771, + "syntheticRegression": { + "intentsMeeting95Precision": 5, + "macroF1": 0.6709, + "macroPrecision": 0.9586, + "macroRecall": 0.5803, + "minimumPrecision": 0.8588, + "perIntent": { + "blessing": { + "f1": 0.022, + "falseNegative": 89, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 1, + "recall": 0.0111, + "total": 2970, + "trueNegative": 2880, + "truePositive": 1, + "wilsonPrecisionLower95": 0.2065 + }, + "complaint": { + "f1": 0.967, + "falseNegative": 23, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 337, + "recall": 0.9361, + "total": 2970, + "trueNegative": 2610, + "truePositive": 337, + "wilsonPrecisionLower95": 0.9887 + }, + "confirmationDecision": { + "f1": 0.8343, + "falseNegative": 34, + "falsePositive": 24, + "precision": 0.8588, + "predictedPositive": 170, + "recall": 0.8111, + "total": 2970, + "trueNegative": 2766, + "truePositive": 146, + "wilsonPrecisionLower95": 0.7985 + }, + "followUpReminder": { + "f1": 0.7173, + "falseNegative": 157, + "falsePositive": 3, + "precision": 0.9854, + "predictedPositive": 206, + "recall": 0.5639, + "total": 2970, + "trueNegative": 2607, + "truePositive": 203, + "wilsonPrecisionLower95": 0.9581 + }, + "invitation": { + "f1": 0.3612, + "falseNegative": 139, + "falsePositive": 6, + "precision": 0.8723, + "predictedPositive": 47, + "recall": 0.2278, + "total": 2970, + "trueNegative": 2784, + "truePositive": 41, + "wilsonPrecisionLower95": 0.7483 + }, + "question": { + "f1": 0.8336, + "falseNegative": 179, + "falsePositive": 1, + "precision": 0.9978, + "predictedPositive": 452, + "recall": 0.7159, + "total": 2970, + "trueNegative": 2339, + "truePositive": 451, + "wilsonPrecisionLower95": 0.9876 + }, + "replyableMessage": { + "f1": 0.912, + "falseNegative": 257, + "falsePositive": 58, + "precision": 0.9657, + "predictedPositive": 1691, + "recall": 0.864, + "total": 2970, + "trueNegative": 1022, + "truePositive": 1633, + "wilsonPrecisionLower95": 0.9559 + }, + "scheduleNegotiation": { + "f1": 0.6364, + "falseNegative": 48, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 42, + "recall": 0.4667, + "total": 2970, + "trueNegative": 2880, + "truePositive": 42, + "wilsonPrecisionLower95": 0.9162 + }, + "task": { + "f1": 0.7542, + "falseNegative": 269, + "falsePositive": 25, + "precision": 0.9475, + "predictedPositive": 476, + "recall": 0.6264, + "total": 2970, + "trueNegative": 2225, + "truePositive": 451, + "wilsonPrecisionLower95": 0.9236 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 180, + "recall": 1.0, + "total": 1485, + "trueNegative": 1305, + "truePositive": 180, + "wilsonPrecisionLower95": 0.9791 + }, + "threshold": 0.69 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9288, + "falseNegative": 17, + "falsePositive": 8, + "precision": 0.9532, + "predictedPositive": 171, + "recall": 0.9056, + "total": 1485, + "trueNegative": 1297, + "truePositive": 163, + "wilsonPrecisionLower95": 0.9104 + }, + "threshold": 0.51 + } + }, + "metrics": { + "f1": 0.9569, + "falseNegative": 16, + "falsePositive": 15, + "precision": 0.9582, + "predictedPositive": 359, + "recall": 0.9556, + "total": 2970, + "trueNegative": 2595, + "truePositive": 344, + "wilsonPrecisionLower95": 0.9322 + }, + "threshold": 0.5 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8589, + "falseNegative": 20, + "falsePositive": 3, + "precision": 0.9589, + "predictedPositive": 73, + "recall": 0.7778, + "total": 1485, + "trueNegative": 1392, + "truePositive": 70, + "wilsonPrecisionLower95": 0.886 + }, + "threshold": 0.57 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6763, + "falseNegative": 43, + "falsePositive": 2, + "precision": 0.9592, + "predictedPositive": 49, + "recall": 0.5222, + "total": 1485, + "trueNegative": 1393, + "truePositive": 47, + "wilsonPrecisionLower95": 0.8629 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.76, + "falseNegative": 66, + "falsePositive": 6, + "precision": 0.95, + "predictedPositive": 120, + "recall": 0.6333, + "total": 2970, + "trueNegative": 2784, + "truePositive": 114, + "wilsonPrecisionLower95": 0.8952 + }, + "threshold": 0.56 + }, + "followUpReminder": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8, + "falseNegative": 56, + "falsePositive": 6, + "precision": 0.9538, + "predictedPositive": 130, + "recall": 0.6889, + "total": 1485, + "trueNegative": 1299, + "truePositive": 124, + "wilsonPrecisionLower95": 0.903 + }, + "threshold": 0.96 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7708, + "falseNegative": 64, + "falsePositive": 5, + "precision": 0.9587, + "predictedPositive": 121, + "recall": 0.6444, + "total": 1485, + "trueNegative": 1300, + "truePositive": 116, + "wilsonPrecisionLower95": 0.9069 + }, + "threshold": 0.58 + } + }, + "metrics": { + "f1": 0.6892, + "falseNegative": 166, + "falsePositive": 9, + "precision": 0.9557, + "predictedPositive": 203, + "recall": 0.5389, + "total": 2970, + "trueNegative": 2601, + "truePositive": 194, + "wilsonPrecisionLower95": 0.9179 + }, + "threshold": 0.915 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.5203, + "falseNegative": 58, + "falsePositive": 1, + "precision": 0.9697, + "predictedPositive": 33, + "recall": 0.3556, + "total": 1485, + "trueNegative": 1394, + "truePositive": 32, + "wilsonPrecisionLower95": 0.8468 + }, + "threshold": 0.88 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8225, + "falseNegative": 88, + "falsePositive": 10, + "precision": 0.9578, + "predictedPositive": 237, + "recall": 0.7206, + "total": 1485, + "trueNegative": 1160, + "truePositive": 227, + "wilsonPrecisionLower95": 0.9241 + }, + "threshold": 0.84 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8388, + "falseNegative": 86, + "falsePositive": 2, + "precision": 0.9913, + "predictedPositive": 231, + "recall": 0.727, + "total": 1485, + "trueNegative": 1168, + "truePositive": 229, + "wilsonPrecisionLower95": 0.969 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.8591, + "falseNegative": 136, + "falsePositive": 26, + "precision": 0.95, + "predictedPositive": 520, + "recall": 0.7841, + "total": 2970, + "trueNegative": 2314, + "truePositive": 494, + "wilsonPrecisionLower95": 0.9277 + }, + "threshold": 0.505 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9541, + "falseNegative": 40, + "falsePositive": 47, + "precision": 0.9506, + "predictedPositive": 952, + "recall": 0.9577, + "total": 1485, + "trueNegative": 493, + "truePositive": 905, + "wilsonPrecisionLower95": 0.935 + }, + "threshold": 0.69 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8807, + "falseNegative": 170, + "falsePositive": 40, + "precision": 0.9509, + "predictedPositive": 815, + "recall": 0.8201, + "total": 1485, + "trueNegative": 500, + "truePositive": 775, + "wilsonPrecisionLower95": 0.9339 + }, + "threshold": 0.8 + } + }, + "metrics": { + "f1": 0.914, + "falseNegative": 227, + "falsePositive": 86, + "precision": 0.9508, + "predictedPositive": 1749, + "recall": 0.8799, + "total": 2970, + "trueNegative": 994, + "truePositive": 1663, + "wilsonPrecisionLower95": 0.9397 + }, + "threshold": 0.78 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9888, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 44, + "recall": 0.9778, + "total": 1485, + "trueNegative": 1440, + "truePositive": 44, + "wilsonPrecisionLower95": 0.9197 + }, + "threshold": 0.635 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.8129, + "falseNegative": 27, + "falsePositive": 2, + "precision": 0.9692, + "predictedPositive": 65, + "recall": 0.7, + "total": 2970, + "trueNegative": 2878, + "truePositive": 63, + "wilsonPrecisionLower95": 0.8946 + }, + "threshold": 0.995 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8185, + "falseNegative": 103, + "falsePositive": 11, + "precision": 0.959, + "predictedPositive": 268, + "recall": 0.7139, + "total": 1485, + "trueNegative": 1114, + "truePositive": 257, + "wilsonPrecisionLower95": 0.928 + }, + "threshold": 0.83 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7702, + "falseNegative": 127, + "falsePositive": 12, + "precision": 0.951, + "predictedPositive": 245, + "recall": 0.6472, + "total": 1485, + "trueNegative": 1113, + "truePositive": 233, + "wilsonPrecisionLower95": 0.9164 + }, + "threshold": 0.765 + } + }, + "metrics": { + "f1": 0.7816, + "falseNegative": 244, + "falsePositive": 22, + "precision": 0.9558, + "predictedPositive": 498, + "recall": 0.6611, + "total": 2970, + "trueNegative": 2228, + "truePositive": 476, + "wilsonPrecisionLower95": 0.934 + }, + "threshold": 0.805 + } + }, + "trainingRecords": 15345, + "validation": { + "intentsMeeting95Precision": 8, + "macroF1": 0.6867, + "macroPrecision": 0.9721, + "macroRecall": 0.5915, + "minimumPrecision": 0.9508, + "perIntent": { + "blessing": { + "f1": 0.0851, + "falseNegative": 86, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 4, + "recall": 0.0444, + "total": 2970, + "trueNegative": 2880, + "truePositive": 4, + "wilsonPrecisionLower95": 0.5101 + }, + "complaint": { + "f1": 0.9648, + "falseNegative": 17, + "falsePositive": 8, + "precision": 0.9772, + "predictedPositive": 351, + "recall": 0.9528, + "total": 2970, + "trueNegative": 2602, + "truePositive": 343, + "wilsonPrecisionLower95": 0.9557 + }, + "confirmationDecision": { + "f1": 0.7748, + "falseNegative": 63, + "falsePositive": 5, + "precision": 0.959, + "predictedPositive": 122, + "recall": 0.65, + "total": 2970, + "trueNegative": 2785, + "truePositive": 117, + "wilsonPrecisionLower95": 0.9076 + }, + "followUpReminder": { + "f1": 0.7856, + "falseNegative": 120, + "falsePositive": 11, + "precision": 0.9562, + "predictedPositive": 251, + "recall": 0.6667, + "total": 2970, + "trueNegative": 2599, + "truePositive": 240, + "wilsonPrecisionLower95": 0.9232 + }, + "invitation": { + "f1": 0.3694, + "falseNegative": 139, + "falsePositive": 1, + "precision": 0.9762, + "predictedPositive": 42, + "recall": 0.2278, + "total": 2970, + "trueNegative": 2789, + "truePositive": 41, + "wilsonPrecisionLower95": 0.8768 + }, + "question": { + "f1": 0.8306, + "falseNegative": 174, + "falsePositive": 12, + "precision": 0.9744, + "predictedPositive": 468, + "recall": 0.7238, + "total": 2970, + "trueNegative": 2328, + "truePositive": 456, + "wilsonPrecisionLower95": 0.9557 + }, + "replyableMessage": { + "f1": 0.9188, + "falseNegative": 210, + "falsePositive": 87, + "precision": 0.9508, + "predictedPositive": 1767, + "recall": 0.8889, + "total": 2970, + "trueNegative": 993, + "truePositive": 1680, + "wilsonPrecisionLower95": 0.9397 + }, + "scheduleNegotiation": { + "f1": 0.6567, + "falseNegative": 46, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 44, + "recall": 0.4889, + "total": 2970, + "trueNegative": 2880, + "truePositive": 44, + "wilsonPrecisionLower95": 0.9197 + }, + "task": { + "f1": 0.7948, + "falseNegative": 230, + "falsePositive": 23, + "precision": 0.9552, + "predictedPositive": 513, + "recall": 0.6806, + "total": 2970, + "trueNegative": 2227, + "truePositive": 490, + "wilsonPrecisionLower95": 0.9336 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 8e-06, + "augmentation": "punctuation", + "char_max": 6, + "char_min": 2, + "external_weight": 0.67, + "hard_example_weight": 1.5, + "l1_ratio": 0.1, + "max_char_features": 65000, + "max_word_features": 28000, + "min_df": 2, + "pseudo_weight": 0.24, + "round": 6, + "word_max": 2 + }, + "featureCount": 78354, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 0 + }, + "complaint": { + "negative": 146, + "positive": 0 + }, + "confirmationDecision": { + "negative": 88, + "positive": 2 + }, + "followUpReminder": { + "negative": 250, + "positive": 0 + }, + "invitation": { + "negative": 250, + "positive": 0 + }, + "question": { + "negative": 36, + "positive": 5 + }, + "replyableMessage": { + "negative": 2, + "positive": 130 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 0 + }, + "task": { + "negative": 83, + "positive": 0 + } + }, + "round": 6, + "selectionObjective": 0.811, + "syntheticRegression": { + "intentsMeeting95Precision": 5, + "macroF1": 0.7064, + "macroPrecision": 0.9443, + "macroRecall": 0.623, + "minimumPrecision": 0.7975, + "perIntent": { + "blessing": { + "f1": 0.0435, + "falseNegative": 88, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 2, + "recall": 0.0222, + "total": 2970, + "trueNegative": 2880, + "truePositive": 2, + "wilsonPrecisionLower95": 0.3424 + }, + "complaint": { + "f1": 0.967, + "falseNegative": 23, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 337, + "recall": 0.9361, + "total": 2970, + "trueNegative": 2610, + "truePositive": 337, + "wilsonPrecisionLower95": 0.9887 + }, + "confirmationDecision": { + "f1": 0.7215, + "falseNegative": 66, + "falsePositive": 22, + "precision": 0.8382, + "predictedPositive": 136, + "recall": 0.6333, + "total": 2970, + "trueNegative": 2768, + "truePositive": 114, + "wilsonPrecisionLower95": 0.7672 + }, + "followUpReminder": { + "f1": 0.6847, + "falseNegative": 170, + "falsePositive": 5, + "precision": 0.9744, + "predictedPositive": 195, + "recall": 0.5278, + "total": 2970, + "trueNegative": 2605, + "truePositive": 190, + "wilsonPrecisionLower95": 0.9414 + }, + "invitation": { + "f1": 0.4865, + "falseNegative": 117, + "falsePositive": 16, + "precision": 0.7975, + "predictedPositive": 79, + "recall": 0.35, + "total": 2970, + "trueNegative": 2774, + "truePositive": 63, + "wilsonPrecisionLower95": 0.696 + }, + "question": { + "f1": 0.8556, + "falseNegative": 156, + "falsePositive": 4, + "precision": 0.9916, + "predictedPositive": 478, + "recall": 0.7524, + "total": 2970, + "trueNegative": 2336, + "truePositive": 474, + "wilsonPrecisionLower95": 0.9787 + }, + "replyableMessage": { + "f1": 0.8888, + "falseNegative": 335, + "falsePositive": 54, + "precision": 0.9664, + "predictedPositive": 1609, + "recall": 0.8228, + "total": 2970, + "trueNegative": 1026, + "truePositive": 1555, + "wilsonPrecisionLower95": 0.9565 + }, + "scheduleNegotiation": { + "f1": 0.9831, + "falseNegative": 3, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 87, + "recall": 0.9667, + "total": 2970, + "trueNegative": 2880, + "truePositive": 87, + "wilsonPrecisionLower95": 0.9577 + }, + "task": { + "f1": 0.7265, + "falseNegative": 291, + "falsePositive": 32, + "precision": 0.9306, + "predictedPositive": 461, + "recall": 0.5958, + "total": 2970, + "trueNegative": 2218, + "truePositive": 429, + "wilsonPrecisionLower95": 0.9037 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9834, + "falseNegative": 2, + "falsePositive": 4, + "precision": 0.978, + "predictedPositive": 182, + "recall": 0.9889, + "total": 1485, + "trueNegative": 1301, + "truePositive": 178, + "wilsonPrecisionLower95": 0.9449 + }, + "threshold": 0.555 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9494, + "falseNegative": 11, + "falsePositive": 7, + "precision": 0.9602, + "predictedPositive": 176, + "recall": 0.9389, + "total": 1485, + "trueNegative": 1298, + "truePositive": 169, + "wilsonPrecisionLower95": 0.9202 + }, + "threshold": 0.535 + } + }, + "metrics": { + "f1": 0.9652, + "falseNegative": 13, + "falsePositive": 12, + "precision": 0.9666, + "predictedPositive": 359, + "recall": 0.9639, + "total": 2970, + "trueNegative": 2598, + "truePositive": 347, + "wilsonPrecisionLower95": 0.9425 + }, + "threshold": 0.535 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8, + "falseNegative": 28, + "falsePositive": 3, + "precision": 0.9538, + "predictedPositive": 65, + "recall": 0.6889, + "total": 1485, + "trueNegative": 1392, + "truePositive": 62, + "wilsonPrecisionLower95": 0.8729 + }, + "threshold": 0.695 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.731, + "falseNegative": 37, + "falsePositive": 2, + "precision": 0.9636, + "predictedPositive": 55, + "recall": 0.5889, + "total": 1485, + "trueNegative": 1393, + "truePositive": 53, + "wilsonPrecisionLower95": 0.8768 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.7354, + "falseNegative": 73, + "falsePositive": 4, + "precision": 0.964, + "predictedPositive": 111, + "recall": 0.5944, + "total": 2970, + "trueNegative": 2786, + "truePositive": 107, + "wilsonPrecisionLower95": 0.911 + }, + "threshold": 0.695 + }, + "followUpReminder": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8632, + "falseNegative": 38, + "falsePositive": 7, + "precision": 0.953, + "predictedPositive": 149, + "recall": 0.7889, + "total": 1485, + "trueNegative": 1298, + "truePositive": 142, + "wilsonPrecisionLower95": 0.9062 + }, + "threshold": 0.9 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.4938, + "falseNegative": 120, + "falsePositive": 3, + "precision": 0.9524, + "predictedPositive": 63, + "recall": 0.3333, + "total": 1485, + "trueNegative": 1302, + "truePositive": 60, + "wilsonPrecisionLower95": 0.8691 + }, + "threshold": 0.845 + } + }, + "metrics": { + "f1": 0.7018, + "falseNegative": 160, + "falsePositive": 10, + "precision": 0.9524, + "predictedPositive": 210, + "recall": 0.5556, + "total": 2970, + "trueNegative": 2600, + "truePositive": 200, + "wilsonPrecisionLower95": 0.9146 + }, + "threshold": 0.885 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7222, + "falseNegative": 38, + "falsePositive": 2, + "precision": 0.963, + "predictedPositive": 54, + "recall": 0.5778, + "total": 1485, + "trueNegative": 1393, + "truePositive": 52, + "wilsonPrecisionLower95": 0.8746 + }, + "threshold": 0.665 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8167, + "falseNegative": 90, + "falsePositive": 11, + "precision": 0.9534, + "predictedPositive": 236, + "recall": 0.7143, + "total": 1485, + "trueNegative": 1159, + "truePositive": 225, + "wilsonPrecisionLower95": 0.9185 + }, + "threshold": 0.865 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8205, + "falseNegative": 91, + "falsePositive": 7, + "precision": 0.9697, + "predictedPositive": 231, + "recall": 0.7111, + "total": 1485, + "trueNegative": 1163, + "truePositive": 224, + "wilsonPrecisionLower95": 0.9388 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.804, + "falseNegative": 191, + "falsePositive": 23, + "precision": 0.9502, + "predictedPositive": 462, + "recall": 0.6968, + "total": 2970, + "trueNegative": 2317, + "truePositive": 439, + "wilsonPrecisionLower95": 0.9264 + }, + "threshold": 0.68 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9268, + "falseNegative": 90, + "falsePositive": 45, + "precision": 0.95, + "predictedPositive": 900, + "recall": 0.9048, + "total": 1485, + "trueNegative": 495, + "truePositive": 855, + "wilsonPrecisionLower95": 0.9338 + }, + "threshold": 0.86 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8789, + "falseNegative": 176, + "falsePositive": 36, + "precision": 0.9553, + "predictedPositive": 805, + "recall": 0.8138, + "total": 1485, + "trueNegative": 504, + "truePositive": 769, + "wilsonPrecisionLower95": 0.9387 + }, + "threshold": 0.85 + } + }, + "metrics": { + "f1": 0.9046, + "falseNegative": 259, + "falsePositive": 85, + "precision": 0.9505, + "predictedPositive": 1716, + "recall": 0.863, + "total": 2970, + "trueNegative": 995, + "truePositive": 1631, + "wilsonPrecisionLower95": 0.9392 + }, + "threshold": 0.85 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9773, + "falseNegative": 2, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 43, + "recall": 0.9556, + "total": 1485, + "trueNegative": 1440, + "truePositive": 43, + "wilsonPrecisionLower95": 0.918 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9888, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 44, + "recall": 0.9778, + "total": 1485, + "trueNegative": 1440, + "truePositive": 44, + "wilsonPrecisionLower95": 0.9197 + }, + "threshold": 0.97 + } + }, + "metrics": { + "f1": 0.9497, + "falseNegative": 5, + "falsePositive": 4, + "precision": 0.9551, + "predictedPositive": 89, + "recall": 0.9444, + "total": 2970, + "trueNegative": 2876, + "truePositive": 85, + "wilsonPrecisionLower95": 0.8901 + }, + "threshold": 0.91 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.827, + "falseNegative": 97, + "falsePositive": 13, + "precision": 0.9529, + "predictedPositive": 276, + "recall": 0.7306, + "total": 1485, + "trueNegative": 1112, + "truePositive": 263, + "wilsonPrecisionLower95": 0.9211 + }, + "threshold": 0.89 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7902, + "falseNegative": 117, + "falsePositive": 12, + "precision": 0.9529, + "predictedPositive": 255, + "recall": 0.675, + "total": 1485, + "trueNegative": 1113, + "truePositive": 243, + "wilsonPrecisionLower95": 0.9196 + }, + "threshold": 0.785 + } + }, + "metrics": { + "f1": 0.7783, + "falseNegative": 246, + "falsePositive": 24, + "precision": 0.9518, + "predictedPositive": 498, + "recall": 0.6583, + "total": 2970, + "trueNegative": 2226, + "truePositive": 474, + "wilsonPrecisionLower95": 0.9293 + }, + "threshold": 0.85 + } + }, + "trainingRecords": 17767, + "validation": { + "intentsMeeting95Precision": 8, + "macroF1": 0.7395, + "macroPrecision": 0.9687, + "macroRecall": 0.6518, + "minimumPrecision": 0.9525, + "perIntent": { + "blessing": { + "f1": 0.1633, + "falseNegative": 82, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 8, + "recall": 0.0889, + "total": 2970, + "trueNegative": 2880, + "truePositive": 8, + "wilsonPrecisionLower95": 0.6756 + }, + "complaint": { + "f1": 0.9666, + "falseNegative": 13, + "falsePositive": 11, + "precision": 0.9693, + "predictedPositive": 358, + "recall": 0.9639, + "total": 2970, + "trueNegative": 2599, + "truePositive": 347, + "wilsonPrecisionLower95": 0.9458 + }, + "confirmationDecision": { + "f1": 0.7667, + "falseNegative": 65, + "falsePositive": 5, + "precision": 0.9583, + "predictedPositive": 120, + "recall": 0.6389, + "total": 2970, + "trueNegative": 2785, + "truePositive": 115, + "wilsonPrecisionLower95": 0.9062 + }, + "followUpReminder": { + "f1": 0.7063, + "falseNegative": 158, + "falsePositive": 10, + "precision": 0.9528, + "predictedPositive": 212, + "recall": 0.5611, + "total": 2970, + "trueNegative": 2600, + "truePositive": 202, + "wilsonPrecisionLower95": 0.9154 + }, + "invitation": { + "f1": 0.5382, + "falseNegative": 113, + "falsePositive": 2, + "precision": 0.971, + "predictedPositive": 69, + "recall": 0.3722, + "total": 2970, + "trueNegative": 2788, + "truePositive": 67, + "wilsonPrecisionLower95": 0.9003 + }, + "question": { + "f1": 0.8186, + "falseNegative": 181, + "falsePositive": 18, + "precision": 0.9615, + "predictedPositive": 467, + "recall": 0.7127, + "total": 2970, + "trueNegative": 2322, + "truePositive": 449, + "wilsonPrecisionLower95": 0.9399 + }, + "replyableMessage": { + "f1": 0.9035, + "falseNegative": 266, + "falsePositive": 81, + "precision": 0.9525, + "predictedPositive": 1705, + "recall": 0.8593, + "total": 2970, + "trueNegative": 999, + "truePositive": 1624, + "wilsonPrecisionLower95": 0.9413 + }, + "scheduleNegotiation": { + "f1": 0.9831, + "falseNegative": 3, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 87, + "recall": 0.9667, + "total": 2970, + "trueNegative": 2880, + "truePositive": 87, + "wilsonPrecisionLower95": 0.9577 + }, + "task": { + "f1": 0.809, + "falseNegative": 214, + "falsePositive": 25, + "precision": 0.9529, + "predictedPositive": 531, + "recall": 0.7028, + "total": 2970, + "trueNegative": 2225, + "truePositive": 506, + "wilsonPrecisionLower95": 0.9314 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 6e-06, + "augmentation": "punctuation", + "char_max": 6, + "char_min": 3, + "external_weight": 0.6900000000000001, + "hard_example_weight": 1.5, + "l1_ratio": 0.1, + "max_char_features": 65000, + "max_word_features": 32000, + "min_df": 2, + "pseudo_weight": 0.255, + "round": 7, + "word_max": 3 + }, + "featureCount": 87507, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 0 + }, + "complaint": { + "negative": 141, + "positive": 0 + }, + "confirmationDecision": { + "negative": 95, + "positive": 2 + }, + "followUpReminder": { + "negative": 250, + "positive": 0 + }, + "invitation": { + "negative": 250, + "positive": 0 + }, + "question": { + "negative": 36, + "positive": 5 + }, + "replyableMessage": { + "negative": 2, + "positive": 129 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 0 + }, + "task": { + "negative": 87, + "positive": 0 + } + }, + "round": 7, + "selectionObjective": 0.7501, + "syntheticRegression": { + "intentsMeeting95Precision": 5, + "macroF1": 0.587, + "macroPrecision": 0.9374, + "macroRecall": 0.4917, + "minimumPrecision": 0.7548, + "perIntent": { + "blessing": { + "f1": 0.0851, + "falseNegative": 86, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 4, + "recall": 0.0444, + "total": 2970, + "trueNegative": 2880, + "truePositive": 4, + "wilsonPrecisionLower95": 0.5101 + }, + "complaint": { + "f1": 0.9304, + "falseNegative": 39, + "falsePositive": 9, + "precision": 0.9727, + "predictedPositive": 330, + "recall": 0.8917, + "total": 2970, + "trueNegative": 2601, + "truePositive": 321, + "wilsonPrecisionLower95": 0.949 + }, + "confirmationDecision": { + "f1": 0.6985, + "falseNegative": 63, + "falsePositive": 38, + "precision": 0.7548, + "predictedPositive": 155, + "recall": 0.65, + "total": 2970, + "trueNegative": 2752, + "truePositive": 117, + "wilsonPrecisionLower95": 0.6815 + }, + "followUpReminder": { + "f1": 0.3974, + "falseNegative": 269, + "falsePositive": 7, + "precision": 0.9286, + "predictedPositive": 98, + "recall": 0.2528, + "total": 2970, + "trueNegative": 2603, + "truePositive": 91, + "wilsonPrecisionLower95": 0.8598 + }, + "invitation": { + "f1": 0.2609, + "falseNegative": 153, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 27, + "recall": 0.15, + "total": 2970, + "trueNegative": 2790, + "truePositive": 27, + "wilsonPrecisionLower95": 0.8754 + }, + "question": { + "f1": 0.697, + "falseNegative": 285, + "falsePositive": 15, + "precision": 0.9583, + "predictedPositive": 360, + "recall": 0.5476, + "total": 2970, + "trueNegative": 2325, + "truePositive": 345, + "wilsonPrecisionLower95": 0.9324 + }, + "replyableMessage": { + "f1": 0.8083, + "falseNegative": 572, + "falsePositive": 53, + "precision": 0.9613, + "predictedPositive": 1371, + "recall": 0.6974, + "total": 2970, + "trueNegative": 1027, + "truePositive": 1318, + "wilsonPrecisionLower95": 0.9498 + }, + "scheduleNegotiation": { + "f1": 0.8929, + "falseNegative": 15, + "falsePositive": 3, + "precision": 0.9615, + "predictedPositive": 78, + "recall": 0.8333, + "total": 2970, + "trueNegative": 2877, + "truePositive": 75, + "wilsonPrecisionLower95": 0.8929 + }, + "task": { + "f1": 0.5124, + "falseNegative": 462, + "falsePositive": 29, + "precision": 0.899, + "predictedPositive": 287, + "recall": 0.3583, + "total": 2970, + "trueNegative": 2221, + "truePositive": 258, + "wilsonPrecisionLower95": 0.8586 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9809, + "falseNegative": 0, + "falsePositive": 7, + "precision": 0.9626, + "predictedPositive": 187, + "recall": 1.0, + "total": 1485, + "trueNegative": 1298, + "truePositive": 180, + "wilsonPrecisionLower95": 0.9248 + }, + "threshold": 0.74 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8974, + "falseNegative": 27, + "falsePositive": 8, + "precision": 0.9503, + "predictedPositive": 161, + "recall": 0.85, + "total": 1485, + "trueNegative": 1297, + "truePositive": 153, + "wilsonPrecisionLower95": 0.905 + }, + "threshold": 0.665 + } + }, + "metrics": { + "f1": 0.9529, + "falseNegative": 16, + "falsePositive": 18, + "precision": 0.9503, + "predictedPositive": 362, + "recall": 0.9556, + "total": 2970, + "trueNegative": 2592, + "truePositive": 344, + "wilsonPrecisionLower95": 0.9228 + }, + "threshold": 0.51 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.7763, + "falseNegative": 31, + "falsePositive": 3, + "precision": 0.9516, + "predictedPositive": 62, + "recall": 0.6556, + "total": 1485, + "trueNegative": 1392, + "truePositive": 59, + "wilsonPrecisionLower95": 0.8671 + }, + "threshold": 0.895 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7568, + "falseNegative": 34, + "falsePositive": 2, + "precision": 0.9655, + "predictedPositive": 58, + "recall": 0.6222, + "total": 1485, + "trueNegative": 1393, + "truePositive": 56, + "wilsonPrecisionLower95": 0.8827 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.6569, + "falseNegative": 90, + "falsePositive": 4, + "precision": 0.9574, + "predictedPositive": 94, + "recall": 0.5, + "total": 2970, + "trueNegative": 2786, + "truePositive": 90, + "wilsonPrecisionLower95": 0.8956 + }, + "threshold": 0.895 + }, + "followUpReminder": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8421, + "falseNegative": 44, + "falsePositive": 7, + "precision": 0.951, + "predictedPositive": 143, + "recall": 0.7556, + "total": 1485, + "trueNegative": 1298, + "truePositive": 136, + "wilsonPrecisionLower95": 0.9024 + }, + "threshold": 0.985 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.5985, + "falseNegative": 205, + "falsePositive": 3, + "precision": 0.981, + "predictedPositive": 158, + "recall": 0.4306, + "total": 2970, + "trueNegative": 2607, + "truePositive": 155, + "wilsonPrecisionLower95": 0.9457 + }, + "threshold": 0.995 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.7143, + "falseNegative": 135, + "falsePositive": 9, + "precision": 0.9524, + "predictedPositive": 189, + "recall": 0.5714, + "total": 1485, + "trueNegative": 1161, + "truePositive": 180, + "wilsonPrecisionLower95": 0.912 + }, + "threshold": 0.965 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7168, + "falseNegative": 134, + "falsePositive": 9, + "precision": 0.9526, + "predictedPositive": 190, + "recall": 0.5746, + "total": 1485, + "trueNegative": 1161, + "truePositive": 181, + "wilsonPrecisionLower95": 0.9124 + }, + "threshold": 0.52 + } + }, + "metrics": { + "f1": 0.7149, + "falseNegative": 269, + "falsePositive": 19, + "precision": 0.95, + "predictedPositive": 380, + "recall": 0.573, + "total": 2970, + "trueNegative": 2321, + "truePositive": 361, + "wilsonPrecisionLower95": 0.9232 + }, + "threshold": 0.795 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8846, + "falseNegative": 163, + "falsePositive": 41, + "precision": 0.9502, + "predictedPositive": 823, + "recall": 0.8275, + "total": 1485, + "trueNegative": 499, + "truePositive": 782, + "wilsonPrecisionLower95": 0.9331 + }, + "threshold": 0.89 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7476, + "falseNegative": 363, + "falsePositive": 30, + "precision": 0.951, + "predictedPositive": 612, + "recall": 0.6159, + "total": 1485, + "trueNegative": 510, + "truePositive": 582, + "wilsonPrecisionLower95": 0.9309 + }, + "threshold": 0.92 + } + }, + "metrics": { + "f1": 0.828, + "falseNegative": 506, + "falsePositive": 69, + "precision": 0.9525, + "predictedPositive": 1453, + "recall": 0.7323, + "total": 2970, + "trueNegative": 1011, + "truePositive": 1384, + "wilsonPrecisionLower95": 0.9403 + }, + "threshold": 0.91 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 45, + "recall": 1.0, + "total": 1485, + "trueNegative": 1440, + "truePositive": 45, + "wilsonPrecisionLower95": 0.9213 + }, + "threshold": 0.66 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 45, + "recall": 1.0, + "total": 1485, + "trueNegative": 1440, + "truePositive": 45, + "wilsonPrecisionLower95": 0.9213 + }, + "threshold": 0.55 + } + }, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 90, + "recall": 1.0, + "total": 2970, + "trueNegative": 2880, + "truePositive": 90, + "wilsonPrecisionLower95": 0.9591 + }, + "threshold": 0.66 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8615, + "falseNegative": 77, + "falsePositive": 14, + "precision": 0.9529, + "predictedPositive": 297, + "recall": 0.7861, + "total": 1485, + "trueNegative": 1111, + "truePositive": 283, + "wilsonPrecisionLower95": 0.9224 + }, + "threshold": 0.805 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.6868, + "falseNegative": 333, + "falsePositive": 20, + "precision": 0.9509, + "predictedPositive": 407, + "recall": 0.5375, + "total": 2970, + "trueNegative": 2230, + "truePositive": 387, + "wilsonPrecisionLower95": 0.9253 + }, + "threshold": 0.925 + } + }, + "trainingRecords": 17708, + "validation": { + "intentsMeeting95Precision": 8, + "macroF1": 0.6491, + "macroPrecision": 0.967, + "macroRecall": 0.5465, + "minimumPrecision": 0.9315, + "perIntent": { + "blessing": { + "f1": 0.2353, + "falseNegative": 78, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 12, + "recall": 0.1333, + "total": 2970, + "trueNegative": 2880, + "truePositive": 12, + "wilsonPrecisionLower95": 0.7575 + }, + "complaint": { + "f1": 0.9407, + "falseNegative": 27, + "falsePositive": 15, + "precision": 0.9569, + "predictedPositive": 348, + "recall": 0.925, + "total": 2970, + "trueNegative": 2595, + "truePositive": 333, + "wilsonPrecisionLower95": 0.9301 + }, + "confirmationDecision": { + "f1": 0.7667, + "falseNegative": 65, + "falsePositive": 5, + "precision": 0.9583, + "predictedPositive": 120, + "recall": 0.6389, + "total": 2970, + "trueNegative": 2785, + "truePositive": 115, + "wilsonPrecisionLower95": 0.9062 + }, + "followUpReminder": { + "f1": 0.5375, + "falseNegative": 224, + "falsePositive": 10, + "precision": 0.9315, + "predictedPositive": 146, + "recall": 0.3778, + "total": 2970, + "trueNegative": 2600, + "truePositive": 136, + "wilsonPrecisionLower95": 0.8785 + }, + "invitation": { + "f1": 0.2692, + "falseNegative": 152, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 28, + "recall": 0.1556, + "total": 2970, + "trueNegative": 2790, + "truePositive": 28, + "wilsonPrecisionLower95": 0.8794 + }, + "question": { + "f1": 0.7156, + "falseNegative": 269, + "falsePositive": 18, + "precision": 0.9525, + "predictedPositive": 379, + "recall": 0.573, + "total": 2970, + "trueNegative": 2322, + "truePositive": 361, + "wilsonPrecisionLower95": 0.9262 + }, + "replyableMessage": { + "f1": 0.8205, + "falseNegative": 526, + "falsePositive": 71, + "precision": 0.9505, + "predictedPositive": 1435, + "recall": 0.7217, + "total": 2970, + "trueNegative": 1009, + "truePositive": 1364, + "wilsonPrecisionLower95": 0.9381 + }, + "scheduleNegotiation": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 90, + "recall": 1.0, + "total": 2970, + "trueNegative": 2880, + "truePositive": 90, + "wilsonPrecisionLower95": 0.9591 + }, + "task": { + "f1": 0.5565, + "falseNegative": 437, + "falsePositive": 14, + "precision": 0.9529, + "predictedPositive": 297, + "recall": 0.3931, + "total": 2970, + "trueNegative": 2236, + "truePositive": 283, + "wilsonPrecisionLower95": 0.9224 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 5e-06, + "augmentation": "punctuation", + "char_max": 7, + "char_min": 2, + "external_weight": 0.7100000000000001, + "hard_example_weight": 1.7, + "l1_ratio": 0.15, + "max_char_features": 70000, + "max_word_features": 28000, + "min_df": 2, + "pseudo_weight": 0.27, + "round": 8, + "word_max": 2 + }, + "featureCount": 83313, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 0 + }, + "complaint": { + "negative": 187, + "positive": 0 + }, + "confirmationDecision": { + "negative": 107, + "positive": 2 + }, + "followUpReminder": { + "negative": 250, + "positive": 0 + }, + "invitation": { + "negative": 250, + "positive": 0 + }, + "question": { + "negative": 40, + "positive": 5 + }, + "replyableMessage": { + "negative": 2, + "positive": 135 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 0 + }, + "task": { + "negative": 94, + "positive": 0 + } + }, + "round": 8, + "selectionObjective": 0.7837, + "syntheticRegression": { + "intentsMeeting95Precision": 4, + "macroF1": 0.6947, + "macroPrecision": 0.9526, + "macroRecall": 0.6066, + "minimumPrecision": 0.8509, + "perIntent": { + "blessing": { + "f1": 0.125, + "falseNegative": 84, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 6, + "recall": 0.0667, + "total": 2970, + "trueNegative": 2880, + "truePositive": 6, + "wilsonPrecisionLower95": 0.6097 + }, + "complaint": { + "f1": 0.9744, + "falseNegative": 18, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 342, + "recall": 0.95, + "total": 2970, + "trueNegative": 2610, + "truePositive": 342, + "wilsonPrecisionLower95": 0.9889 + }, + "confirmationDecision": { + "f1": 0.8035, + "falseNegative": 43, + "falsePositive": 24, + "precision": 0.8509, + "predictedPositive": 161, + "recall": 0.7611, + "total": 2970, + "trueNegative": 2766, + "truePositive": 137, + "wilsonPrecisionLower95": 0.7878 + }, + "followUpReminder": { + "f1": 0.5422, + "falseNegative": 222, + "falsePositive": 11, + "precision": 0.9262, + "predictedPositive": 149, + "recall": 0.3833, + "total": 2970, + "trueNegative": 2599, + "truePositive": 138, + "wilsonPrecisionLower95": 0.8726 + }, + "invitation": { + "f1": 0.5062, + "falseNegative": 119, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 61, + "recall": 0.3389, + "total": 2970, + "trueNegative": 2790, + "truePositive": 61, + "wilsonPrecisionLower95": 0.9408 + }, + "question": { + "f1": 0.7719, + "falseNegative": 229, + "falsePositive": 8, + "precision": 0.9804, + "predictedPositive": 409, + "recall": 0.6365, + "total": 2970, + "trueNegative": 2332, + "truePositive": 401, + "wilsonPrecisionLower95": 0.9619 + }, + "replyableMessage": { + "f1": 0.8832, + "falseNegative": 366, + "falsePositive": 37, + "precision": 0.9763, + "predictedPositive": 1561, + "recall": 0.8063, + "total": 2970, + "trueNegative": 1043, + "truePositive": 1524, + "wilsonPrecisionLower95": 0.9675 + }, + "scheduleNegotiation": { + "f1": 0.9162, + "falseNegative": 8, + "falsePositive": 7, + "precision": 0.9213, + "predictedPositive": 89, + "recall": 0.9111, + "total": 2970, + "trueNegative": 2873, + "truePositive": 82, + "wilsonPrecisionLower95": 0.8464 + }, + "task": { + "f1": 0.7297, + "falseNegative": 284, + "falsePositive": 39, + "precision": 0.9179, + "predictedPositive": 475, + "recall": 0.6056, + "total": 2970, + "trueNegative": 2211, + "truePositive": 436, + "wilsonPrecisionLower95": 0.8897 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.989, + "falseNegative": 1, + "falsePositive": 3, + "precision": 0.9835, + "predictedPositive": 182, + "recall": 0.9944, + "total": 1485, + "trueNegative": 1302, + "truePositive": 179, + "wilsonPrecisionLower95": 0.9527 + }, + "threshold": 0.97 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9405, + "falseNegative": 14, + "falsePositive": 7, + "precision": 0.9595, + "predictedPositive": 173, + "recall": 0.9222, + "total": 1485, + "trueNegative": 1298, + "truePositive": 166, + "wilsonPrecisionLower95": 0.9188 + }, + "threshold": 0.54 + } + }, + "metrics": { + "f1": 0.9237, + "falseNegative": 39, + "falsePositive": 14, + "precision": 0.9582, + "predictedPositive": 335, + "recall": 0.8917, + "total": 2970, + "trueNegative": 2596, + "truePositive": 321, + "wilsonPrecisionLower95": 0.9311 + }, + "threshold": 0.925 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9613, + "falseNegative": 3, + "falsePositive": 4, + "precision": 0.956, + "predictedPositive": 91, + "recall": 0.9667, + "total": 1485, + "trueNegative": 1391, + "truePositive": 87, + "wilsonPrecisionLower95": 0.8924 + }, + "threshold": 0.72 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.5556, + "falseNegative": 55, + "falsePositive": 1, + "precision": 0.9722, + "predictedPositive": 36, + "recall": 0.3889, + "total": 1485, + "trueNegative": 1394, + "truePositive": 35, + "wilsonPrecisionLower95": 0.8583 + }, + "threshold": 0.905 + } + }, + "metrics": { + "f1": 0.7922, + "falseNegative": 58, + "falsePositive": 6, + "precision": 0.9531, + "predictedPositive": 128, + "recall": 0.6778, + "total": 2970, + "trueNegative": 2784, + "truePositive": 122, + "wilsonPrecisionLower95": 0.9015 + }, + "threshold": 0.88 + }, + "followUpReminder": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8437, + "falseNegative": 45, + "falsePositive": 5, + "precision": 0.9643, + "predictedPositive": 140, + "recall": 0.75, + "total": 1485, + "trueNegative": 1300, + "truePositive": 135, + "wilsonPrecisionLower95": 0.9191 + }, + "threshold": 0.96 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.7063, + "falseNegative": 158, + "falsePositive": 10, + "precision": 0.9528, + "predictedPositive": 212, + "recall": 0.5611, + "total": 2970, + "trueNegative": 2600, + "truePositive": 202, + "wilsonPrecisionLower95": 0.9154 + }, + "threshold": 0.96 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.7157, + "falseNegative": 135, + "falsePositive": 8, + "precision": 0.9574, + "predictedPositive": 188, + "recall": 0.5714, + "total": 1485, + "trueNegative": 1162, + "truePositive": 180, + "wilsonPrecisionLower95": 0.9183 + }, + "threshold": 0.975 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.822, + "falseNegative": 91, + "falsePositive": 6, + "precision": 0.9739, + "predictedPositive": 230, + "recall": 0.7111, + "total": 1485, + "trueNegative": 1164, + "truePositive": 224, + "wilsonPrecisionLower95": 0.9443 + }, + "threshold": 0.505 + } + }, + "metrics": { + "f1": 0.8095, + "falseNegative": 186, + "falsePositive": 23, + "precision": 0.9507, + "predictedPositive": 467, + "recall": 0.7048, + "total": 2970, + "trueNegative": 2317, + "truePositive": 444, + "wilsonPrecisionLower95": 0.9272 + }, + "threshold": 0.71 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8741, + "falseNegative": 181, + "falsePositive": 39, + "precision": 0.9514, + "predictedPositive": 803, + "recall": 0.8085, + "total": 1485, + "trueNegative": 501, + "truePositive": 764, + "wilsonPrecisionLower95": 0.9343 + }, + "threshold": 0.95 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8775, + "falseNegative": 175, + "falsePositive": 40, + "precision": 0.9506, + "predictedPositive": 810, + "recall": 0.8148, + "total": 1485, + "trueNegative": 500, + "truePositive": 770, + "wilsonPrecisionLower95": 0.9335 + }, + "threshold": 0.88 + } + }, + "metrics": { + "f1": 0.8762, + "falseNegative": 354, + "falsePositive": 80, + "precision": 0.9505, + "predictedPositive": 1616, + "recall": 0.8127, + "total": 2970, + "trueNegative": 1000, + "truePositive": 1536, + "wilsonPrecisionLower95": 0.9388 + }, + "threshold": 0.905 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 45, + "recall": 1.0, + "total": 1485, + "trueNegative": 1440, + "truePositive": 45, + "wilsonPrecisionLower95": 0.9213 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9556, + "falseNegative": 2, + "falsePositive": 2, + "precision": 0.9556, + "predictedPositive": 45, + "recall": 0.9556, + "total": 1485, + "trueNegative": 1438, + "truePositive": 43, + "wilsonPrecisionLower95": 0.8517 + }, + "threshold": 0.75 + } + }, + "metrics": { + "f1": 0.9724, + "falseNegative": 2, + "falsePositive": 3, + "precision": 0.967, + "predictedPositive": 91, + "recall": 0.9778, + "total": 2970, + "trueNegative": 2877, + "truePositive": 88, + "wilsonPrecisionLower95": 0.9075 + }, + "threshold": 0.505 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8331, + "falseNegative": 93, + "falsePositive": 14, + "precision": 0.9502, + "predictedPositive": 281, + "recall": 0.7417, + "total": 1485, + "trueNegative": 1111, + "truePositive": 267, + "wilsonPrecisionLower95": 0.9181 + }, + "threshold": 0.95 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7445, + "falseNegative": 140, + "falsePositive": 11, + "precision": 0.9524, + "predictedPositive": 231, + "recall": 0.6111, + "total": 1485, + "trueNegative": 1114, + "truePositive": 220, + "wilsonPrecisionLower95": 0.9168 + }, + "threshold": 0.86 + } + }, + "metrics": { + "f1": 0.7672, + "falseNegative": 257, + "falsePositive": 24, + "precision": 0.9507, + "predictedPositive": 487, + "recall": 0.6431, + "total": 2970, + "trueNegative": 2226, + "truePositive": 463, + "wilsonPrecisionLower95": 0.9277 + }, + "threshold": 0.92 + } + }, + "trainingRecords": 17762, + "validation": { + "intentsMeeting95Precision": 8, + "macroF1": 0.698, + "macroPrecision": 0.9692, + "macroRecall": 0.6056, + "minimumPrecision": 0.9444, + "perIntent": { + "blessing": { + "f1": 0.2524, + "falseNegative": 77, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 13, + "recall": 0.1444, + "total": 2970, + "trueNegative": 2880, + "truePositive": 13, + "wilsonPrecisionLower95": 0.7719 + }, + "complaint": { + "f1": 0.965, + "falseNegative": 15, + "falsePositive": 10, + "precision": 0.9718, + "predictedPositive": 355, + "recall": 0.9583, + "total": 2970, + "trueNegative": 2600, + "truePositive": 345, + "wilsonPrecisionLower95": 0.9489 + }, + "confirmationDecision": { + "f1": 0.7948, + "falseNegative": 58, + "falsePositive": 5, + "precision": 0.9606, + "predictedPositive": 127, + "recall": 0.6778, + "total": 2970, + "trueNegative": 2785, + "truePositive": 122, + "wilsonPrecisionLower95": 0.9111 + }, + "followUpReminder": { + "f1": 0.5397, + "falseNegative": 224, + "falsePositive": 8, + "precision": 0.9444, + "predictedPositive": 144, + "recall": 0.3778, + "total": 2970, + "trueNegative": 2602, + "truePositive": 136, + "wilsonPrecisionLower95": 0.8942 + }, + "invitation": { + "f1": 0.3178, + "falseNegative": 146, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 34, + "recall": 0.1889, + "total": 2970, + "trueNegative": 2790, + "truePositive": 34, + "wilsonPrecisionLower95": 0.8985 + }, + "question": { + "f1": 0.771, + "falseNegative": 226, + "falsePositive": 14, + "precision": 0.9665, + "predictedPositive": 418, + "recall": 0.6413, + "total": 2970, + "trueNegative": 2326, + "truePositive": 404, + "wilsonPrecisionLower95": 0.9446 + }, + "replyableMessage": { + "f1": 0.8758, + "falseNegative": 356, + "falsePositive": 79, + "precision": 0.951, + "predictedPositive": 1613, + "recall": 0.8116, + "total": 2970, + "trueNegative": 1001, + "truePositive": 1534, + "wilsonPrecisionLower95": 0.9394 + }, + "scheduleNegotiation": { + "f1": 0.9778, + "falseNegative": 2, + "falsePositive": 2, + "precision": 0.9778, + "predictedPositive": 90, + "recall": 0.9778, + "total": 2970, + "trueNegative": 2878, + "truePositive": 88, + "wilsonPrecisionLower95": 0.9226 + }, + "task": { + "f1": 0.7876, + "falseNegative": 236, + "falsePositive": 25, + "precision": 0.9509, + "predictedPositive": 509, + "recall": 0.6722, + "total": 2970, + "trueNegative": 2225, + "truePositive": 484, + "wilsonPrecisionLower95": 0.9285 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 4e-06, + "augmentation": "punctuation", + "char_max": 7, + "char_min": 3, + "external_weight": 0.73, + "hard_example_weight": 1.7, + "l1_ratio": 0.15, + "max_char_features": 70000, + "max_word_features": 35000, + "min_df": 2, + "pseudo_weight": 0.28500000000000003, + "round": 9, + "word_max": 3 + }, + "featureCount": 92641, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 0 + }, + "complaint": { + "negative": 250, + "positive": 0 + }, + "confirmationDecision": { + "negative": 248, + "positive": 6 + }, + "followUpReminder": { + "negative": 250, + "positive": 1 + }, + "invitation": { + "negative": 250, + "positive": 0 + }, + "question": { + "negative": 146, + "positive": 8 + }, + "replyableMessage": { + "negative": 8, + "positive": 250 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 0 + }, + "task": { + "negative": 208, + "positive": 0 + } + }, + "round": 9, + "selectionObjective": 0.7717, + "syntheticRegression": { + "intentsMeeting95Precision": 4, + "macroF1": 0.6445, + "macroPrecision": 0.9357, + "macroRecall": 0.5386, + "minimumPrecision": 0.8583, + "perIntent": { + "blessing": { + "f1": 0.125, + "falseNegative": 84, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 6, + "recall": 0.0667, + "total": 2970, + "trueNegative": 2880, + "truePositive": 6, + "wilsonPrecisionLower95": 0.6097 + }, + "complaint": { + "f1": 0.949, + "falseNegative": 25, + "falsePositive": 11, + "precision": 0.9682, + "predictedPositive": 346, + "recall": 0.9306, + "total": 2970, + "trueNegative": 2599, + "truePositive": 335, + "wilsonPrecisionLower95": 0.944 + }, + "confirmationDecision": { + "f1": 0.7101, + "falseNegative": 71, + "falsePositive": 18, + "precision": 0.8583, + "predictedPositive": 127, + "recall": 0.6056, + "total": 2970, + "trueNegative": 2772, + "truePositive": 109, + "wilsonPrecisionLower95": 0.7871 + }, + "followUpReminder": { + "f1": 0.5244, + "falseNegative": 231, + "falsePositive": 3, + "precision": 0.9773, + "predictedPositive": 132, + "recall": 0.3583, + "total": 2970, + "trueNegative": 2607, + "truePositive": 129, + "wilsonPrecisionLower95": 0.9353 + }, + "invitation": { + "f1": 0.6286, + "falseNegative": 92, + "falsePositive": 12, + "precision": 0.88, + "predictedPositive": 100, + "recall": 0.4889, + "total": 2970, + "trueNegative": 2778, + "truePositive": 88, + "wilsonPrecisionLower95": 0.8019 + }, + "question": { + "f1": 0.6667, + "falseNegative": 310, + "falsePositive": 10, + "precision": 0.9697, + "predictedPositive": 330, + "recall": 0.5079, + "total": 2970, + "trueNegative": 2330, + "truePositive": 320, + "wilsonPrecisionLower95": 0.9451 + }, + "replyableMessage": { + "f1": 0.7919, + "falseNegative": 594, + "falsePositive": 87, + "precision": 0.9371, + "predictedPositive": 1383, + "recall": 0.6857, + "total": 2970, + "trueNegative": 993, + "truePositive": 1296, + "wilsonPrecisionLower95": 0.923 + }, + "scheduleNegotiation": { + "f1": 0.8941, + "falseNegative": 14, + "falsePositive": 4, + "precision": 0.95, + "predictedPositive": 80, + "recall": 0.8444, + "total": 2970, + "trueNegative": 2876, + "truePositive": 76, + "wilsonPrecisionLower95": 0.8784 + }, + "task": { + "f1": 0.5108, + "falseNegative": 461, + "falsePositive": 35, + "precision": 0.881, + "predictedPositive": 294, + "recall": 0.3597, + "total": 2970, + "trueNegative": 2215, + "truePositive": 259, + "wilsonPrecisionLower95": 0.8389 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9862, + "falseNegative": 1, + "falsePositive": 4, + "precision": 0.9781, + "predictedPositive": 183, + "recall": 0.9944, + "total": 1485, + "trueNegative": 1301, + "truePositive": 179, + "wilsonPrecisionLower95": 0.9452 + }, + "threshold": 0.74 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9222, + "falseNegative": 20, + "falsePositive": 7, + "precision": 0.9581, + "predictedPositive": 167, + "recall": 0.8889, + "total": 1485, + "trueNegative": 1298, + "truePositive": 160, + "wilsonPrecisionLower95": 0.916 + }, + "threshold": 0.62 + } + }, + "metrics": { + "f1": 0.9528, + "falseNegative": 17, + "falsePositive": 17, + "precision": 0.9528, + "predictedPositive": 360, + "recall": 0.9528, + "total": 2970, + "trueNegative": 2593, + "truePositive": 343, + "wilsonPrecisionLower95": 0.9257 + }, + "threshold": 0.5 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9492, + "falseNegative": 6, + "falsePositive": 3, + "precision": 0.9655, + "predictedPositive": 87, + "recall": 0.9333, + "total": 1485, + "trueNegative": 1392, + "truePositive": 84, + "wilsonPrecisionLower95": 0.9035 + }, + "threshold": 0.965 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7042, + "falseNegative": 40, + "falsePositive": 2, + "precision": 0.9615, + "predictedPositive": 52, + "recall": 0.5556, + "total": 1485, + "trueNegative": 1393, + "truePositive": 50, + "wilsonPrecisionLower95": 0.8702 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.76, + "falseNegative": 66, + "falsePositive": 6, + "precision": 0.95, + "predictedPositive": 120, + "recall": 0.6333, + "total": 2970, + "trueNegative": 2784, + "truePositive": 114, + "wilsonPrecisionLower95": 0.8952 + }, + "threshold": 0.945 + }, + "followUpReminder": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8243, + "falseNegative": 51, + "falsePositive": 4, + "precision": 0.9699, + "predictedPositive": 133, + "recall": 0.7167, + "total": 1485, + "trueNegative": 1301, + "truePositive": 129, + "wilsonPrecisionLower95": 0.9252 + }, + "threshold": 0.99 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.6316, + "falseNegative": 192, + "falsePositive": 4, + "precision": 0.9767, + "predictedPositive": 172, + "recall": 0.4667, + "total": 2970, + "trueNegative": 2606, + "truePositive": 168, + "wilsonPrecisionLower95": 0.9417 + }, + "threshold": 0.995 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.544, + "falseNegative": 56, + "falsePositive": 1, + "precision": 0.9714, + "predictedPositive": 35, + "recall": 0.3778, + "total": 1485, + "trueNegative": 1394, + "truePositive": 34, + "wilsonPrecisionLower95": 0.8547 + }, + "threshold": 0.82 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.6694, + "falseNegative": 153, + "falsePositive": 7, + "precision": 0.9586, + "predictedPositive": 169, + "recall": 0.5143, + "total": 1485, + "trueNegative": 1163, + "truePositive": 162, + "wilsonPrecisionLower95": 0.917 + }, + "threshold": 0.985 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6762, + "falseNegative": 150, + "falsePositive": 8, + "precision": 0.9538, + "predictedPositive": 173, + "recall": 0.5238, + "total": 1485, + "trueNegative": 1162, + "truePositive": 165, + "wilsonPrecisionLower95": 0.9114 + }, + "threshold": 0.64 + } + }, + "metrics": { + "f1": 0.5392, + "falseNegative": 393, + "falsePositive": 12, + "precision": 0.9518, + "predictedPositive": 249, + "recall": 0.3762, + "total": 2970, + "trueNegative": 2328, + "truePositive": 237, + "wilsonPrecisionLower95": 0.9177 + }, + "threshold": 0.965 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9407, + "falseNegative": 65, + "falsePositive": 46, + "precision": 0.9503, + "predictedPositive": 926, + "recall": 0.9312, + "total": 1485, + "trueNegative": 494, + "truePositive": 880, + "wilsonPrecisionLower95": 0.9344 + }, + "threshold": 0.745 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6118, + "falseNegative": 521, + "falsePositive": 17, + "precision": 0.9615, + "predictedPositive": 441, + "recall": 0.4487, + "total": 1485, + "trueNegative": 523, + "truePositive": 424, + "wilsonPrecisionLower95": 0.9391 + }, + "threshold": 0.965 + } + }, + "metrics": { + "f1": 0.8037, + "falseNegative": 576, + "falsePositive": 66, + "precision": 0.9522, + "predictedPositive": 1380, + "recall": 0.6952, + "total": 2970, + "trueNegative": 1014, + "truePositive": 1314, + "wilsonPrecisionLower95": 0.9396 + }, + "threshold": 0.95 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9286, + "falseNegative": 6, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 39, + "recall": 0.8667, + "total": 1485, + "trueNegative": 1440, + "truePositive": 39, + "wilsonPrecisionLower95": 0.9103 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9302, + "falseNegative": 5, + "falsePositive": 1, + "precision": 0.9756, + "predictedPositive": 41, + "recall": 0.8889, + "total": 1485, + "trueNegative": 1439, + "truePositive": 40, + "wilsonPrecisionLower95": 0.874 + }, + "threshold": 0.61 + } + }, + "metrics": { + "f1": 0.9294, + "falseNegative": 11, + "falsePositive": 1, + "precision": 0.9875, + "predictedPositive": 80, + "recall": 0.8778, + "total": 2970, + "trueNegative": 2879, + "truePositive": 79, + "wilsonPrecisionLower95": 0.9325 + }, + "threshold": 0.61 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8403, + "falseNegative": 89, + "falsePositive": 14, + "precision": 0.9509, + "predictedPositive": 285, + "recall": 0.7528, + "total": 1485, + "trueNegative": 1111, + "truePositive": 271, + "wilsonPrecisionLower95": 0.9192 + }, + "threshold": 0.93 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.2384, + "falseNegative": 311, + "falsePositive": 2, + "precision": 0.9608, + "predictedPositive": 51, + "recall": 0.1361, + "total": 1485, + "trueNegative": 1123, + "truePositive": 49, + "wilsonPrecisionLower95": 0.8678 + }, + "threshold": 0.995 + } + }, + "metrics": { + "f1": 0.6415, + "falseNegative": 372, + "falsePositive": 17, + "precision": 0.9534, + "predictedPositive": 365, + "recall": 0.4833, + "total": 2970, + "trueNegative": 2233, + "truePositive": 348, + "wilsonPrecisionLower95": 0.9267 + }, + "threshold": 0.975 + } + }, + "trainingRecords": 17823, + "validation": { + "intentsMeeting95Precision": 8, + "macroF1": 0.6858, + "macroPrecision": 0.9638, + "macroRecall": 0.5714, + "minimumPrecision": 0.9211, + "perIntent": { + "blessing": { + "f1": 0.2857, + "falseNegative": 75, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 15, + "recall": 0.1667, + "total": 2970, + "trueNegative": 2880, + "truePositive": 15, + "wilsonPrecisionLower95": 0.7961 + }, + "complaint": { + "f1": 0.9549, + "falseNegative": 21, + "falsePositive": 11, + "precision": 0.9686, + "predictedPositive": 350, + "recall": 0.9417, + "total": 2970, + "trueNegative": 2599, + "truePositive": 339, + "wilsonPrecisionLower95": 0.9446 + }, + "confirmationDecision": { + "f1": 0.8401, + "falseNegative": 46, + "falsePositive": 5, + "precision": 0.964, + "predictedPositive": 139, + "recall": 0.7444, + "total": 2970, + "trueNegative": 2785, + "truePositive": 134, + "wilsonPrecisionLower95": 0.9186 + }, + "followUpReminder": { + "f1": 0.5352, + "falseNegative": 227, + "falsePositive": 4, + "precision": 0.9708, + "predictedPositive": 137, + "recall": 0.3694, + "total": 2970, + "trueNegative": 2606, + "truePositive": 133, + "wilsonPrecisionLower95": 0.9273 + }, + "invitation": { + "f1": 0.5469, + "falseNegative": 110, + "falsePositive": 6, + "precision": 0.9211, + "predictedPositive": 76, + "recall": 0.3889, + "total": 2970, + "trueNegative": 2784, + "truePositive": 70, + "wilsonPrecisionLower95": 0.8383 + }, + "question": { + "f1": 0.6728, + "falseNegative": 303, + "falsePositive": 15, + "precision": 0.9561, + "predictedPositive": 342, + "recall": 0.519, + "total": 2970, + "trueNegative": 2325, + "truePositive": 327, + "wilsonPrecisionLower95": 0.9289 + }, + "replyableMessage": { + "f1": 0.8007, + "falseNegative": 586, + "falsePositive": 63, + "precision": 0.9539, + "predictedPositive": 1367, + "recall": 0.6899, + "total": 2970, + "trueNegative": 1017, + "truePositive": 1304, + "wilsonPrecisionLower95": 0.9415 + }, + "scheduleNegotiation": { + "f1": 0.9294, + "falseNegative": 11, + "falsePositive": 1, + "precision": 0.9875, + "predictedPositive": 80, + "recall": 0.8778, + "total": 2970, + "trueNegative": 2879, + "truePositive": 79, + "wilsonPrecisionLower95": 0.9325 + }, + "task": { + "f1": 0.6061, + "falseNegative": 400, + "falsePositive": 16, + "precision": 0.9524, + "predictedPositive": 336, + "recall": 0.4444, + "total": 2970, + "trueNegative": 2234, + "truePositive": 320, + "wilsonPrecisionLower95": 0.9241 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 6e-06, + "augmentation": "punctuation", + "char_max": 6, + "char_min": 2, + "external_weight": 0.75, + "hard_example_weight": 1.8, + "l1_ratio": 0.2, + "max_char_features": 75000, + "max_word_features": 38000, + "min_df": 1, + "pseudo_weight": 0.3, + "round": 10, + "word_max": 3 + }, + "featureCount": 113000, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 0 + }, + "complaint": { + "negative": 250, + "positive": 0 + }, + "confirmationDecision": { + "negative": 250, + "positive": 11 + }, + "followUpReminder": { + "negative": 250, + "positive": 4 + }, + "invitation": { + "negative": 250, + "positive": 0 + }, + "question": { + "negative": 182, + "positive": 10 + }, + "replyableMessage": { + "negative": 11, + "positive": 250 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 0 + }, + "task": { + "negative": 250, + "positive": 1 + } + }, + "round": 10, + "selectionObjective": 0.7542, + "syntheticRegression": { + "intentsMeeting95Precision": 4, + "macroF1": 0.6448, + "macroPrecision": 0.9609, + "macroRecall": 0.5363, + "minimumPrecision": 0.8734, + "perIntent": { + "blessing": { + "f1": 0.125, + "falseNegative": 84, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 6, + "recall": 0.0667, + "total": 2970, + "trueNegative": 2880, + "truePositive": 6, + "wilsonPrecisionLower95": 0.6097 + }, + "complaint": { + "f1": 0.9366, + "falseNegative": 20, + "falsePositive": 26, + "precision": 0.929, + "predictedPositive": 366, + "recall": 0.9444, + "total": 2970, + "trueNegative": 2584, + "truePositive": 340, + "wilsonPrecisionLower95": 0.8979 + }, + "confirmationDecision": { + "f1": 0.8166, + "falseNegative": 42, + "falsePositive": 20, + "precision": 0.8734, + "predictedPositive": 158, + "recall": 0.7667, + "total": 2970, + "trueNegative": 2770, + "truePositive": 138, + "wilsonPrecisionLower95": 0.8126 + }, + "followUpReminder": { + "f1": 0.5397, + "falseNegative": 224, + "falsePositive": 8, + "precision": 0.9444, + "predictedPositive": 144, + "recall": 0.3778, + "total": 2970, + "trueNegative": 2602, + "truePositive": 136, + "wilsonPrecisionLower95": 0.8942 + }, + "invitation": { + "f1": 0.5124, + "falseNegative": 118, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 62, + "recall": 0.3444, + "total": 2970, + "trueNegative": 2790, + "truePositive": 62, + "wilsonPrecisionLower95": 0.9417 + }, + "question": { + "f1": 0.5488, + "falseNegative": 391, + "falsePositive": 2, + "precision": 0.9917, + "predictedPositive": 241, + "recall": 0.3794, + "total": 2970, + "trueNegative": 2338, + "truePositive": 239, + "wilsonPrecisionLower95": 0.9703 + }, + "replyableMessage": { + "f1": 0.7216, + "falseNegative": 808, + "falsePositive": 27, + "precision": 0.9757, + "predictedPositive": 1109, + "recall": 0.5725, + "total": 2970, + "trueNegative": 1053, + "truePositive": 1082, + "wilsonPrecisionLower95": 0.9648 + }, + "scheduleNegotiation": { + "f1": 0.882, + "falseNegative": 19, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 71, + "recall": 0.7889, + "total": 2970, + "trueNegative": 2880, + "truePositive": 71, + "wilsonPrecisionLower95": 0.9487 + }, + "task": { + "f1": 0.7201, + "falseNegative": 298, + "falsePositive": 30, + "precision": 0.9336, + "predictedPositive": 452, + "recall": 0.5861, + "total": 2970, + "trueNegative": 2220, + "truePositive": 422, + "wilsonPrecisionLower95": 0.9068 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9807, + "falseNegative": 2, + "falsePositive": 5, + "precision": 0.9727, + "predictedPositive": 183, + "recall": 0.9889, + "total": 1485, + "trueNegative": 1300, + "truePositive": 178, + "wilsonPrecisionLower95": 0.9376 + }, + "threshold": 0.765 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9288, + "falseNegative": 17, + "falsePositive": 8, + "precision": 0.9532, + "predictedPositive": 171, + "recall": 0.9056, + "total": 1485, + "trueNegative": 1297, + "truePositive": 163, + "wilsonPrecisionLower95": 0.9104 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.9455, + "falseNegative": 22, + "falsePositive": 17, + "precision": 0.9521, + "predictedPositive": 355, + "recall": 0.9389, + "total": 2970, + "trueNegative": 2593, + "truePositive": 338, + "wilsonPrecisionLower95": 0.9247 + }, + "threshold": 0.595 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.7568, + "falseNegative": 34, + "falsePositive": 2, + "precision": 0.9655, + "predictedPositive": 58, + "recall": 0.6222, + "total": 1485, + "trueNegative": 1393, + "truePositive": 56, + "wilsonPrecisionLower95": 0.8827 + }, + "threshold": 0.955 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7733, + "falseNegative": 32, + "falsePositive": 2, + "precision": 0.9667, + "predictedPositive": 60, + "recall": 0.6444, + "total": 1485, + "trueNegative": 1393, + "truePositive": 58, + "wilsonPrecisionLower95": 0.8864 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.7723, + "falseNegative": 63, + "falsePositive": 6, + "precision": 0.9512, + "predictedPositive": 123, + "recall": 0.65, + "total": 2970, + "trueNegative": 2784, + "truePositive": 117, + "wilsonPrecisionLower95": 0.8977 + }, + "threshold": 0.765 + }, + "followUpReminder": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8483, + "falseNegative": 43, + "falsePositive": 6, + "precision": 0.958, + "predictedPositive": 143, + "recall": 0.7611, + "total": 1485, + "trueNegative": 1299, + "truePositive": 137, + "wilsonPrecisionLower95": 0.9115 + }, + "threshold": 0.995 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.3529, + "falseNegative": 141, + "falsePositive": 2, + "precision": 0.9512, + "predictedPositive": 41, + "recall": 0.2167, + "total": 1485, + "trueNegative": 1303, + "truePositive": 39, + "wilsonPrecisionLower95": 0.8386 + }, + "threshold": 0.99 + } + }, + "metrics": { + "f1": 0.6292, + "falseNegative": 192, + "falsePositive": 6, + "precision": 0.9655, + "predictedPositive": 174, + "recall": 0.4667, + "total": 2970, + "trueNegative": 2604, + "truePositive": 168, + "wilsonPrecisionLower95": 0.9268 + }, + "threshold": 0.995 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.3894, + "falseNegative": 68, + "falsePositive": 1, + "precision": 0.9565, + "predictedPositive": 23, + "recall": 0.2444, + "total": 1485, + "trueNegative": 1394, + "truePositive": 22, + "wilsonPrecisionLower95": 0.7901 + }, + "threshold": 0.98 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8175, + "falseNegative": 91, + "falsePositive": 9, + "precision": 0.9614, + "predictedPositive": 233, + "recall": 0.7111, + "total": 1485, + "trueNegative": 1161, + "truePositive": 224, + "wilsonPrecisionLower95": 0.9282 + }, + "threshold": 0.6 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.7182, + "falseNegative": 401, + "falsePositive": 26, + "precision": 0.9544, + "predictedPositive": 570, + "recall": 0.5757, + "total": 1485, + "trueNegative": 514, + "truePositive": 544, + "wilsonPrecisionLower95": 0.934 + }, + "threshold": 0.985 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6667, + "falseNegative": 460, + "falsePositive": 25, + "precision": 0.951, + "predictedPositive": 510, + "recall": 0.5132, + "total": 1485, + "trueNegative": 515, + "truePositive": 485, + "wilsonPrecisionLower95": 0.9286 + }, + "threshold": 0.945 + } + }, + "metrics": { + "f1": 0.7151, + "falseNegative": 807, + "falsePositive": 56, + "precision": 0.9508, + "predictedPositive": 1139, + "recall": 0.573, + "total": 2970, + "trueNegative": 1024, + "truePositive": 1083, + "wilsonPrecisionLower95": 0.9367 + }, + "threshold": 0.96 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9412, + "falseNegative": 5, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 40, + "recall": 0.8889, + "total": 1485, + "trueNegative": 1440, + "truePositive": 40, + "wilsonPrecisionLower95": 0.9124 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8052, + "falseNegative": 14, + "falsePositive": 1, + "precision": 0.9688, + "predictedPositive": 32, + "recall": 0.6889, + "total": 1485, + "trueNegative": 1439, + "truePositive": 31, + "wilsonPrecisionLower95": 0.8426 + }, + "threshold": 0.98 + } + }, + "metrics": { + "f1": 0.8375, + "falseNegative": 23, + "falsePositive": 3, + "precision": 0.9571, + "predictedPositive": 70, + "recall": 0.7444, + "total": 2970, + "trueNegative": 2877, + "truePositive": 67, + "wilsonPrecisionLower95": 0.8814 + }, + "threshold": 0.92 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.7783, + "falseNegative": 123, + "falsePositive": 12, + "precision": 0.9518, + "predictedPositive": 249, + "recall": 0.6583, + "total": 1485, + "trueNegative": 1113, + "truePositive": 237, + "wilsonPrecisionLower95": 0.9177 + }, + "threshold": 0.84 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7994, + "falseNegative": 113, + "falsePositive": 11, + "precision": 0.9574, + "predictedPositive": 258, + "recall": 0.6861, + "total": 1485, + "trueNegative": 1114, + "truePositive": 247, + "wilsonPrecisionLower95": 0.9253 + }, + "threshold": 0.755 + } + }, + "metrics": { + "f1": 0.7977, + "falseNegative": 225, + "falsePositive": 26, + "precision": 0.9501, + "predictedPositive": 521, + "recall": 0.6875, + "total": 2970, + "trueNegative": 2224, + "truePositive": 495, + "wilsonPrecisionLower95": 0.9279 + }, + "threshold": 0.77 + } + }, + "trainingRecords": 17763, + "validation": { + "intentsMeeting95Precision": 8, + "macroF1": 0.6563, + "macroPrecision": 0.9695, + "macroRecall": 0.5386, + "minimumPrecision": 0.9528, + "perIntent": { + "blessing": { + "f1": 0.1818, + "falseNegative": 81, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 9, + "recall": 0.1, + "total": 2970, + "trueNegative": 2880, + "truePositive": 9, + "wilsonPrecisionLower95": 0.7009 + }, + "complaint": { + "f1": 0.9552, + "falseNegative": 19, + "falsePositive": 13, + "precision": 0.9633, + "predictedPositive": 354, + "recall": 0.9472, + "total": 2970, + "trueNegative": 2597, + "truePositive": 341, + "wilsonPrecisionLower95": 0.9382 + }, + "confirmationDecision": { + "f1": 0.7651, + "falseNegative": 66, + "falsePositive": 4, + "precision": 0.9661, + "predictedPositive": 118, + "recall": 0.6333, + "total": 2970, + "trueNegative": 2786, + "truePositive": 114, + "wilsonPrecisionLower95": 0.9161 + }, + "followUpReminder": { + "f1": 0.6471, + "falseNegative": 184, + "falsePositive": 8, + "precision": 0.9565, + "predictedPositive": 184, + "recall": 0.4889, + "total": 2970, + "trueNegative": 2602, + "truePositive": 176, + "wilsonPrecisionLower95": 0.9166 + }, + "invitation": { + "f1": 0.479, + "falseNegative": 123, + "falsePositive": 1, + "precision": 0.9828, + "predictedPositive": 58, + "recall": 0.3167, + "total": 2970, + "trueNegative": 2789, + "truePositive": 57, + "wilsonPrecisionLower95": 0.9086 + }, + "question": { + "f1": 0.5191, + "falseNegative": 406, + "falsePositive": 9, + "precision": 0.9614, + "predictedPositive": 233, + "recall": 0.3556, + "total": 2970, + "trueNegative": 2331, + "truePositive": 224, + "wilsonPrecisionLower95": 0.9282 + }, + "replyableMessage": { + "f1": 0.6929, + "falseNegative": 861, + "falsePositive": 51, + "precision": 0.9528, + "predictedPositive": 1080, + "recall": 0.5444, + "total": 2970, + "trueNegative": 1029, + "truePositive": 1029, + "wilsonPrecisionLower95": 0.9384 + }, + "scheduleNegotiation": { + "f1": 0.8765, + "falseNegative": 19, + "falsePositive": 1, + "precision": 0.9861, + "predictedPositive": 72, + "recall": 0.7889, + "total": 2970, + "trueNegative": 2879, + "truePositive": 71, + "wilsonPrecisionLower95": 0.9254 + }, + "task": { + "f1": 0.7896, + "falseNegative": 236, + "falsePositive": 22, + "precision": 0.9565, + "predictedPositive": 506, + "recall": 0.6722, + "total": 2970, + "trueNegative": 2228, + "truePositive": 484, + "wilsonPrecisionLower95": 0.935 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 5e-06, + "augmentation": "prefix", + "char_max": 6, + "char_min": 2, + "external_weight": 0.77, + "hard_example_weight": 1.8, + "l1_ratio": 0.1, + "max_char_features": 70000, + "max_word_features": 30000, + "min_df": 2, + "pseudo_weight": 0.31499999999999995, + "round": 11, + "word_max": 2 + }, + "featureCount": 81451, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 0 + }, + "complaint": { + "negative": 250, + "positive": 0 + }, + "confirmationDecision": { + "negative": 250, + "positive": 14 + }, + "followUpReminder": { + "negative": 250, + "positive": 2 + }, + "invitation": { + "negative": 250, + "positive": 1 + }, + "question": { + "negative": 233, + "positive": 0 + }, + "replyableMessage": { + "negative": 8, + "positive": 250 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 0 + }, + "task": { + "negative": 250, + "positive": 0 + } + }, + "round": 11, + "selectionObjective": 0.8015, + "syntheticRegression": { + "intentsMeeting95Precision": 4, + "macroF1": 0.6807, + "macroPrecision": 0.963, + "macroRecall": 0.6053, + "minimumPrecision": 0.8782, + "perIntent": { + "blessing": { + "f1": 0.125, + "falseNegative": 84, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 6, + "recall": 0.0667, + "total": 2970, + "trueNegative": 2880, + "truePositive": 6, + "wilsonPrecisionLower95": 0.6097 + }, + "complaint": { + "f1": 0.9655, + "falseNegative": 24, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 336, + "recall": 0.9333, + "total": 2970, + "trueNegative": 2610, + "truePositive": 336, + "wilsonPrecisionLower95": 0.9887 + }, + "confirmationDecision": { + "f1": 0.8155, + "falseNegative": 43, + "falsePositive": 19, + "precision": 0.8782, + "predictedPositive": 156, + "recall": 0.7611, + "total": 2970, + "trueNegative": 2771, + "truePositive": 137, + "wilsonPrecisionLower95": 0.8176 + }, + "followUpReminder": { + "f1": 0.2439, + "falseNegative": 310, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 50, + "recall": 0.1389, + "total": 2970, + "trueNegative": 2610, + "truePositive": 50, + "wilsonPrecisionLower95": 0.9287 + }, + "invitation": { + "f1": 0.5425, + "falseNegative": 113, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 67, + "recall": 0.3722, + "total": 2970, + "trueNegative": 2790, + "truePositive": 67, + "wilsonPrecisionLower95": 0.9458 + }, + "question": { + "f1": 0.8494, + "falseNegative": 162, + "falsePositive": 4, + "precision": 0.9915, + "predictedPositive": 472, + "recall": 0.7429, + "total": 2970, + "trueNegative": 2336, + "truePositive": 468, + "wilsonPrecisionLower95": 0.9784 + }, + "replyableMessage": { + "f1": 0.8975, + "falseNegative": 253, + "falsePositive": 121, + "precision": 0.9312, + "predictedPositive": 1758, + "recall": 0.8661, + "total": 2970, + "trueNegative": 959, + "truePositive": 1637, + "wilsonPrecisionLower95": 0.9184 + }, + "scheduleNegotiation": { + "f1": 0.9444, + "falseNegative": 5, + "falsePositive": 5, + "precision": 0.9444, + "predictedPositive": 90, + "recall": 0.9444, + "total": 2970, + "trueNegative": 2875, + "truePositive": 85, + "wilsonPrecisionLower95": 0.8765 + }, + "task": { + "f1": 0.743, + "falseNegative": 272, + "falsePositive": 38, + "precision": 0.9218, + "predictedPositive": 486, + "recall": 0.6222, + "total": 2970, + "trueNegative": 2212, + "truePositive": 448, + "wilsonPrecisionLower95": 0.8945 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9917, + "falseNegative": 0, + "falsePositive": 3, + "precision": 0.9836, + "predictedPositive": 183, + "recall": 1.0, + "total": 1485, + "trueNegative": 1302, + "truePositive": 180, + "wilsonPrecisionLower95": 0.9529 + }, + "threshold": 0.99 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9318, + "falseNegative": 16, + "falsePositive": 8, + "precision": 0.9535, + "predictedPositive": 172, + "recall": 0.9111, + "total": 1485, + "trueNegative": 1297, + "truePositive": 164, + "wilsonPrecisionLower95": 0.9109 + }, + "threshold": 0.84 + } + }, + "metrics": { + "f1": 0.9529, + "falseNegative": 16, + "falsePositive": 18, + "precision": 0.9503, + "predictedPositive": 362, + "recall": 0.9556, + "total": 2970, + "trueNegative": 2592, + "truePositive": 344, + "wilsonPrecisionLower95": 0.9228 + }, + "threshold": 0.84 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.931, + "falseNegative": 9, + "falsePositive": 3, + "precision": 0.9643, + "predictedPositive": 84, + "recall": 0.9, + "total": 1485, + "trueNegative": 1392, + "truePositive": 81, + "wilsonPrecisionLower95": 0.9002 + }, + "threshold": 0.97 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7222, + "falseNegative": 38, + "falsePositive": 2, + "precision": 0.963, + "predictedPositive": 54, + "recall": 0.5778, + "total": 1485, + "trueNegative": 1393, + "truePositive": 52, + "wilsonPrecisionLower95": 0.8746 + }, + "threshold": 0.86 + } + }, + "metrics": { + "f1": 0.8077, + "falseNegative": 54, + "falsePositive": 6, + "precision": 0.9545, + "predictedPositive": 132, + "recall": 0.7, + "total": 2970, + "trueNegative": 2784, + "truePositive": 126, + "wilsonPrecisionLower95": 0.9044 + }, + "threshold": 0.95 + }, + "followUpReminder": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.4833, + "falseNegative": 61, + "falsePositive": 1, + "precision": 0.9667, + "predictedPositive": 30, + "recall": 0.3222, + "total": 1485, + "trueNegative": 1394, + "truePositive": 29, + "wilsonPrecisionLower95": 0.8333 + }, + "threshold": 0.965 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.7993, + "falseNegative": 98, + "falsePositive": 11, + "precision": 0.9518, + "predictedPositive": 228, + "recall": 0.6889, + "total": 1485, + "trueNegative": 1159, + "truePositive": 217, + "wilsonPrecisionLower95": 0.9157 + }, + "threshold": 0.92 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7572, + "falseNegative": 117, + "falsePositive": 10, + "precision": 0.9519, + "predictedPositive": 208, + "recall": 0.6286, + "total": 1485, + "trueNegative": 1160, + "truePositive": 198, + "wilsonPrecisionLower95": 0.9138 + }, + "threshold": 0.735 + } + }, + "metrics": { + "f1": 0.797, + "falseNegative": 198, + "falsePositive": 22, + "precision": 0.9515, + "predictedPositive": 454, + "recall": 0.6857, + "total": 2970, + "trueNegative": 2318, + "truePositive": 432, + "wilsonPrecisionLower95": 0.9277 + }, + "threshold": 0.8 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9373, + "falseNegative": 71, + "falsePositive": 46, + "precision": 0.95, + "predictedPositive": 920, + "recall": 0.9249, + "total": 1485, + "trueNegative": 494, + "truePositive": 874, + "wilsonPrecisionLower95": 0.934 + }, + "threshold": 0.79 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8754, + "falseNegative": 179, + "falsePositive": 39, + "precision": 0.9516, + "predictedPositive": 805, + "recall": 0.8106, + "total": 1485, + "trueNegative": 501, + "truePositive": 766, + "wilsonPrecisionLower95": 0.9345 + }, + "threshold": 0.91 + } + }, + "metrics": { + "f1": 0.9105, + "falseNegative": 242, + "falsePositive": 82, + "precision": 0.9526, + "predictedPositive": 1730, + "recall": 0.872, + "total": 2970, + "trueNegative": 998, + "truePositive": 1648, + "wilsonPrecisionLower95": 0.9415 + }, + "threshold": 0.895 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9773, + "falseNegative": 2, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 43, + "recall": 0.9556, + "total": 1485, + "trueNegative": 1440, + "truePositive": 43, + "wilsonPrecisionLower95": 0.918 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9318, + "falseNegative": 4, + "falsePositive": 2, + "precision": 0.9535, + "predictedPositive": 43, + "recall": 0.9111, + "total": 1485, + "trueNegative": 1438, + "truePositive": 41, + "wilsonPrecisionLower95": 0.8454 + }, + "threshold": 0.9 + } + }, + "metrics": { + "f1": 0.9551, + "falseNegative": 5, + "falsePositive": 3, + "precision": 0.9659, + "predictedPositive": 88, + "recall": 0.9444, + "total": 2970, + "trueNegative": 2877, + "truePositive": 85, + "wilsonPrecisionLower95": 0.9045 + }, + "threshold": 0.885 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8315, + "falseNegative": 96, + "falsePositive": 11, + "precision": 0.96, + "predictedPositive": 275, + "recall": 0.7333, + "total": 1485, + "trueNegative": 1114, + "truePositive": 264, + "wilsonPrecisionLower95": 0.9298 + }, + "threshold": 0.985 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8349, + "falseNegative": 92, + "falsePositive": 14, + "precision": 0.9504, + "predictedPositive": 282, + "recall": 0.7444, + "total": 1485, + "trueNegative": 1111, + "truePositive": 268, + "wilsonPrecisionLower95": 0.9184 + }, + "threshold": 0.845 + } + }, + "metrics": { + "f1": 0.7688, + "falseNegative": 256, + "falsePositive": 23, + "precision": 0.9528, + "predictedPositive": 487, + "recall": 0.6444, + "total": 2970, + "trueNegative": 2227, + "truePositive": 464, + "wilsonPrecisionLower95": 0.9301 + }, + "threshold": 0.975 + } + }, + "trainingRecords": 15345, + "validation": { + "intentsMeeting95Precision": 9, + "macroF1": 0.7269, + "macroPrecision": 0.9659, + "macroRecall": 0.6366, + "minimumPrecision": 0.9507, + "perIntent": { + "blessing": { + "f1": 0.2692, + "falseNegative": 76, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 14, + "recall": 0.1556, + "total": 2970, + "trueNegative": 2880, + "truePositive": 14, + "wilsonPrecisionLower95": 0.7847 + }, + "complaint": { + "f1": 0.9622, + "falseNegative": 16, + "falsePositive": 11, + "precision": 0.969, + "predictedPositive": 355, + "recall": 0.9556, + "total": 2970, + "trueNegative": 2599, + "truePositive": 344, + "wilsonPrecisionLower95": 0.9454 + }, + "confirmationDecision": { + "f1": 0.8365, + "falseNegative": 47, + "falsePositive": 5, + "precision": 0.9638, + "predictedPositive": 138, + "recall": 0.7389, + "total": 2970, + "trueNegative": 2785, + "truePositive": 133, + "wilsonPrecisionLower95": 0.918 + }, + "followUpReminder": { + "f1": 0.4335, + "falseNegative": 259, + "falsePositive": 5, + "precision": 0.9528, + "predictedPositive": 106, + "recall": 0.2806, + "total": 2970, + "trueNegative": 2605, + "truePositive": 101, + "wilsonPrecisionLower95": 0.8943 + }, + "invitation": { + "f1": 0.5669, + "falseNegative": 108, + "falsePositive": 2, + "precision": 0.973, + "predictedPositive": 74, + "recall": 0.4, + "total": 2970, + "trueNegative": 2788, + "truePositive": 72, + "wilsonPrecisionLower95": 0.9067 + }, + "question": { + "f1": 0.7786, + "falseNegative": 215, + "falsePositive": 21, + "precision": 0.9518, + "predictedPositive": 436, + "recall": 0.6587, + "total": 2970, + "trueNegative": 2319, + "truePositive": 415, + "wilsonPrecisionLower95": 0.9275 + }, + "replyableMessage": { + "f1": 0.9073, + "falseNegative": 250, + "falsePositive": 85, + "precision": 0.9507, + "predictedPositive": 1725, + "recall": 0.8677, + "total": 2970, + "trueNegative": 995, + "truePositive": 1640, + "wilsonPrecisionLower95": 0.9395 + }, + "scheduleNegotiation": { + "f1": 0.9545, + "falseNegative": 6, + "falsePositive": 2, + "precision": 0.9767, + "predictedPositive": 86, + "recall": 0.9333, + "total": 2970, + "trueNegative": 2878, + "truePositive": 84, + "wilsonPrecisionLower95": 0.9191 + }, + "task": { + "f1": 0.8332, + "falseNegative": 188, + "falsePositive": 25, + "precision": 0.9551, + "predictedPositive": 557, + "recall": 0.7389, + "total": 2970, + "trueNegative": 2225, + "truePositive": 532, + "wilsonPrecisionLower95": 0.9346 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 4e-06, + "augmentation": "prefix", + "char_max": 6, + "char_min": 3, + "external_weight": 0.79, + "hard_example_weight": 2.0, + "l1_ratio": 0.15, + "max_char_features": 75000, + "max_word_features": 35000, + "min_df": 2, + "pseudo_weight": 0.32999999999999996, + "round": 12, + "word_max": 3 + }, + "featureCount": 94458, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 0 + }, + "complaint": { + "negative": 250, + "positive": 0 + }, + "confirmationDecision": { + "negative": 250, + "positive": 15 + }, + "followUpReminder": { + "negative": 250, + "positive": 0 + }, + "invitation": { + "negative": 250, + "positive": 0 + }, + "question": { + "negative": 232, + "positive": 0 + }, + "replyableMessage": { + "negative": 10, + "positive": 250 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 0 + }, + "task": { + "negative": 250, + "positive": 0 + } + }, + "round": 12, + "selectionObjective": 0.7563, + "syntheticRegression": { + "intentsMeeting95Precision": 5, + "macroF1": 0.6379, + "macroPrecision": 0.9476, + "macroRecall": 0.5322, + "minimumPrecision": 0.8452, + "perIntent": { + "blessing": { + "f1": 0.1053, + "falseNegative": 85, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 5, + "recall": 0.0556, + "total": 2970, + "trueNegative": 2880, + "truePositive": 5, + "wilsonPrecisionLower95": 0.5655 + }, + "complaint": { + "f1": 0.9625, + "falseNegative": 26, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 334, + "recall": 0.9278, + "total": 2970, + "trueNegative": 2610, + "truePositive": 334, + "wilsonPrecisionLower95": 0.9886 + }, + "confirmationDecision": { + "f1": 0.7821, + "falseNegative": 49, + "falsePositive": 24, + "precision": 0.8452, + "predictedPositive": 155, + "recall": 0.7278, + "total": 2970, + "trueNegative": 2766, + "truePositive": 131, + "wilsonPrecisionLower95": 0.7799 + }, + "followUpReminder": { + "f1": 0.6142, + "falseNegative": 200, + "falsePositive": 1, + "precision": 0.9938, + "predictedPositive": 161, + "recall": 0.4444, + "total": 2970, + "trueNegative": 2609, + "truePositive": 160, + "wilsonPrecisionLower95": 0.9657 + }, + "invitation": { + "f1": 0.5992, + "falseNegative": 103, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 77, + "recall": 0.4278, + "total": 2970, + "trueNegative": 2790, + "truePositive": 77, + "wilsonPrecisionLower95": 0.9525 + }, + "question": { + "f1": 0.4319, + "falseNegative": 454, + "falsePositive": 9, + "precision": 0.9514, + "predictedPositive": 185, + "recall": 0.2794, + "total": 2970, + "trueNegative": 2331, + "truePositive": 176, + "wilsonPrecisionLower95": 0.9101 + }, + "replyableMessage": { + "f1": 0.7888, + "falseNegative": 624, + "falsePositive": 54, + "precision": 0.9591, + "predictedPositive": 1320, + "recall": 0.6698, + "total": 2970, + "trueNegative": 1026, + "truePositive": 1266, + "wilsonPrecisionLower95": 0.947 + }, + "scheduleNegotiation": { + "f1": 0.8144, + "falseNegative": 22, + "falsePositive": 9, + "precision": 0.8831, + "predictedPositive": 77, + "recall": 0.7556, + "total": 2970, + "trueNegative": 2871, + "truePositive": 68, + "wilsonPrecisionLower95": 0.7925 + }, + "task": { + "f1": 0.6429, + "falseNegative": 359, + "falsePositive": 42, + "precision": 0.8958, + "predictedPositive": 403, + "recall": 0.5014, + "total": 2970, + "trueNegative": 2208, + "truePositive": 361, + "wilsonPrecisionLower95": 0.8621 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9917, + "falseNegative": 0, + "falsePositive": 3, + "precision": 0.9836, + "predictedPositive": 183, + "recall": 1.0, + "total": 1485, + "trueNegative": 1302, + "truePositive": 180, + "wilsonPrecisionLower95": 0.9529 + }, + "threshold": 0.885 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8974, + "falseNegative": 27, + "falsePositive": 8, + "precision": 0.9503, + "predictedPositive": 161, + "recall": 0.85, + "total": 1485, + "trueNegative": 1297, + "truePositive": 153, + "wilsonPrecisionLower95": 0.905 + }, + "threshold": 0.9 + } + }, + "metrics": { + "f1": 0.9515, + "falseNegative": 17, + "falsePositive": 18, + "precision": 0.9501, + "predictedPositive": 361, + "recall": 0.9528, + "total": 2970, + "trueNegative": 2592, + "truePositive": 343, + "wilsonPrecisionLower95": 0.9226 + }, + "threshold": 0.77 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9944, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 89, + "recall": 0.9889, + "total": 1485, + "trueNegative": 1395, + "truePositive": 89, + "wilsonPrecisionLower95": 0.9586 + }, + "threshold": 0.915 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.4959, + "falseNegative": 60, + "falsePositive": 1, + "precision": 0.9677, + "predictedPositive": 31, + "recall": 0.3333, + "total": 1485, + "trueNegative": 1394, + "truePositive": 30, + "wilsonPrecisionLower95": 0.8381 + }, + "threshold": 0.985 + } + }, + "metrics": { + "f1": 0.8217, + "falseNegative": 51, + "falsePositive": 5, + "precision": 0.9627, + "predictedPositive": 134, + "recall": 0.7167, + "total": 2970, + "trueNegative": 2785, + "truePositive": 129, + "wilsonPrecisionLower95": 0.9156 + }, + "threshold": 0.915 + }, + "followUpReminder": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.5647, + "falseNegative": 108, + "falsePositive": 3, + "precision": 0.96, + "predictedPositive": 75, + "recall": 0.4, + "total": 1485, + "trueNegative": 1302, + "truePositive": 72, + "wilsonPrecisionLower95": 0.8889 + }, + "threshold": 0.995 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6542, + "falseNegative": 158, + "falsePositive": 8, + "precision": 0.9515, + "predictedPositive": 165, + "recall": 0.4984, + "total": 1485, + "trueNegative": 1162, + "truePositive": 157, + "wilsonPrecisionLower95": 0.9073 + }, + "threshold": 0.805 + } + }, + "metrics": { + "f1": 0.5711, + "falseNegative": 373, + "falsePositive": 13, + "precision": 0.9519, + "predictedPositive": 270, + "recall": 0.4079, + "total": 2970, + "trueNegative": 2327, + "truePositive": 257, + "wilsonPrecisionLower95": 0.9194 + }, + "threshold": 0.985 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8956, + "falseNegative": 147, + "falsePositive": 39, + "precision": 0.9534, + "predictedPositive": 837, + "recall": 0.8444, + "total": 1485, + "trueNegative": 501, + "truePositive": 798, + "wilsonPrecisionLower95": 0.9369 + }, + "threshold": 0.975 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6735, + "falseNegative": 453, + "falsePositive": 24, + "precision": 0.9535, + "predictedPositive": 516, + "recall": 0.5206, + "total": 1485, + "trueNegative": 516, + "truePositive": 492, + "wilsonPrecisionLower95": 0.9317 + }, + "threshold": 0.985 + } + }, + "metrics": { + "f1": 0.7866, + "falseNegative": 626, + "falsePositive": 60, + "precision": 0.9547, + "predictedPositive": 1324, + "recall": 0.6688, + "total": 2970, + "trueNegative": 1020, + "truePositive": 1264, + "wilsonPrecisionLower95": 0.9421 + }, + "threshold": 0.985 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9888, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 44, + "recall": 0.9778, + "total": 1485, + "trueNegative": 1440, + "truePositive": 44, + "wilsonPrecisionLower95": 0.9197 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8052, + "falseNegative": 14, + "falsePositive": 1, + "precision": 0.9688, + "predictedPositive": 32, + "recall": 0.6889, + "total": 1485, + "trueNegative": 1439, + "truePositive": 31, + "wilsonPrecisionLower95": 0.8426 + }, + "threshold": 0.94 + } + }, + "metrics": { + "f1": 0.9364, + "falseNegative": 9, + "falsePositive": 2, + "precision": 0.9759, + "predictedPositive": 83, + "recall": 0.9, + "total": 2970, + "trueNegative": 2878, + "truePositive": 81, + "wilsonPrecisionLower95": 0.9163 + }, + "threshold": 0.715 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8475, + "falseNegative": 85, + "falsePositive": 14, + "precision": 0.9516, + "predictedPositive": 289, + "recall": 0.7639, + "total": 1485, + "trueNegative": 1111, + "truePositive": 275, + "wilsonPrecisionLower95": 0.9203 + }, + "threshold": 0.985 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.501, + "falseNegative": 238, + "falsePositive": 5, + "precision": 0.9606, + "predictedPositive": 127, + "recall": 0.3389, + "total": 1485, + "trueNegative": 1120, + "truePositive": 122, + "wilsonPrecisionLower95": 0.9111 + }, + "threshold": 0.985 + } + }, + "metrics": { + "f1": 0.6989, + "falseNegative": 323, + "falsePositive": 19, + "precision": 0.9543, + "predictedPositive": 416, + "recall": 0.5514, + "total": 2970, + "trueNegative": 2231, + "truePositive": 397, + "wilsonPrecisionLower95": 0.9298 + }, + "threshold": 0.985 + } + }, + "trainingRecords": 15345, + "validation": { + "intentsMeeting95Precision": 8, + "macroF1": 0.6603, + "macroPrecision": 0.9686, + "macroRecall": 0.5413, + "minimumPrecision": 0.9298, + "perIntent": { + "blessing": { + "f1": 0.2857, + "falseNegative": 75, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 15, + "recall": 0.1667, + "total": 2970, + "trueNegative": 2880, + "truePositive": 15, + "wilsonPrecisionLower95": 0.7961 + }, + "complaint": { + "f1": 0.946, + "falseNegative": 27, + "falsePositive": 11, + "precision": 0.968, + "predictedPositive": 344, + "recall": 0.925, + "total": 2970, + "trueNegative": 2599, + "truePositive": 333, + "wilsonPrecisionLower95": 0.9437 + }, + "confirmationDecision": { + "f1": 0.7933, + "falseNegative": 61, + "falsePositive": 1, + "precision": 0.9917, + "predictedPositive": 120, + "recall": 0.6611, + "total": 2970, + "trueNegative": 2789, + "truePositive": 119, + "wilsonPrecisionLower95": 0.9543 + }, + "followUpReminder": { + "f1": 0.6494, + "falseNegative": 185, + "falsePositive": 4, + "precision": 0.9777, + "predictedPositive": 179, + "recall": 0.4861, + "total": 2970, + "trueNegative": 2606, + "truePositive": 175, + "wilsonPrecisionLower95": 0.944 + }, + "invitation": { + "f1": 0.4473, + "falseNegative": 127, + "falsePositive": 4, + "precision": 0.9298, + "predictedPositive": 57, + "recall": 0.2944, + "total": 2970, + "trueNegative": 2786, + "truePositive": 53, + "wilsonPrecisionLower95": 0.833 + }, + "question": { + "f1": 0.4227, + "falseNegative": 459, + "falsePositive": 8, + "precision": 0.9553, + "predictedPositive": 179, + "recall": 0.2714, + "total": 2970, + "trueNegative": 2332, + "truePositive": 171, + "wilsonPrecisionLower95": 0.9143 + }, + "replyableMessage": { + "f1": 0.7956, + "falseNegative": 600, + "falsePositive": 63, + "precision": 0.9534, + "predictedPositive": 1353, + "recall": 0.6825, + "total": 2970, + "trueNegative": 1017, + "truePositive": 1290, + "wilsonPrecisionLower95": 0.9409 + }, + "scheduleNegotiation": { + "f1": 0.9036, + "falseNegative": 15, + "falsePositive": 1, + "precision": 0.9868, + "predictedPositive": 76, + "recall": 0.8333, + "total": 2970, + "trueNegative": 2879, + "truePositive": 75, + "wilsonPrecisionLower95": 0.9292 + }, + "task": { + "f1": 0.6989, + "falseNegative": 323, + "falsePositive": 19, + "precision": 0.9543, + "predictedPositive": 416, + "recall": 0.5514, + "total": 2970, + "trueNegative": 2231, + "truePositive": 397, + "wilsonPrecisionLower95": 0.9298 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 3e-06, + "augmentation": "prefix", + "char_max": 7, + "char_min": 2, + "external_weight": 0.81, + "hard_example_weight": 2.0, + "l1_ratio": 0.15, + "max_char_features": 80000, + "max_word_features": 38000, + "min_df": 2, + "pseudo_weight": 0.345, + "round": 13, + "word_max": 3 + }, + "featureCount": 99458, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 0 + }, + "complaint": { + "negative": 250, + "positive": 1 + }, + "confirmationDecision": { + "negative": 250, + "positive": 18 + }, + "followUpReminder": { + "negative": 250, + "positive": 0 + }, + "invitation": { + "negative": 250, + "positive": 0 + }, + "question": { + "negative": 250, + "positive": 0 + }, + "replyableMessage": { + "negative": 12, + "positive": 250 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 0 + }, + "task": { + "negative": 250, + "positive": 0 + } + }, + "round": 13, + "selectionObjective": 0.768, + "syntheticRegression": { + "intentsMeeting95Precision": 4, + "macroF1": 0.6577, + "macroPrecision": 0.9568, + "macroRecall": 0.5722, + "minimumPrecision": 0.859, + "perIntent": { + "blessing": { + "f1": 0.1443, + "falseNegative": 83, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 7, + "recall": 0.0778, + "total": 2970, + "trueNegative": 2880, + "truePositive": 7, + "wilsonPrecisionLower95": 0.6457 + }, + "complaint": { + "f1": 0.9744, + "falseNegative": 17, + "falsePositive": 1, + "precision": 0.9971, + "predictedPositive": 344, + "recall": 0.9528, + "total": 2970, + "trueNegative": 2609, + "truePositive": 343, + "wilsonPrecisionLower95": 0.9837 + }, + "confirmationDecision": { + "f1": 0.7976, + "falseNegative": 46, + "falsePositive": 22, + "precision": 0.859, + "predictedPositive": 156, + "recall": 0.7444, + "total": 2970, + "trueNegative": 2768, + "truePositive": 134, + "wilsonPrecisionLower95": 0.7957 + }, + "followUpReminder": { + "f1": 0.2972, + "falseNegative": 297, + "falsePositive": 1, + "precision": 0.9844, + "predictedPositive": 64, + "recall": 0.175, + "total": 2970, + "trueNegative": 2609, + "truePositive": 63, + "wilsonPrecisionLower95": 0.9167 + }, + "invitation": { + "f1": 0.6077, + "falseNegative": 101, + "falsePositive": 1, + "precision": 0.9875, + "predictedPositive": 80, + "recall": 0.4389, + "total": 2970, + "trueNegative": 2789, + "truePositive": 79, + "wilsonPrecisionLower95": 0.9325 + }, + "question": { + "f1": 0.5229, + "falseNegative": 407, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 223, + "recall": 0.354, + "total": 2970, + "trueNegative": 2340, + "truePositive": 223, + "wilsonPrecisionLower95": 0.9831 + }, + "replyableMessage": { + "f1": 0.9179, + "falseNegative": 202, + "falsePositive": 100, + "precision": 0.9441, + "predictedPositive": 1788, + "recall": 0.8931, + "total": 2970, + "trueNegative": 980, + "truePositive": 1688, + "wilsonPrecisionLower95": 0.9324 + }, + "scheduleNegotiation": { + "f1": 0.8889, + "falseNegative": 14, + "falsePositive": 5, + "precision": 0.9383, + "predictedPositive": 81, + "recall": 0.8444, + "total": 2970, + "trueNegative": 2875, + "truePositive": 76, + "wilsonPrecisionLower95": 0.8635 + }, + "task": { + "f1": 0.7681, + "falseNegative": 238, + "falsePositive": 53, + "precision": 0.9009, + "predictedPositive": 535, + "recall": 0.6694, + "total": 2970, + "trueNegative": 2197, + "truePositive": 482, + "wilsonPrecisionLower95": 0.8727 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9727, + "falseNegative": 2, + "falsePositive": 8, + "precision": 0.957, + "predictedPositive": 186, + "recall": 0.9889, + "total": 1485, + "trueNegative": 1297, + "truePositive": 178, + "wilsonPrecisionLower95": 0.9174 + }, + "threshold": 0.98 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.95, + "falseNegative": 9, + "falsePositive": 9, + "precision": 0.95, + "predictedPositive": 180, + "recall": 0.95, + "total": 1485, + "trueNegative": 1296, + "truePositive": 171, + "wilsonPrecisionLower95": 0.9077 + }, + "threshold": 0.905 + } + }, + "metrics": { + "f1": 0.9529, + "falseNegative": 16, + "falsePositive": 18, + "precision": 0.9503, + "predictedPositive": 362, + "recall": 0.9556, + "total": 2970, + "trueNegative": 2592, + "truePositive": 344, + "wilsonPrecisionLower95": 0.9228 + }, + "threshold": 0.965 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.828, + "falseNegative": 25, + "falsePositive": 2, + "precision": 0.9701, + "predictedPositive": 67, + "recall": 0.7222, + "total": 1485, + "trueNegative": 1393, + "truePositive": 65, + "wilsonPrecisionLower95": 0.8975 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6763, + "falseNegative": 43, + "falsePositive": 2, + "precision": 0.9592, + "predictedPositive": 49, + "recall": 0.5222, + "total": 1485, + "trueNegative": 1393, + "truePositive": 47, + "wilsonPrecisionLower95": 0.8629 + }, + "threshold": 0.93 + } + }, + "metrics": { + "f1": 0.7855, + "falseNegative": 61, + "falsePositive": 4, + "precision": 0.9675, + "predictedPositive": 123, + "recall": 0.6611, + "total": 2970, + "trueNegative": 2786, + "truePositive": 119, + "wilsonPrecisionLower95": 0.9194 + }, + "threshold": 0.82 + }, + "followUpReminder": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7452, + "falseNegative": 122, + "falsePositive": 10, + "precision": 0.9507, + "predictedPositive": 203, + "recall": 0.6127, + "total": 1485, + "trueNegative": 1160, + "truePositive": 193, + "wilsonPrecisionLower95": 0.9117 + }, + "threshold": 0.915 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9441, + "falseNegative": 59, + "falsePositive": 46, + "precision": 0.9506, + "predictedPositive": 932, + "recall": 0.9376, + "total": 1485, + "trueNegative": 494, + "truePositive": 886, + "wilsonPrecisionLower95": 0.9348 + }, + "threshold": 0.935 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9067, + "falseNegative": 129, + "falsePositive": 39, + "precision": 0.9544, + "predictedPositive": 855, + "recall": 0.8635, + "total": 1485, + "trueNegative": 501, + "truePositive": 816, + "wilsonPrecisionLower95": 0.9383 + }, + "threshold": 0.905 + } + }, + "metrics": { + "f1": 0.9267, + "falseNegative": 182, + "falsePositive": 88, + "precision": 0.951, + "predictedPositive": 1796, + "recall": 0.9037, + "total": 2970, + "trueNegative": 992, + "truePositive": 1708, + "wilsonPrecisionLower95": 0.94 + }, + "threshold": 0.905 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9888, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 44, + "recall": 0.9778, + "total": 1485, + "trueNegative": 1440, + "truePositive": 44, + "wilsonPrecisionLower95": 0.9197 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9655, + "falseNegative": 3, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 42, + "recall": 0.9333, + "total": 1485, + "trueNegative": 1440, + "truePositive": 42, + "wilsonPrecisionLower95": 0.9162 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.9773, + "falseNegative": 4, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 86, + "recall": 0.9556, + "total": 2970, + "trueNegative": 2880, + "truePositive": 86, + "wilsonPrecisionLower95": 0.9572 + }, + "threshold": 0.5 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.7876, + "falseNegative": 119, + "falsePositive": 11, + "precision": 0.9563, + "predictedPositive": 252, + "recall": 0.6694, + "total": 1485, + "trueNegative": 1114, + "truePositive": 241, + "wilsonPrecisionLower95": 0.9235 + }, + "threshold": 0.995 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8416, + "falseNegative": 89, + "falsePositive": 13, + "precision": 0.9542, + "predictedPositive": 284, + "recall": 0.7528, + "total": 1485, + "trueNegative": 1112, + "truePositive": 271, + "wilsonPrecisionLower95": 0.9233 + }, + "threshold": 0.895 + } + }, + "metrics": { + "f1": 0.6962, + "falseNegative": 327, + "falsePositive": 16, + "precision": 0.9609, + "predictedPositive": 409, + "recall": 0.5458, + "total": 2970, + "trueNegative": 2234, + "truePositive": 393, + "wilsonPrecisionLower95": 0.9374 + }, + "threshold": 0.995 + } + }, + "trainingRecords": 15346, + "validation": { + "intentsMeeting95Precision": 9, + "macroF1": 0.6724, + "macroPrecision": 0.9724, + "macroRecall": 0.5779, + "minimumPrecision": 0.9524, + "perIntent": { + "blessing": { + "f1": 0.2178, + "falseNegative": 79, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 11, + "recall": 0.1222, + "total": 2970, + "trueNegative": 2880, + "truePositive": 11, + "wilsonPrecisionLower95": 0.7412 + }, + "complaint": { + "f1": 0.9614, + "falseNegative": 11, + "falsePositive": 17, + "precision": 0.9536, + "predictedPositive": 366, + "recall": 0.9694, + "total": 2970, + "trueNegative": 2593, + "truePositive": 349, + "wilsonPrecisionLower95": 0.9269 + }, + "confirmationDecision": { + "f1": 0.7568, + "falseNegative": 68, + "falsePositive": 4, + "precision": 0.9655, + "predictedPositive": 116, + "recall": 0.6222, + "total": 2970, + "trueNegative": 2786, + "truePositive": 112, + "wilsonPrecisionLower95": 0.9147 + }, + "followUpReminder": { + "f1": 0.4895, + "falseNegative": 243, + "falsePositive": 1, + "precision": 0.9915, + "predictedPositive": 118, + "recall": 0.325, + "total": 2970, + "trueNegative": 2609, + "truePositive": 117, + "wilsonPrecisionLower95": 0.9536 + }, + "invitation": { + "f1": 0.4192, + "falseNegative": 132, + "falsePositive": 1, + "precision": 0.9796, + "predictedPositive": 49, + "recall": 0.2667, + "total": 2970, + "trueNegative": 2789, + "truePositive": 48, + "wilsonPrecisionLower95": 0.8931 + }, + "question": { + "f1": 0.4888, + "falseNegative": 423, + "falsePositive": 10, + "precision": 0.9539, + "predictedPositive": 217, + "recall": 0.3286, + "total": 2970, + "trueNegative": 2330, + "truePositive": 207, + "wilsonPrecisionLower95": 0.9173 + }, + "replyableMessage": { + "f1": 0.9258, + "falseNegative": 188, + "falsePositive": 85, + "precision": 0.9524, + "predictedPositive": 1787, + "recall": 0.9005, + "total": 2970, + "trueNegative": 995, + "truePositive": 1702, + "wilsonPrecisionLower95": 0.9416 + }, + "scheduleNegotiation": { + "f1": 0.9773, + "falseNegative": 4, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 86, + "recall": 0.9556, + "total": 2970, + "trueNegative": 2880, + "truePositive": 86, + "wilsonPrecisionLower95": 0.9572 + }, + "task": { + "f1": 0.8153, + "falseNegative": 208, + "falsePositive": 24, + "precision": 0.9552, + "predictedPositive": 536, + "recall": 0.7111, + "total": 2970, + "trueNegative": 2226, + "truePositive": 512, + "wilsonPrecisionLower95": 0.9342 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 2.5e-06, + "augmentation": "prefix", + "char_max": 7, + "char_min": 3, + "external_weight": 0.8300000000000001, + "hard_example_weight": 2.2, + "l1_ratio": 0.2, + "max_char_features": 80000, + "max_word_features": 40000, + "min_df": 1, + "pseudo_weight": 0.36, + "round": 14, + "word_max": 3 + }, + "featureCount": 120000, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 0 + }, + "complaint": { + "negative": 250, + "positive": 8 + }, + "confirmationDecision": { + "negative": 250, + "positive": 26 + }, + "followUpReminder": { + "negative": 250, + "positive": 0 + }, + "invitation": { + "negative": 250, + "positive": 0 + }, + "question": { + "negative": 250, + "positive": 0 + }, + "replyableMessage": { + "negative": 34, + "positive": 250 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 0 + }, + "task": { + "negative": 250, + "positive": 3 + } + }, + "round": 14, + "selectionObjective": 0.7391, + "syntheticRegression": { + "intentsMeeting95Precision": 5, + "macroF1": 0.6159, + "macroPrecision": 0.9488, + "macroRecall": 0.5171, + "minimumPrecision": 0.8315, + "perIntent": { + "blessing": { + "f1": 0.1443, + "falseNegative": 83, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 7, + "recall": 0.0778, + "total": 2970, + "trueNegative": 2880, + "truePositive": 7, + "wilsonPrecisionLower95": 0.6457 + }, + "complaint": { + "f1": 0.7987, + "falseNegative": 114, + "falsePositive": 10, + "precision": 0.9609, + "predictedPositive": 256, + "recall": 0.6833, + "total": 2970, + "trueNegative": 2600, + "truePositive": 246, + "wilsonPrecisionLower95": 0.9296 + }, + "confirmationDecision": { + "f1": 0.8407, + "falseNegative": 27, + "falsePositive": 31, + "precision": 0.8315, + "predictedPositive": 184, + "recall": 0.85, + "total": 2970, + "trueNegative": 2759, + "truePositive": 153, + "wilsonPrecisionLower95": 0.7708 + }, + "followUpReminder": { + "f1": 0.7361, + "falseNegative": 148, + "falsePositive": 4, + "precision": 0.9815, + "predictedPositive": 216, + "recall": 0.5889, + "total": 2970, + "trueNegative": 2606, + "truePositive": 212, + "wilsonPrecisionLower95": 0.9534 + }, + "invitation": { + "f1": 0.5591, + "falseNegative": 109, + "falsePositive": 3, + "precision": 0.9595, + "predictedPositive": 74, + "recall": 0.3944, + "total": 2970, + "trueNegative": 2787, + "truePositive": 71, + "wilsonPrecisionLower95": 0.8875 + }, + "question": { + "f1": 0.7917, + "falseNegative": 212, + "falsePositive": 8, + "precision": 0.9812, + "predictedPositive": 426, + "recall": 0.6635, + "total": 2970, + "trueNegative": 2332, + "truePositive": 418, + "wilsonPrecisionLower95": 0.9634 + }, + "replyableMessage": { + "f1": 0.6113, + "falseNegative": 1037, + "falsePositive": 48, + "precision": 0.9467, + "predictedPositive": 901, + "recall": 0.4513, + "total": 2970, + "trueNegative": 1032, + "truePositive": 853, + "wilsonPrecisionLower95": 0.9301 + }, + "scheduleNegotiation": { + "f1": 0.9176, + "falseNegative": 12, + "falsePositive": 2, + "precision": 0.975, + "predictedPositive": 80, + "recall": 0.8667, + "total": 2970, + "trueNegative": 2878, + "truePositive": 78, + "wilsonPrecisionLower95": 0.9134 + }, + "task": { + "f1": 0.1432, + "falseNegative": 664, + "falsePositive": 6, + "precision": 0.9032, + "predictedPositive": 62, + "recall": 0.0778, + "total": 2970, + "trueNegative": 2244, + "truePositive": 56, + "wilsonPrecisionLower95": 0.8045 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9809, + "falseNegative": 0, + "falsePositive": 7, + "precision": 0.9626, + "predictedPositive": 187, + "recall": 1.0, + "total": 1485, + "trueNegative": 1298, + "truePositive": 180, + "wilsonPrecisionLower95": 0.9248 + }, + "threshold": 0.99 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8625, + "falseNegative": 21, + "falsePositive": 1, + "precision": 0.9857, + "predictedPositive": 70, + "recall": 0.7667, + "total": 1485, + "trueNegative": 1394, + "truePositive": 69, + "wilsonPrecisionLower95": 0.9234 + }, + "threshold": 0.905 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.828, + "falseNegative": 25, + "falsePositive": 2, + "precision": 0.9701, + "predictedPositive": 67, + "recall": 0.7222, + "total": 1485, + "trueNegative": 1393, + "truePositive": 65, + "wilsonPrecisionLower95": 0.8975 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.8328, + "falseNegative": 48, + "falsePositive": 5, + "precision": 0.9635, + "predictedPositive": 137, + "recall": 0.7333, + "total": 2970, + "trueNegative": 2785, + "truePositive": 132, + "wilsonPrecisionLower95": 0.9174 + }, + "threshold": 0.685 + }, + "followUpReminder": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8788, + "falseNegative": 35, + "falsePositive": 5, + "precision": 0.9667, + "predictedPositive": 150, + "recall": 0.8056, + "total": 1485, + "trueNegative": 1300, + "truePositive": 145, + "wilsonPrecisionLower95": 0.9243 + }, + "threshold": 0.845 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6113, + "falseNegative": 99, + "falsePositive": 4, + "precision": 0.9529, + "predictedPositive": 85, + "recall": 0.45, + "total": 1485, + "trueNegative": 1301, + "truePositive": 81, + "wilsonPrecisionLower95": 0.8852 + }, + "threshold": 0.97 + } + }, + "metrics": { + "f1": 0.7682, + "falseNegative": 128, + "falsePositive": 12, + "precision": 0.9508, + "predictedPositive": 244, + "recall": 0.6444, + "total": 2970, + "trueNegative": 2598, + "truePositive": 232, + "wilsonPrecisionLower95": 0.916 + }, + "threshold": 0.91 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.847, + "falseNegative": 77, + "falsePositive": 9, + "precision": 0.9636, + "predictedPositive": 247, + "recall": 0.7556, + "total": 1485, + "trueNegative": 1161, + "truePositive": 238, + "wilsonPrecisionLower95": 0.9322 + }, + "threshold": 0.995 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7168, + "falseNegative": 134, + "falsePositive": 9, + "precision": 0.9526, + "predictedPositive": 190, + "recall": 0.5746, + "total": 1485, + "trueNegative": 1161, + "truePositive": 181, + "wilsonPrecisionLower95": 0.9124 + }, + "threshold": 0.865 + } + }, + "metrics": { + "f1": 0.7524, + "falseNegative": 238, + "falsePositive": 20, + "precision": 0.9515, + "predictedPositive": 412, + "recall": 0.6222, + "total": 2970, + "trueNegative": 2320, + "truePositive": 392, + "wilsonPrecisionLower95": 0.9262 + }, + "threshold": 0.975 + }, + "replyableMessage": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9349, + "falseNegative": 76, + "falsePositive": 45, + "precision": 0.9508, + "predictedPositive": 914, + "recall": 0.9196, + "total": 1485, + "trueNegative": 495, + "truePositive": 869, + "wilsonPrecisionLower95": 0.9348 + }, + "threshold": 0.955 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 45, + "recall": 1.0, + "total": 1485, + "trueNegative": 1440, + "truePositive": 45, + "wilsonPrecisionLower95": 0.9213 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.967, + "falseNegative": 1, + "falsePositive": 2, + "precision": 0.9565, + "predictedPositive": 46, + "recall": 0.9778, + "total": 1485, + "trueNegative": 1438, + "truePositive": 44, + "wilsonPrecisionLower95": 0.8547 + }, + "threshold": 0.995 + } + }, + "metrics": { + "f1": 0.9727, + "falseNegative": 1, + "falsePositive": 4, + "precision": 0.957, + "predictedPositive": 93, + "recall": 0.9889, + "total": 2970, + "trueNegative": 2876, + "truePositive": 89, + "wilsonPrecisionLower95": 0.8946 + }, + "threshold": 0.64 + }, + "task": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "trainingRecords": 15345, + "validation": { + "intentsMeeting95Precision": 8, + "macroF1": 0.6332, + "macroPrecision": 0.9668, + "macroRecall": 0.5246, + "minimumPrecision": 0.9, + "perIntent": { + "blessing": { + "f1": 0.3019, + "falseNegative": 74, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 16, + "recall": 0.1778, + "total": 2970, + "trueNegative": 2880, + "truePositive": 16, + "wilsonPrecisionLower95": 0.8064 + }, + "complaint": { + "f1": 0.8199, + "falseNegative": 105, + "falsePositive": 7, + "precision": 0.9733, + "predictedPositive": 262, + "recall": 0.7083, + "total": 2970, + "trueNegative": 2603, + "truePositive": 255, + "wilsonPrecisionLower95": 0.9459 + }, + "confirmationDecision": { + "f1": 0.8454, + "falseNegative": 46, + "falsePositive": 3, + "precision": 0.9781, + "predictedPositive": 137, + "recall": 0.7444, + "total": 2970, + "trueNegative": 2787, + "truePositive": 134, + "wilsonPrecisionLower95": 0.9376 + }, + "followUpReminder": { + "f1": 0.7597, + "falseNegative": 134, + "falsePositive": 9, + "precision": 0.9617, + "predictedPositive": 235, + "recall": 0.6278, + "total": 2970, + "trueNegative": 2601, + "truePositive": 226, + "wilsonPrecisionLower95": 0.9288 + }, + "invitation": { + "f1": 0.3273, + "falseNegative": 144, + "falsePositive": 4, + "precision": 0.9, + "predictedPositive": 40, + "recall": 0.2, + "total": 2970, + "trueNegative": 2786, + "truePositive": 36, + "wilsonPrecisionLower95": 0.7695 + }, + "question": { + "f1": 0.7854, + "falseNegative": 211, + "falsePositive": 18, + "precision": 0.9588, + "predictedPositive": 437, + "recall": 0.6651, + "total": 2970, + "trueNegative": 2322, + "truePositive": 419, + "wilsonPrecisionLower95": 0.9358 + }, + "replyableMessage": { + "f1": 0.6272, + "falseNegative": 1006, + "falsePositive": 45, + "precision": 0.9516, + "predictedPositive": 929, + "recall": 0.4677, + "total": 2970, + "trueNegative": 1035, + "truePositive": 884, + "wilsonPrecisionLower95": 0.9358 + }, + "scheduleNegotiation": { + "f1": 0.9834, + "falseNegative": 1, + "falsePositive": 2, + "precision": 0.978, + "predictedPositive": 91, + "recall": 0.9889, + "total": 2970, + "trueNegative": 2878, + "truePositive": 89, + "wilsonPrecisionLower95": 0.9234 + }, + "task": { + "f1": 0.2482, + "falseNegative": 618, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 102, + "recall": 0.1417, + "total": 2970, + "trueNegative": 2250, + "truePositive": 102, + "wilsonPrecisionLower95": 0.9637 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 3.5e-06, + "augmentation": "prefix", + "char_max": 6, + "char_min": 2, + "external_weight": 0.8500000000000001, + "hard_example_weight": 2.2, + "l1_ratio": 0.2, + "max_char_features": 85000, + "max_word_features": 42000, + "min_df": 1, + "pseudo_weight": 0.375, + "round": 15, + "word_max": 3 + }, + "featureCount": 127000, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 1 + }, + "complaint": { + "negative": 250, + "positive": 8 + }, + "confirmationDecision": { + "negative": 250, + "positive": 30 + }, + "followUpReminder": { + "negative": 250, + "positive": 0 + }, + "invitation": { + "negative": 250, + "positive": 0 + }, + "question": { + "negative": 250, + "positive": 0 + }, + "replyableMessage": { + "negative": 39, + "positive": 250 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 0 + }, + "task": { + "negative": 250, + "positive": 0 + } + }, + "round": 15, + "selectionObjective": 0.7896, + "syntheticRegression": { + "intentsMeeting95Precision": 7, + "macroF1": 0.7019, + "macroPrecision": 0.9601, + "macroRecall": 0.6059, + "minimumPrecision": 0.8634, + "perIntent": { + "blessing": { + "f1": 0.2, + "falseNegative": 80, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 10, + "recall": 0.1111, + "total": 2970, + "trueNegative": 2880, + "truePositive": 10, + "wilsonPrecisionLower95": 0.7225 + }, + "complaint": { + "f1": 0.9641, + "falseNegative": 24, + "falsePositive": 1, + "precision": 0.997, + "predictedPositive": 337, + "recall": 0.9333, + "total": 2970, + "trueNegative": 2609, + "truePositive": 336, + "wilsonPrecisionLower95": 0.9834 + }, + "confirmationDecision": { + "f1": 0.8152, + "falseNegative": 41, + "falsePositive": 22, + "precision": 0.8634, + "predictedPositive": 161, + "recall": 0.7722, + "total": 2970, + "trueNegative": 2768, + "truePositive": 139, + "wilsonPrecisionLower95": 0.8018 + }, + "followUpReminder": { + "f1": 0.6115, + "falseNegative": 201, + "falsePositive": 1, + "precision": 0.9938, + "predictedPositive": 160, + "recall": 0.4417, + "total": 2970, + "trueNegative": 2609, + "truePositive": 159, + "wilsonPrecisionLower95": 0.9655 + }, + "invitation": { + "f1": 0.6084, + "falseNegative": 100, + "falsePositive": 3, + "precision": 0.9639, + "predictedPositive": 83, + "recall": 0.4444, + "total": 2970, + "trueNegative": 2787, + "truePositive": 80, + "wilsonPrecisionLower95": 0.899 + }, + "question": { + "f1": 0.5421, + "falseNegative": 395, + "falsePositive": 2, + "precision": 0.9916, + "predictedPositive": 237, + "recall": 0.373, + "total": 2970, + "trueNegative": 2338, + "truePositive": 235, + "wilsonPrecisionLower95": 0.9698 + }, + "replyableMessage": { + "f1": 0.862, + "falseNegative": 410, + "falsePositive": 64, + "precision": 0.9585, + "predictedPositive": 1544, + "recall": 0.7831, + "total": 2970, + "trueNegative": 1016, + "truePositive": 1480, + "wilsonPrecisionLower95": 0.9474 + }, + "scheduleNegotiation": { + "f1": 0.9379, + "falseNegative": 7, + "falsePositive": 4, + "precision": 0.954, + "predictedPositive": 87, + "recall": 0.9222, + "total": 2970, + "trueNegative": 2876, + "truePositive": 83, + "wilsonPrecisionLower95": 0.8877 + }, + "task": { + "f1": 0.7763, + "falseNegative": 236, + "falsePositive": 43, + "precision": 0.9184, + "predictedPositive": 527, + "recall": 0.6722, + "total": 2970, + "trueNegative": 2207, + "truePositive": 484, + "wilsonPrecisionLower95": 0.8919 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9836, + "falseNegative": 0, + "falsePositive": 6, + "precision": 0.9677, + "predictedPositive": 186, + "recall": 1.0, + "total": 1485, + "trueNegative": 1299, + "truePositive": 180, + "wilsonPrecisionLower95": 0.9314 + }, + "threshold": 0.945 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.928, + "falseNegative": 19, + "falsePositive": 6, + "precision": 0.9641, + "predictedPositive": 167, + "recall": 0.8944, + "total": 1485, + "trueNegative": 1299, + "truePositive": 161, + "wilsonPrecisionLower95": 0.9238 + }, + "threshold": 0.77 + } + }, + "metrics": { + "f1": 0.9365, + "falseNegative": 28, + "falsePositive": 17, + "precision": 0.9513, + "predictedPositive": 349, + "recall": 0.9222, + "total": 2970, + "trueNegative": 2593, + "truePositive": 332, + "wilsonPrecisionLower95": 0.9234 + }, + "threshold": 0.88 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.954, + "falseNegative": 7, + "falsePositive": 1, + "precision": 0.9881, + "predictedPositive": 84, + "recall": 0.9222, + "total": 1485, + "trueNegative": 1394, + "truePositive": 83, + "wilsonPrecisionLower95": 0.9356 + }, + "threshold": 0.85 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6906, + "falseNegative": 42, + "falsePositive": 1, + "precision": 0.9796, + "predictedPositive": 49, + "recall": 0.5333, + "total": 1485, + "trueNegative": 1394, + "truePositive": 48, + "wilsonPrecisionLower95": 0.8931 + }, + "threshold": 0.985 + } + }, + "metrics": { + "f1": 0.8598, + "falseNegative": 39, + "falsePositive": 7, + "precision": 0.9527, + "predictedPositive": 148, + "recall": 0.7833, + "total": 2970, + "trueNegative": 2783, + "truePositive": 141, + "wilsonPrecisionLower95": 0.9056 + }, + "threshold": 0.935 + }, + "followUpReminder": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8328, + "falseNegative": 48, + "falsePositive": 5, + "precision": 0.9635, + "predictedPositive": 137, + "recall": 0.7333, + "total": 1485, + "trueNegative": 1300, + "truePositive": 132, + "wilsonPrecisionLower95": 0.9174 + }, + "threshold": 0.99 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9133, + "falseNegative": 11, + "falsePositive": 4, + "precision": 0.9518, + "predictedPositive": 83, + "recall": 0.8778, + "total": 1485, + "trueNegative": 1391, + "truePositive": 79, + "wilsonPrecisionLower95": 0.8825 + }, + "threshold": 0.97 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7548, + "falseNegative": 118, + "falsePositive": 10, + "precision": 0.9517, + "predictedPositive": 207, + "recall": 0.6254, + "total": 1485, + "trueNegative": 1160, + "truePositive": 197, + "wilsonPrecisionLower95": 0.9134 + }, + "threshold": 0.8 + } + }, + "metrics": { + "f1": 0.7678, + "falseNegative": 225, + "falsePositive": 20, + "precision": 0.9529, + "predictedPositive": 425, + "recall": 0.6429, + "total": 2970, + "trueNegative": 2320, + "truePositive": 405, + "wilsonPrecisionLower95": 0.9284 + }, + "threshold": 0.985 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.907, + "falseNegative": 126, + "falsePositive": 42, + "precision": 0.9512, + "predictedPositive": 861, + "recall": 0.8667, + "total": 1485, + "trueNegative": 498, + "truePositive": 819, + "wilsonPrecisionLower95": 0.9347 + }, + "threshold": 0.98 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8252, + "falseNegative": 258, + "falsePositive": 33, + "precision": 0.9542, + "predictedPositive": 720, + "recall": 0.727, + "total": 1485, + "trueNegative": 507, + "truePositive": 687, + "wilsonPrecisionLower95": 0.9363 + }, + "threshold": 0.96 + } + }, + "metrics": { + "f1": 0.8678, + "falseNegative": 384, + "falsePositive": 75, + "precision": 0.9526, + "predictedPositive": 1581, + "recall": 0.7968, + "total": 2970, + "trueNegative": 1005, + "truePositive": 1506, + "wilsonPrecisionLower95": 0.9409 + }, + "threshold": 0.965 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9157, + "falseNegative": 7, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 38, + "recall": 0.8444, + "total": 1485, + "trueNegative": 1440, + "truePositive": 38, + "wilsonPrecisionLower95": 0.9082 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9556, + "falseNegative": 2, + "falsePositive": 2, + "precision": 0.9556, + "predictedPositive": 45, + "recall": 0.9556, + "total": 1485, + "trueNegative": 1438, + "truePositive": 43, + "wilsonPrecisionLower95": 0.8517 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.9364, + "falseNegative": 9, + "falsePositive": 2, + "precision": 0.9759, + "predictedPositive": 83, + "recall": 0.9, + "total": 2970, + "trueNegative": 2878, + "truePositive": 81, + "wilsonPrecisionLower95": 0.9163 + }, + "threshold": 0.5 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8593, + "falseNegative": 79, + "falsePositive": 13, + "precision": 0.9558, + "predictedPositive": 294, + "recall": 0.7806, + "total": 1485, + "trueNegative": 1112, + "truePositive": 281, + "wilsonPrecisionLower95": 0.9258 + }, + "threshold": 0.98 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8166, + "falseNegative": 104, + "falsePositive": 11, + "precision": 0.9588, + "predictedPositive": 267, + "recall": 0.7111, + "total": 1485, + "trueNegative": 1114, + "truePositive": 256, + "wilsonPrecisionLower95": 0.9277 + }, + "threshold": 0.96 + } + }, + "metrics": { + "f1": 0.8434, + "falseNegative": 176, + "falsePositive": 26, + "precision": 0.9544, + "predictedPositive": 570, + "recall": 0.7556, + "total": 2970, + "trueNegative": 2224, + "truePositive": 544, + "wilsonPrecisionLower95": 0.934 + }, + "threshold": 0.965 + } + }, + "trainingRecords": 15345, + "validation": { + "intentsMeeting95Precision": 8, + "macroF1": 0.711, + "macroPrecision": 0.9652, + "macroRecall": 0.6071, + "minimumPrecision": 0.9441, + "perIntent": { + "blessing": { + "f1": 0.3333, + "falseNegative": 72, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 18, + "recall": 0.2, + "total": 2970, + "trueNegative": 2880, + "truePositive": 18, + "wilsonPrecisionLower95": 0.8241 + }, + "complaint": { + "f1": 0.9565, + "falseNegative": 19, + "falsePositive": 12, + "precision": 0.966, + "predictedPositive": 353, + "recall": 0.9472, + "total": 2970, + "trueNegative": 2598, + "truePositive": 341, + "wilsonPrecisionLower95": 0.9415 + }, + "confirmationDecision": { + "f1": 0.8371, + "falseNegative": 49, + "falsePositive": 2, + "precision": 0.985, + "predictedPositive": 133, + "recall": 0.7278, + "total": 2970, + "trueNegative": 2788, + "truePositive": 131, + "wilsonPrecisionLower95": 0.9468 + }, + "followUpReminder": { + "f1": 0.5368, + "falseNegative": 225, + "falsePositive": 8, + "precision": 0.9441, + "predictedPositive": 143, + "recall": 0.375, + "total": 2970, + "trueNegative": 2602, + "truePositive": 135, + "wilsonPrecisionLower95": 0.8935 + }, + "invitation": { + "f1": 0.6061, + "falseNegative": 100, + "falsePositive": 4, + "precision": 0.9524, + "predictedPositive": 84, + "recall": 0.4444, + "total": 2970, + "trueNegative": 2786, + "truePositive": 80, + "wilsonPrecisionLower95": 0.8839 + }, + "question": { + "f1": 0.487, + "falseNegative": 424, + "falsePositive": 10, + "precision": 0.9537, + "predictedPositive": 216, + "recall": 0.327, + "total": 2970, + "trueNegative": 2330, + "truePositive": 206, + "wilsonPrecisionLower95": 0.9169 + }, + "replyableMessage": { + "f1": 0.8678, + "falseNegative": 384, + "falsePositive": 75, + "precision": 0.9526, + "predictedPositive": 1581, + "recall": 0.7968, + "total": 2970, + "trueNegative": 1005, + "truePositive": 1506, + "wilsonPrecisionLower95": 0.9409 + }, + "scheduleNegotiation": { + "f1": 0.9364, + "falseNegative": 9, + "falsePositive": 2, + "precision": 0.9759, + "predictedPositive": 83, + "recall": 0.9, + "total": 2970, + "trueNegative": 2878, + "truePositive": 81, + "wilsonPrecisionLower95": 0.9163 + }, + "task": { + "f1": 0.8384, + "falseNegative": 183, + "falsePositive": 24, + "precision": 0.9572, + "predictedPositive": 561, + "recall": 0.7458, + "total": 2970, + "trueNegative": 2226, + "truePositive": 537, + "wilsonPrecisionLower95": 0.9371 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 3e-06, + "augmentation": "numeric", + "char_max": 7, + "char_min": 2, + "external_weight": 0.8700000000000001, + "hard_example_weight": 2.3, + "l1_ratio": 0.15, + "max_char_features": 80000, + "max_word_features": 38000, + "min_df": 2, + "pseudo_weight": 0.39, + "round": 16, + "word_max": 3 + }, + "featureCount": 100252, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 3 + }, + "complaint": { + "negative": 250, + "positive": 11 + }, + "confirmationDecision": { + "negative": 250, + "positive": 25 + }, + "followUpReminder": { + "negative": 250, + "positive": 0 + }, + "invitation": { + "negative": 250, + "positive": 0 + }, + "question": { + "negative": 250, + "positive": 0 + }, + "replyableMessage": { + "negative": 76, + "positive": 250 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 0 + }, + "task": { + "negative": 250, + "positive": 0 + } + }, + "round": 16, + "selectionObjective": 0.8105, + "syntheticRegression": { + "intentsMeeting95Precision": 5, + "macroF1": 0.726, + "macroPrecision": 0.9485, + "macroRecall": 0.6388, + "minimumPrecision": 0.8195, + "perIntent": { + "blessing": { + "f1": 0.125, + "falseNegative": 84, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 6, + "recall": 0.0667, + "total": 2970, + "trueNegative": 2880, + "truePositive": 6, + "wilsonPrecisionLower95": 0.6097 + }, + "complaint": { + "f1": 0.9132, + "falseNegative": 55, + "falsePositive": 3, + "precision": 0.9903, + "predictedPositive": 308, + "recall": 0.8472, + "total": 2970, + "trueNegative": 2607, + "truePositive": 305, + "wilsonPrecisionLower95": 0.9718 + }, + "confirmationDecision": { + "f1": 0.6965, + "falseNegative": 71, + "falsePositive": 24, + "precision": 0.8195, + "predictedPositive": 133, + "recall": 0.6056, + "total": 2970, + "trueNegative": 2766, + "truePositive": 109, + "wilsonPrecisionLower95": 0.7455 + }, + "followUpReminder": { + "f1": 0.7409, + "falseNegative": 147, + "falsePositive": 2, + "precision": 0.9907, + "predictedPositive": 215, + "recall": 0.5917, + "total": 2970, + "trueNegative": 2608, + "truePositive": 213, + "wilsonPrecisionLower95": 0.9667 + }, + "invitation": { + "f1": 0.6264, + "falseNegative": 97, + "falsePositive": 2, + "precision": 0.9765, + "predictedPositive": 85, + "recall": 0.4611, + "total": 2970, + "trueNegative": 2788, + "truePositive": 83, + "wilsonPrecisionLower95": 0.9182 + }, + "question": { + "f1": 0.8457, + "falseNegative": 164, + "falsePositive": 6, + "precision": 0.9873, + "predictedPositive": 472, + "recall": 0.7397, + "total": 2970, + "trueNegative": 2334, + "truePositive": 466, + "wilsonPrecisionLower95": 0.9725 + }, + "replyableMessage": { + "f1": 0.8725, + "falseNegative": 371, + "falsePositive": 73, + "precision": 0.9541, + "predictedPositive": 1592, + "recall": 0.8037, + "total": 2970, + "trueNegative": 1007, + "truePositive": 1519, + "wilsonPrecisionLower95": 0.9427 + }, + "scheduleNegotiation": { + "f1": 0.9061, + "falseNegative": 8, + "falsePositive": 9, + "precision": 0.9011, + "predictedPositive": 91, + "recall": 0.9111, + "total": 2970, + "trueNegative": 2871, + "truePositive": 82, + "wilsonPrecisionLower95": 0.8226 + }, + "task": { + "f1": 0.8081, + "falseNegative": 200, + "falsePositive": 47, + "precision": 0.9171, + "predictedPositive": 567, + "recall": 0.7222, + "total": 2970, + "trueNegative": 2203, + "truePositive": 520, + "wilsonPrecisionLower95": 0.8915 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9371, + "falseNegative": 16, + "falsePositive": 6, + "precision": 0.9647, + "predictedPositive": 170, + "recall": 0.9111, + "total": 1485, + "trueNegative": 1299, + "truePositive": 164, + "wilsonPrecisionLower95": 0.9251 + }, + "threshold": 0.995 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8836, + "falseNegative": 32, + "falsePositive": 7, + "precision": 0.9548, + "predictedPositive": 155, + "recall": 0.8222, + "total": 1485, + "trueNegative": 1298, + "truePositive": 148, + "wilsonPrecisionLower95": 0.9097 + }, + "threshold": 0.99 + } + }, + "metrics": { + "f1": 0.8994, + "falseNegative": 56, + "falsePositive": 12, + "precision": 0.962, + "predictedPositive": 316, + "recall": 0.8444, + "total": 2970, + "trueNegative": 2598, + "truePositive": 304, + "wilsonPrecisionLower95": 0.9348 + }, + "threshold": 0.995 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8625, + "falseNegative": 21, + "falsePositive": 1, + "precision": 0.9857, + "predictedPositive": 70, + "recall": 0.7667, + "total": 1485, + "trueNegative": 1394, + "truePositive": 69, + "wilsonPrecisionLower95": 0.9234 + }, + "threshold": 0.915 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.7415, + "falseNegative": 71, + "falsePositive": 5, + "precision": 0.9561, + "predictedPositive": 114, + "recall": 0.6056, + "total": 2970, + "trueNegative": 2785, + "truePositive": 109, + "wilsonPrecisionLower95": 0.9014 + }, + "threshold": 0.96 + }, + "followUpReminder": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8667, + "falseNegative": 37, + "falsePositive": 7, + "precision": 0.9533, + "predictedPositive": 150, + "recall": 0.7944, + "total": 1485, + "trueNegative": 1298, + "truePositive": 143, + "wilsonPrecisionLower95": 0.9068 + }, + "threshold": 0.96 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6031, + "falseNegative": 101, + "falsePositive": 3, + "precision": 0.9634, + "predictedPositive": 82, + "recall": 0.4389, + "total": 1485, + "trueNegative": 1302, + "truePositive": 79, + "wilsonPrecisionLower95": 0.8979 + }, + "threshold": 0.97 + } + }, + "metrics": { + "f1": 0.7492, + "falseNegative": 139, + "falsePositive": 9, + "precision": 0.9609, + "predictedPositive": 230, + "recall": 0.6139, + "total": 2970, + "trueNegative": 2601, + "truePositive": 221, + "wilsonPrecisionLower95": 0.9273 + }, + "threshold": 0.97 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.3604, + "falseNegative": 70, + "falsePositive": 1, + "precision": 0.9524, + "predictedPositive": 21, + "recall": 0.2222, + "total": 1485, + "trueNegative": 1394, + "truePositive": 20, + "wilsonPrecisionLower95": 0.7733 + }, + "threshold": 0.965 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8273, + "falseNegative": 85, + "falsePositive": 11, + "precision": 0.9544, + "predictedPositive": 241, + "recall": 0.7302, + "total": 1485, + "trueNegative": 1159, + "truePositive": 230, + "wilsonPrecisionLower95": 0.9201 + }, + "threshold": 0.985 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.797, + "falseNegative": 99, + "falsePositive": 11, + "precision": 0.9515, + "predictedPositive": 227, + "recall": 0.6857, + "total": 1485, + "trueNegative": 1159, + "truePositive": 216, + "wilsonPrecisionLower95": 0.9153 + }, + "threshold": 0.835 + } + }, + "metrics": { + "f1": 0.8051, + "falseNegative": 190, + "falsePositive": 23, + "precision": 0.9503, + "predictedPositive": 463, + "recall": 0.6984, + "total": 2970, + "trueNegative": 2317, + "truePositive": 440, + "wilsonPrecisionLower95": 0.9266 + }, + "threshold": 0.94 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9343, + "falseNegative": 77, + "falsePositive": 45, + "precision": 0.9507, + "predictedPositive": 913, + "recall": 0.9185, + "total": 1485, + "trueNegative": 495, + "truePositive": 868, + "wilsonPrecisionLower95": 0.9347 + }, + "threshold": 0.91 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8207, + "falseNegative": 263, + "falsePositive": 35, + "precision": 0.9512, + "predictedPositive": 717, + "recall": 0.7217, + "total": 1485, + "trueNegative": 505, + "truePositive": 682, + "wilsonPrecisionLower95": 0.9329 + }, + "threshold": 0.98 + } + }, + "metrics": { + "f1": 0.8572, + "falseNegative": 419, + "falsePositive": 71, + "precision": 0.954, + "predictedPositive": 1542, + "recall": 0.7783, + "total": 2970, + "trueNegative": 1009, + "truePositive": 1471, + "wilsonPrecisionLower95": 0.9423 + }, + "threshold": 0.98 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9888, + "falseNegative": 1, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 44, + "recall": 0.9778, + "total": 1485, + "trueNegative": 1440, + "truePositive": 44, + "wilsonPrecisionLower95": 0.9197 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.989, + "falseNegative": 0, + "falsePositive": 1, + "precision": 0.9783, + "predictedPositive": 46, + "recall": 1.0, + "total": 1485, + "trueNegative": 1439, + "truePositive": 45, + "wilsonPrecisionLower95": 0.8866 + }, + "threshold": 0.955 + } + }, + "metrics": { + "f1": 0.9778, + "falseNegative": 2, + "falsePositive": 2, + "precision": 0.9778, + "predictedPositive": 90, + "recall": 0.9778, + "total": 2970, + "trueNegative": 2878, + "truePositive": 88, + "wilsonPrecisionLower95": 0.9226 + }, + "threshold": 0.935 + }, + "task": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8836, + "falseNegative": 64, + "falsePositive": 14, + "precision": 0.9548, + "predictedPositive": 310, + "recall": 0.8222, + "total": 1485, + "trueNegative": 1111, + "truePositive": 296, + "wilsonPrecisionLower95": 0.9256 + }, + "threshold": 0.975 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.7896, + "falseNegative": 118, + "falsePositive": 11, + "precision": 0.9565, + "predictedPositive": 253, + "recall": 0.6722, + "total": 1485, + "trueNegative": 1114, + "truePositive": 242, + "wilsonPrecisionLower95": 0.9238 + }, + "threshold": 0.96 + } + }, + "metrics": { + "f1": 0.8316, + "falseNegative": 189, + "falsePositive": 26, + "precision": 0.9533, + "predictedPositive": 557, + "recall": 0.7375, + "total": 2970, + "trueNegative": 2224, + "truePositive": 531, + "wilsonPrecisionLower95": 0.9325 + }, + "threshold": 0.97 + } + }, + "trainingRecords": 16726, + "validation": { + "intentsMeeting95Precision": 9, + "macroF1": 0.7397, + "macroPrecision": 0.9707, + "macroRecall": 0.6388, + "minimumPrecision": 0.9509, + "perIntent": { + "blessing": { + "f1": 0.3636, + "falseNegative": 70, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 20, + "recall": 0.2222, + "total": 2970, + "trueNegative": 2880, + "truePositive": 20, + "wilsonPrecisionLower95": 0.8389 + }, + "complaint": { + "f1": 0.9109, + "falseNegative": 48, + "falsePositive": 13, + "precision": 0.96, + "predictedPositive": 325, + "recall": 0.8667, + "total": 2970, + "trueNegative": 2597, + "truePositive": 312, + "wilsonPrecisionLower95": 0.9328 + }, + "confirmationDecision": { + "f1": 0.6236, + "falseNegative": 98, + "falsePositive": 1, + "precision": 0.988, + "predictedPositive": 83, + "recall": 0.4556, + "total": 2970, + "trueNegative": 2789, + "truePositive": 82, + "wilsonPrecisionLower95": 0.9349 + }, + "followUpReminder": { + "f1": 0.75, + "falseNegative": 138, + "falsePositive": 10, + "precision": 0.9569, + "predictedPositive": 232, + "recall": 0.6167, + "total": 2970, + "trueNegative": 2600, + "truePositive": 222, + "wilsonPrecisionLower95": 0.9225 + }, + "invitation": { + "f1": 0.4917, + "falseNegative": 121, + "falsePositive": 1, + "precision": 0.9833, + "predictedPositive": 60, + "recall": 0.3278, + "total": 2970, + "trueNegative": 2789, + "truePositive": 59, + "wilsonPrecisionLower95": 0.9114 + }, + "question": { + "f1": 0.8124, + "falseNegative": 184, + "falsePositive": 22, + "precision": 0.953, + "predictedPositive": 468, + "recall": 0.7079, + "total": 2970, + "trueNegative": 2318, + "truePositive": 446, + "wilsonPrecisionLower95": 0.9299 + }, + "replyableMessage": { + "f1": 0.8807, + "falseNegative": 340, + "falsePositive": 80, + "precision": 0.9509, + "predictedPositive": 1630, + "recall": 0.8201, + "total": 2970, + "trueNegative": 1000, + "truePositive": 1550, + "wilsonPrecisionLower95": 0.9393 + }, + "scheduleNegotiation": { + "f1": 0.9889, + "falseNegative": 1, + "falsePositive": 1, + "precision": 0.9889, + "predictedPositive": 90, + "recall": 0.9889, + "total": 2970, + "trueNegative": 2879, + "truePositive": 89, + "wilsonPrecisionLower95": 0.9397 + }, + "task": { + "f1": 0.8359, + "falseNegative": 185, + "falsePositive": 25, + "precision": 0.9554, + "predictedPositive": 560, + "recall": 0.7431, + "total": 2970, + "trueNegative": 2225, + "truePositive": 535, + "wilsonPrecisionLower95": 0.9349 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 2e-06, + "augmentation": "numeric", + "char_max": 7, + "char_min": 3, + "external_weight": 0.8900000000000001, + "hard_example_weight": 2.5, + "l1_ratio": 0.2, + "max_char_features": 85000, + "max_word_features": 42000, + "min_df": 1, + "pseudo_weight": 0.405, + "round": 17, + "word_max": 3 + }, + "featureCount": 127000, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 1 + }, + "complaint": { + "negative": 250, + "positive": 4 + }, + "confirmationDecision": { + "negative": 250, + "positive": 23 + }, + "followUpReminder": { + "negative": 250, + "positive": 3 + }, + "invitation": { + "negative": 250, + "positive": 0 + }, + "question": { + "negative": 250, + "positive": 0 + }, + "replyableMessage": { + "negative": 70, + "positive": 250 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 0 + }, + "task": { + "negative": 250, + "positive": 0 + } + }, + "round": 17, + "selectionObjective": 0.6932, + "syntheticRegression": { + "intentsMeeting95Precision": 4, + "macroF1": 0.5086, + "macroPrecision": 0.9202, + "macroRecall": 0.4091, + "minimumPrecision": 0.7329, + "perIntent": { + "blessing": { + "f1": 0.1633, + "falseNegative": 82, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 8, + "recall": 0.0889, + "total": 2970, + "trueNegative": 2880, + "truePositive": 8, + "wilsonPrecisionLower95": 0.6756 + }, + "complaint": { + "f1": 0.914, + "falseNegative": 36, + "falsePositive": 25, + "precision": 0.9284, + "predictedPositive": 349, + "recall": 0.9, + "total": 2970, + "trueNegative": 2585, + "truePositive": 324, + "wilsonPrecisionLower95": 0.8964 + }, + "confirmationDecision": { + "f1": 0.6564, + "falseNegative": 73, + "falsePositive": 39, + "precision": 0.7329, + "predictedPositive": 146, + "recall": 0.5944, + "total": 2970, + "trueNegative": 2751, + "truePositive": 107, + "wilsonPrecisionLower95": 0.6558 + }, + "followUpReminder": { + "f1": 0.4, + "falseNegative": 270, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 90, + "recall": 0.25, + "total": 2970, + "trueNegative": 2610, + "truePositive": 90, + "wilsonPrecisionLower95": 0.9591 + }, + "invitation": { + "f1": 0.5954, + "falseNegative": 102, + "falsePositive": 4, + "precision": 0.9512, + "predictedPositive": 82, + "recall": 0.4333, + "total": 2970, + "trueNegative": 2786, + "truePositive": 78, + "wilsonPrecisionLower95": 0.8812 + }, + "question": { + "f1": 0.4976, + "falseNegative": 419, + "falsePositive": 7, + "precision": 0.9679, + "predictedPositive": 218, + "recall": 0.3349, + "total": 2970, + "trueNegative": 2333, + "truePositive": 211, + "wilsonPrecisionLower95": 0.9352 + }, + "replyableMessage": { + "f1": 0.2549, + "falseNegative": 1612, + "falsePositive": 13, + "precision": 0.9553, + "predictedPositive": 291, + "recall": 0.1471, + "total": 2970, + "trueNegative": 1067, + "truePositive": 278, + "wilsonPrecisionLower95": 0.9251 + }, + "scheduleNegotiation": { + "f1": 0.8333, + "falseNegative": 20, + "falsePositive": 8, + "precision": 0.8974, + "predictedPositive": 78, + "recall": 0.7778, + "total": 2970, + "trueNegative": 2872, + "truePositive": 70, + "wilsonPrecisionLower95": 0.8105 + }, + "task": { + "f1": 0.2629, + "falseNegative": 608, + "falsePositive": 20, + "precision": 0.8485, + "predictedPositive": 132, + "recall": 0.1556, + "total": 2970, + "trueNegative": 2230, + "truePositive": 112, + "wilsonPrecisionLower95": 0.7775 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.989, + "falseNegative": 0, + "falsePositive": 4, + "precision": 0.9783, + "predictedPositive": 184, + "recall": 1.0, + "total": 1485, + "trueNegative": 1301, + "truePositive": 180, + "wilsonPrecisionLower95": 0.9454 + }, + "threshold": 0.995 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8869, + "falseNegative": 31, + "falsePositive": 7, + "precision": 0.9551, + "predictedPositive": 156, + "recall": 0.8278, + "total": 1485, + "trueNegative": 1298, + "truePositive": 149, + "wilsonPrecisionLower95": 0.9103 + }, + "threshold": 0.995 + } + }, + "metrics": { + "f1": 0.944, + "falseNegative": 23, + "falsePositive": 17, + "precision": 0.952, + "predictedPositive": 354, + "recall": 0.9361, + "total": 2970, + "trueNegative": 2593, + "truePositive": 337, + "wilsonPrecisionLower95": 0.9244 + }, + "threshold": 0.975 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8447, + "falseNegative": 22, + "falsePositive": 3, + "precision": 0.9577, + "predictedPositive": 71, + "recall": 0.7556, + "total": 1485, + "trueNegative": 1392, + "truePositive": 68, + "wilsonPrecisionLower95": 0.883 + }, + "threshold": 0.825 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.7723, + "falseNegative": 63, + "falsePositive": 6, + "precision": 0.9512, + "predictedPositive": 123, + "recall": 0.65, + "total": 2970, + "trueNegative": 2784, + "truePositive": 117, + "wilsonPrecisionLower95": 0.8977 + }, + "threshold": 0.965 + }, + "followUpReminder": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.6625, + "falseNegative": 155, + "falsePositive": 8, + "precision": 0.9524, + "predictedPositive": 168, + "recall": 0.5079, + "total": 1485, + "trueNegative": 1162, + "truePositive": 160, + "wilsonPrecisionLower95": 0.9089 + }, + "threshold": 0.97 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "replyableMessage": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9655, + "falseNegative": 3, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 42, + "recall": 0.9333, + "total": 1485, + "trueNegative": 1440, + "truePositive": 42, + "wilsonPrecisionLower95": 0.9162 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9425, + "falseNegative": 4, + "falsePositive": 1, + "precision": 0.9762, + "predictedPositive": 42, + "recall": 0.9111, + "total": 1485, + "trueNegative": 1439, + "truePositive": 41, + "wilsonPrecisionLower95": 0.8768 + }, + "threshold": 0.73 + } + }, + "metrics": { + "f1": 0.9486, + "falseNegative": 7, + "falsePositive": 2, + "precision": 0.9765, + "predictedPositive": 85, + "recall": 0.9222, + "total": 2970, + "trueNegative": 2878, + "truePositive": 83, + "wilsonPrecisionLower95": 0.9182 + }, + "threshold": 0.5 + }, + "task": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "trainingRecords": 16820, + "validation": { + "intentsMeeting95Precision": 8, + "macroF1": 0.5631, + "macroPrecision": 0.9693, + "macroRecall": 0.4421, + "minimumPrecision": 0.9474, + "perIntent": { + "blessing": { + "f1": 0.3019, + "falseNegative": 74, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 16, + "recall": 0.1778, + "total": 2970, + "trueNegative": 2880, + "truePositive": 16, + "wilsonPrecisionLower95": 0.8064 + }, + "complaint": { + "f1": 0.94, + "falseNegative": 31, + "falsePositive": 11, + "precision": 0.9676, + "predictedPositive": 340, + "recall": 0.9139, + "total": 2970, + "trueNegative": 2599, + "truePositive": 329, + "wilsonPrecisionLower95": 0.943 + }, + "confirmationDecision": { + "f1": 0.6545, + "falseNegative": 90, + "falsePositive": 5, + "precision": 0.9474, + "predictedPositive": 95, + "recall": 0.5, + "total": 2970, + "trueNegative": 2785, + "truePositive": 90, + "wilsonPrecisionLower95": 0.8827 + }, + "followUpReminder": { + "f1": 0.5282, + "falseNegative": 229, + "falsePositive": 5, + "precision": 0.9632, + "predictedPositive": 136, + "recall": 0.3639, + "total": 2970, + "trueNegative": 2605, + "truePositive": 131, + "wilsonPrecisionLower95": 0.9168 + }, + "invitation": { + "f1": 0.4378, + "falseNegative": 129, + "falsePositive": 2, + "precision": 0.9623, + "predictedPositive": 53, + "recall": 0.2833, + "total": 2970, + "trueNegative": 2788, + "truePositive": 51, + "wilsonPrecisionLower95": 0.8725 + }, + "question": { + "f1": 0.5428, + "falseNegative": 392, + "falsePositive": 9, + "precision": 0.9636, + "predictedPositive": 247, + "recall": 0.3778, + "total": 2970, + "trueNegative": 2331, + "truePositive": 238, + "wilsonPrecisionLower95": 0.9322 + }, + "replyableMessage": { + "f1": 0.2757, + "falseNegative": 1586, + "falsePositive": 11, + "precision": 0.9651, + "predictedPositive": 315, + "recall": 0.1608, + "total": 2970, + "trueNegative": 1069, + "truePositive": 304, + "wilsonPrecisionLower95": 0.9386 + }, + "scheduleNegotiation": { + "f1": 0.954, + "falseNegative": 7, + "falsePositive": 1, + "precision": 0.9881, + "predictedPositive": 84, + "recall": 0.9222, + "total": 2970, + "trueNegative": 2879, + "truePositive": 83, + "wilsonPrecisionLower95": 0.9356 + }, + "task": { + "f1": 0.4332, + "falseNegative": 519, + "falsePositive": 7, + "precision": 0.9663, + "predictedPositive": 208, + "recall": 0.2792, + "total": 2970, + "trueNegative": 2243, + "truePositive": 201, + "wilsonPrecisionLower95": 0.9322 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 1.5e-06, + "augmentation": "numeric", + "char_max": 8, + "char_min": 2, + "external_weight": 0.91, + "hard_example_weight": 2.5, + "l1_ratio": 0.2, + "max_char_features": 90000, + "max_word_features": 45000, + "min_df": 1, + "pseudo_weight": 0.42000000000000004, + "round": 18, + "word_max": 3 + }, + "featureCount": 135000, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 1 + }, + "complaint": { + "negative": 250, + "positive": 8 + }, + "confirmationDecision": { + "negative": 250, + "positive": 27 + }, + "followUpReminder": { + "negative": 250, + "positive": 0 + }, + "invitation": { + "negative": 250, + "positive": 0 + }, + "question": { + "negative": 250, + "positive": 0 + }, + "replyableMessage": { + "negative": 90, + "positive": 32 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 0 + }, + "task": { + "negative": 250, + "positive": 0 + } + }, + "round": 18, + "selectionObjective": 0.7725, + "syntheticRegression": { + "intentsMeeting95Precision": 4, + "macroF1": 0.6489, + "macroPrecision": 0.9452, + "macroRecall": 0.5429, + "minimumPrecision": 0.8205, + "perIntent": { + "blessing": { + "f1": 0.2178, + "falseNegative": 79, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 11, + "recall": 0.1222, + "total": 2970, + "trueNegative": 2880, + "truePositive": 11, + "wilsonPrecisionLower95": 0.7412 + }, + "complaint": { + "f1": 0.8661, + "falseNegative": 85, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 275, + "recall": 0.7639, + "total": 2970, + "trueNegative": 2610, + "truePositive": 275, + "wilsonPrecisionLower95": 0.9862 + }, + "confirmationDecision": { + "f1": 0.6465, + "falseNegative": 84, + "falsePositive": 21, + "precision": 0.8205, + "predictedPositive": 117, + "recall": 0.5333, + "total": 2970, + "trueNegative": 2769, + "truePositive": 96, + "wilsonPrecisionLower95": 0.7411 + }, + "followUpReminder": { + "f1": 0.5851, + "falseNegative": 207, + "falsePositive": 10, + "precision": 0.9387, + "predictedPositive": 163, + "recall": 0.425, + "total": 2970, + "trueNegative": 2600, + "truePositive": 153, + "wilsonPrecisionLower95": 0.8908 + }, + "invitation": { + "f1": 0.626, + "falseNegative": 98, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 82, + "recall": 0.4556, + "total": 2970, + "trueNegative": 2790, + "truePositive": 82, + "wilsonPrecisionLower95": 0.9552 + }, + "question": { + "f1": 0.7311, + "falseNegative": 267, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 363, + "recall": 0.5762, + "total": 2970, + "trueNegative": 2340, + "truePositive": 363, + "wilsonPrecisionLower95": 0.9895 + }, + "replyableMessage": { + "f1": 0.8939, + "falseNegative": 273, + "falsePositive": 111, + "precision": 0.9358, + "predictedPositive": 1728, + "recall": 0.8556, + "total": 2970, + "trueNegative": 969, + "truePositive": 1617, + "wilsonPrecisionLower95": 0.9232 + }, + "scheduleNegotiation": { + "f1": 0.9341, + "falseNegative": 5, + "falsePositive": 7, + "precision": 0.9239, + "predictedPositive": 92, + "recall": 0.9444, + "total": 2970, + "trueNegative": 2873, + "truePositive": 85, + "wilsonPrecisionLower95": 0.8512 + }, + "task": { + "f1": 0.3393, + "falseNegative": 569, + "falsePositive": 19, + "precision": 0.8882, + "predictedPositive": 170, + "recall": 0.2097, + "total": 2970, + "trueNegative": 2231, + "truePositive": 151, + "wilsonPrecisionLower95": 0.832 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "confirmationDecision": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9486, + "falseNegative": 7, + "falsePositive": 2, + "precision": 0.9765, + "predictedPositive": 85, + "recall": 0.9222, + "total": 1485, + "trueNegative": 1393, + "truePositive": 83, + "wilsonPrecisionLower95": 0.9182 + }, + "threshold": 0.9 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.8447, + "falseNegative": 44, + "falsePositive": 6, + "precision": 0.9577, + "predictedPositive": 142, + "recall": 0.7556, + "total": 2970, + "trueNegative": 2784, + "truePositive": 136, + "wilsonPrecisionLower95": 0.9109 + }, + "threshold": 0.955 + }, + "followUpReminder": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.8822, + "falseNegative": 34, + "falsePositive": 5, + "precision": 0.9669, + "predictedPositive": 151, + "recall": 0.8111, + "total": 1485, + "trueNegative": 1300, + "truePositive": 146, + "wilsonPrecisionLower95": 0.9248 + }, + "threshold": 0.98 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.7641, + "falseNegative": 130, + "falsePositive": 12, + "precision": 0.9504, + "predictedPositive": 242, + "recall": 0.6389, + "total": 2970, + "trueNegative": 2598, + "truePositive": 230, + "wilsonPrecisionLower95": 0.9153 + }, + "threshold": 0.99 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8103, + "falseNegative": 95, + "falsePositive": 8, + "precision": 0.9649, + "predictedPositive": 228, + "recall": 0.6984, + "total": 1485, + "trueNegative": 1162, + "truePositive": 220, + "wilsonPrecisionLower95": 0.9323 + }, + "threshold": 0.915 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9575, + "falseNegative": 33, + "falsePositive": 48, + "precision": 0.95, + "predictedPositive": 960, + "recall": 0.9651, + "total": 1485, + "trueNegative": 492, + "truePositive": 912, + "wilsonPrecisionLower95": 0.9343 + }, + "threshold": 0.845 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8361, + "falseNegative": 241, + "falsePositive": 35, + "precision": 0.9526, + "predictedPositive": 739, + "recall": 0.745, + "total": 1485, + "trueNegative": 505, + "truePositive": 704, + "wilsonPrecisionLower95": 0.9348 + }, + "threshold": 0.98 + } + }, + "metrics": { + "f1": 0.901, + "falseNegative": 274, + "falsePositive": 81, + "precision": 0.9523, + "predictedPositive": 1697, + "recall": 0.855, + "total": 2970, + "trueNegative": 999, + "truePositive": 1616, + "wilsonPrecisionLower95": 0.9411 + }, + "threshold": 0.975 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9655, + "falseNegative": 3, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 42, + "recall": 0.9333, + "total": 1485, + "trueNegative": 1440, + "truePositive": 42, + "wilsonPrecisionLower95": 0.9162 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 45, + "recall": 1.0, + "total": 1485, + "trueNegative": 1440, + "truePositive": 45, + "wilsonPrecisionLower95": 0.9213 + }, + "threshold": 0.5 + } + }, + "metrics": { + "f1": 0.9831, + "falseNegative": 3, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 87, + "recall": 0.9667, + "total": 2970, + "trueNegative": 2880, + "truePositive": 87, + "wilsonPrecisionLower95": 0.9577 + }, + "threshold": 0.5 + }, + "task": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "trainingRecords": 16841, + "validation": { + "intentsMeeting95Precision": 8, + "macroF1": 0.6817, + "macroPrecision": 0.9748, + "macroRecall": 0.5634, + "minimumPrecision": 0.931, + "perIntent": { + "blessing": { + "f1": 0.3486, + "falseNegative": 71, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 19, + "recall": 0.2111, + "total": 2970, + "trueNegative": 2880, + "truePositive": 19, + "wilsonPrecisionLower95": 0.8318 + }, + "complaint": { + "f1": 0.88, + "falseNegative": 74, + "falsePositive": 4, + "precision": 0.9862, + "predictedPositive": 290, + "recall": 0.7944, + "total": 2970, + "trueNegative": 2606, + "truePositive": 286, + "wilsonPrecisionLower95": 0.9651 + }, + "confirmationDecision": { + "f1": 0.7183, + "falseNegative": 78, + "falsePositive": 2, + "precision": 0.9808, + "predictedPositive": 104, + "recall": 0.5667, + "total": 2970, + "trueNegative": 2788, + "truePositive": 102, + "wilsonPrecisionLower95": 0.9326 + }, + "followUpReminder": { + "f1": 0.6345, + "falseNegative": 189, + "falsePositive": 8, + "precision": 0.9553, + "predictedPositive": 179, + "recall": 0.475, + "total": 2970, + "trueNegative": 2602, + "truePositive": 171, + "wilsonPrecisionLower95": 0.9143 + }, + "invitation": { + "f1": 0.4538, + "falseNegative": 126, + "falsePositive": 4, + "precision": 0.931, + "predictedPositive": 58, + "recall": 0.3, + "total": 2970, + "trueNegative": 2786, + "truePositive": 54, + "wilsonPrecisionLower95": 0.8357 + }, + "question": { + "f1": 0.7187, + "falseNegative": 271, + "falsePositive": 10, + "precision": 0.9729, + "predictedPositive": 369, + "recall": 0.5698, + "total": 2970, + "trueNegative": 2330, + "truePositive": 359, + "wilsonPrecisionLower95": 0.9508 + }, + "replyableMessage": { + "f1": 0.9005, + "falseNegative": 274, + "falsePositive": 83, + "precision": 0.9511, + "predictedPositive": 1699, + "recall": 0.855, + "total": 2970, + "trueNegative": 997, + "truePositive": 1616, + "wilsonPrecisionLower95": 0.9398 + }, + "scheduleNegotiation": { + "f1": 0.9831, + "falseNegative": 3, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 87, + "recall": 0.9667, + "total": 2970, + "trueNegative": 2880, + "truePositive": 87, + "wilsonPrecisionLower95": 0.9577 + }, + "task": { + "f1": 0.4979, + "falseNegative": 481, + "falsePositive": 1, + "precision": 0.9958, + "predictedPositive": 240, + "recall": 0.3319, + "total": 2970, + "trueNegative": 2249, + "truePositive": 239, + "wilsonPrecisionLower95": 0.9768 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 1.2e-06, + "augmentation": "numeric", + "char_max": 8, + "char_min": 3, + "external_weight": 0.93, + "hard_example_weight": 2.7, + "l1_ratio": 0.25, + "max_char_features": 95000, + "max_word_features": 45000, + "min_df": 1, + "pseudo_weight": 0.43499999999999994, + "round": 19, + "word_max": 3 + }, + "featureCount": 140000, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 1 + }, + "complaint": { + "negative": 250, + "positive": 2 + }, + "confirmationDecision": { + "negative": 250, + "positive": 36 + }, + "followUpReminder": { + "negative": 250, + "positive": 0 + }, + "invitation": { + "negative": 250, + "positive": 0 + }, + "question": { + "negative": 250, + "positive": 4 + }, + "replyableMessage": { + "negative": 99, + "positive": 32 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 0 + }, + "task": { + "negative": 250, + "positive": 0 + } + }, + "round": 19, + "selectionObjective": 0.7336, + "syntheticRegression": { + "intentsMeeting95Precision": 6, + "macroF1": 0.6005, + "macroPrecision": 0.9664, + "macroRecall": 0.4678, + "minimumPrecision": 0.887, + "perIntent": { + "blessing": { + "f1": 0.2, + "falseNegative": 80, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 10, + "recall": 0.1111, + "total": 2970, + "trueNegative": 2880, + "truePositive": 10, + "wilsonPrecisionLower95": 0.7225 + }, + "complaint": { + "f1": 0.9205, + "falseNegative": 53, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 307, + "recall": 0.8528, + "total": 2970, + "trueNegative": 2610, + "truePositive": 307, + "wilsonPrecisionLower95": 0.9876 + }, + "confirmationDecision": { + "f1": 0.6915, + "falseNegative": 78, + "falsePositive": 13, + "precision": 0.887, + "predictedPositive": 115, + "recall": 0.5667, + "total": 2970, + "trueNegative": 2777, + "truePositive": 102, + "wilsonPrecisionLower95": 0.8162 + }, + "followUpReminder": { + "f1": 0.4778, + "falseNegative": 247, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 113, + "recall": 0.3139, + "total": 2970, + "trueNegative": 2610, + "truePositive": 113, + "wilsonPrecisionLower95": 0.9671 + }, + "invitation": { + "f1": 0.7092, + "falseNegative": 80, + "falsePositive": 2, + "precision": 0.9804, + "predictedPositive": 102, + "recall": 0.5556, + "total": 2970, + "trueNegative": 2788, + "truePositive": 100, + "wilsonPrecisionLower95": 0.9313 + }, + "question": { + "f1": 0.4703, + "falseNegative": 436, + "falsePositive": 1, + "precision": 0.9949, + "predictedPositive": 195, + "recall": 0.3079, + "total": 2970, + "trueNegative": 2339, + "truePositive": 194, + "wilsonPrecisionLower95": 0.9715 + }, + "replyableMessage": { + "f1": 0.6399, + "falseNegative": 957, + "falsePositive": 93, + "precision": 0.9094, + "predictedPositive": 1026, + "recall": 0.4937, + "total": 2970, + "trueNegative": 987, + "truePositive": 933, + "wilsonPrecisionLower95": 0.8902 + }, + "scheduleNegotiation": { + "f1": 0.8079, + "falseNegative": 29, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 61, + "recall": 0.6778, + "total": 2970, + "trueNegative": 2880, + "truePositive": 61, + "wilsonPrecisionLower95": 0.9408 + }, + "task": { + "f1": 0.4872, + "falseNegative": 482, + "falsePositive": 19, + "precision": 0.9261, + "predictedPositive": 257, + "recall": 0.3306, + "total": 2970, + "trueNegative": 2231, + "truePositive": 238, + "wilsonPrecisionLower95": 0.8874 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "confirmationDecision": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "followUpReminder": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "replyableMessage": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 0.9309, + "falseNegative": 83, + "falsePositive": 45, + "precision": 0.9504, + "predictedPositive": 907, + "recall": 0.9122, + "total": 1485, + "trueNegative": 495, + "truePositive": 862, + "wilsonPrecisionLower95": 0.9343 + }, + "threshold": 0.98 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 45, + "recall": 1.0, + "total": 1485, + "trueNegative": 1440, + "truePositive": 45, + "wilsonPrecisionLower95": 0.9213 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.9195, + "falseNegative": 5, + "falsePositive": 2, + "precision": 0.9524, + "predictedPositive": 42, + "recall": 0.8889, + "total": 1485, + "trueNegative": 1438, + "truePositive": 40, + "wilsonPrecisionLower95": 0.8421 + }, + "threshold": 0.945 + } + }, + "metrics": { + "f1": 0.9667, + "falseNegative": 3, + "falsePositive": 3, + "precision": 0.9667, + "predictedPositive": 90, + "recall": 0.9667, + "total": 2970, + "trueNegative": 2877, + "truePositive": 87, + "wilsonPrecisionLower95": 0.9065 + }, + "threshold": 0.83 + }, + "task": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "trainingRecords": 16801, + "validation": { + "intentsMeeting95Precision": 6, + "macroF1": 0.6316, + "macroPrecision": 0.9616, + "macroRecall": 0.4969, + "minimumPrecision": 0.9242, + "perIntent": { + "blessing": { + "f1": 0.4483, + "falseNegative": 64, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 26, + "recall": 0.2889, + "total": 2970, + "trueNegative": 2880, + "truePositive": 26, + "wilsonPrecisionLower95": 0.8713 + }, + "complaint": { + "f1": 0.8953, + "falseNegative": 65, + "falsePositive": 4, + "precision": 0.9866, + "predictedPositive": 299, + "recall": 0.8194, + "total": 2970, + "trueNegative": 2606, + "truePositive": 295, + "wilsonPrecisionLower95": 0.9661 + }, + "confirmationDecision": { + "f1": 0.5891, + "falseNegative": 104, + "falsePositive": 2, + "precision": 0.9744, + "predictedPositive": 78, + "recall": 0.4222, + "total": 2970, + "trueNegative": 2788, + "truePositive": 76, + "wilsonPrecisionLower95": 0.9112 + }, + "followUpReminder": { + "f1": 0.528, + "falseNegative": 228, + "falsePositive": 8, + "precision": 0.9429, + "predictedPositive": 140, + "recall": 0.3667, + "total": 2970, + "trueNegative": 2602, + "truePositive": 132, + "wilsonPrecisionLower95": 0.8913 + }, + "invitation": { + "f1": 0.6015, + "falseNegative": 100, + "falsePositive": 6, + "precision": 0.9302, + "predictedPositive": 86, + "recall": 0.4444, + "total": 2970, + "trueNegative": 2784, + "truePositive": 80, + "wilsonPrecisionLower95": 0.856 + }, + "question": { + "f1": 0.442, + "falseNegative": 447, + "falsePositive": 15, + "precision": 0.9242, + "predictedPositive": 198, + "recall": 0.2905, + "total": 2970, + "trueNegative": 2325, + "truePositive": 183, + "wilsonPrecisionLower95": 0.8788 + }, + "replyableMessage": { + "f1": 0.6347, + "falseNegative": 990, + "falsePositive": 46, + "precision": 0.9514, + "predictedPositive": 946, + "recall": 0.4762, + "total": 2970, + "trueNegative": 1034, + "truePositive": 900, + "wilsonPrecisionLower95": 0.9357 + }, + "scheduleNegotiation": { + "f1": 0.9605, + "falseNegative": 5, + "falsePositive": 2, + "precision": 0.977, + "predictedPositive": 87, + "recall": 0.9444, + "total": 2970, + "trueNegative": 2878, + "truePositive": 85, + "wilsonPrecisionLower95": 0.92 + }, + "task": { + "f1": 0.5853, + "falseNegative": 418, + "falsePositive": 10, + "precision": 0.9679, + "predictedPositive": 312, + "recall": 0.4194, + "total": 2970, + "trueNegative": 2240, + "truePositive": 302, + "wilsonPrecisionLower95": 0.942 + } + }, + "recordCount": 2970 + } + }, + { + "configuration": { + "alpha": 1e-06, + "augmentation": "numeric", + "char_max": 8, + "char_min": 2, + "external_weight": 0.9500000000000001, + "hard_example_weight": 3.0, + "l1_ratio": 0.25, + "max_char_features": 100000, + "max_word_features": 50000, + "min_df": 1, + "pseudo_weight": 0.44999999999999996, + "round": 20, + "word_max": 3 + }, + "featureCount": 150000, + "pseudoLabelCounts": { + "blessing": { + "negative": 250, + "positive": 1 + }, + "complaint": { + "negative": 250, + "positive": 2 + }, + "confirmationDecision": { + "negative": 250, + "positive": 25 + }, + "followUpReminder": { + "negative": 250, + "positive": 0 + }, + "invitation": { + "negative": 250, + "positive": 1 + }, + "question": { + "negative": 250, + "positive": 2 + }, + "replyableMessage": { + "negative": 104, + "positive": 32 + }, + "scheduleNegotiation": { + "negative": 250, + "positive": 1 + }, + "task": { + "negative": 250, + "positive": 0 + } + }, + "round": 20, + "selectionObjective": 0.7724, + "syntheticRegression": { + "intentsMeeting95Precision": 7, + "macroF1": 0.6637, + "macroPrecision": 0.9685, + "macroRecall": 0.5421, + "minimumPrecision": 0.9094, + "perIntent": { + "blessing": { + "f1": 0.2353, + "falseNegative": 78, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 12, + "recall": 0.1333, + "total": 2970, + "trueNegative": 2880, + "truePositive": 12, + "wilsonPrecisionLower95": 0.7575 + }, + "complaint": { + "f1": 0.8274, + "falseNegative": 106, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 254, + "recall": 0.7056, + "total": 2970, + "trueNegative": 2610, + "truePositive": 254, + "wilsonPrecisionLower95": 0.9851 + }, + "confirmationDecision": { + "f1": 0.8261, + "falseNegative": 47, + "falsePositive": 9, + "precision": 0.9366, + "predictedPositive": 142, + "recall": 0.7389, + "total": 2970, + "trueNegative": 2781, + "truePositive": 133, + "wilsonPrecisionLower95": 0.8839 + }, + "followUpReminder": { + "f1": 0.5155, + "falseNegative": 235, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 125, + "recall": 0.3472, + "total": 2970, + "trueNegative": 2610, + "truePositive": 125, + "wilsonPrecisionLower95": 0.9702 + }, + "invitation": { + "f1": 0.7241, + "falseNegative": 75, + "falsePositive": 5, + "precision": 0.9545, + "predictedPositive": 110, + "recall": 0.5833, + "total": 2970, + "trueNegative": 2785, + "truePositive": 105, + "wilsonPrecisionLower95": 0.898 + }, + "question": { + "f1": 0.5534, + "falseNegative": 389, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 241, + "recall": 0.3825, + "total": 2970, + "trueNegative": 2340, + "truePositive": 241, + "wilsonPrecisionLower95": 0.9843 + }, + "replyableMessage": { + "f1": 0.7688, + "falseNegative": 671, + "falsePositive": 62, + "precision": 0.9516, + "predictedPositive": 1281, + "recall": 0.645, + "total": 2970, + "trueNegative": 1018, + "truePositive": 1219, + "wilsonPrecisionLower95": 0.9384 + }, + "scheduleNegotiation": { + "f1": 0.9371, + "falseNegative": 8, + "falsePositive": 3, + "precision": 0.9647, + "predictedPositive": 85, + "recall": 0.9111, + "total": 2970, + "trueNegative": 2877, + "truePositive": 82, + "wilsonPrecisionLower95": 0.9013 + }, + "task": { + "f1": 0.5857, + "falseNegative": 409, + "falsePositive": 31, + "precision": 0.9094, + "predictedPositive": 342, + "recall": 0.4319, + "total": 2970, + "trueNegative": 2219, + "truePositive": 311, + "wilsonPrecisionLower95": 0.8742 + } + }, + "recordCount": 2970 + }, + "thresholds": { + "blessing": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "complaint": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "confirmationDecision": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.5954, + "falseNegative": 51, + "falsePositive": 2, + "precision": 0.9512, + "predictedPositive": 41, + "recall": 0.4333, + "total": 1485, + "trueNegative": 1393, + "truePositive": 39, + "wilsonPrecisionLower95": 0.8386 + }, + "threshold": 0.995 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "followUpReminder": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "invitation": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "question": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "replyableMessage": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.8117, + "falseNegative": 279, + "falsePositive": 30, + "precision": 0.9569, + "predictedPositive": 696, + "recall": 0.7048, + "total": 1485, + "trueNegative": 510, + "truePositive": 666, + "wilsonPrecisionLower95": 0.9391 + }, + "threshold": 0.995 + } + }, + "metrics": { + "f1": 0.8736, + "falseNegative": 363, + "falsePositive": 79, + "precision": 0.9508, + "predictedPositive": 1606, + "recall": 0.8079, + "total": 2970, + "trueNegative": 1001, + "truePositive": 1527, + "wilsonPrecisionLower95": 0.9391 + }, + "threshold": 0.995 + }, + "scheduleNegotiation": { + "abstained": false, + "byLanguage": { + "en": { + "abstained": false, + "metrics": { + "f1": 1.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 45, + "recall": 1.0, + "total": 1485, + "trueNegative": 1440, + "truePositive": 45, + "wilsonPrecisionLower95": 0.9213 + }, + "threshold": 0.5 + }, + "zh-Hans": { + "abstained": false, + "metrics": { + "f1": 0.967, + "falseNegative": 1, + "falsePositive": 2, + "precision": 0.9565, + "predictedPositive": 46, + "recall": 0.9778, + "total": 1485, + "trueNegative": 1438, + "truePositive": 44, + "wilsonPrecisionLower95": 0.8547 + }, + "threshold": 0.83 + } + }, + "metrics": { + "f1": 0.9834, + "falseNegative": 1, + "falsePositive": 2, + "precision": 0.978, + "predictedPositive": 91, + "recall": 0.9889, + "total": 2970, + "trueNegative": 2878, + "truePositive": 89, + "wilsonPrecisionLower95": 0.9234 + }, + "threshold": 0.83 + }, + "task": { + "abstained": true, + "byLanguage": { + "en": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + }, + "zh-Hans": { + "abstained": true, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "metrics": { + "f1": 0.0, + "falseNegative": 0, + "falsePositive": 0, + "precision": 0.0, + "predictedPositive": 0, + "recall": 0.0, + "total": 0, + "trueNegative": 0, + "truePositive": 0, + "wilsonPrecisionLower95": 0 + }, + "threshold": 1.0 + } + }, + "trainingRecords": 16755, + "validation": { + "intentsMeeting95Precision": 8, + "macroF1": 0.6867, + "macroPrecision": 0.9687, + "macroRecall": 0.5569, + "minimumPrecision": 0.9038, + "perIntent": { + "blessing": { + "f1": 0.3929, + "falseNegative": 68, + "falsePositive": 0, + "precision": 1.0, + "predictedPositive": 22, + "recall": 0.2444, + "total": 2970, + "trueNegative": 2880, + "truePositive": 22, + "wilsonPrecisionLower95": 0.8513 + }, + "complaint": { + "f1": 0.8066, + "falseNegative": 114, + "falsePositive": 4, + "precision": 0.984, + "predictedPositive": 250, + "recall": 0.6833, + "total": 2970, + "trueNegative": 2606, + "truePositive": 246, + "wilsonPrecisionLower95": 0.9596 + }, + "confirmationDecision": { + "f1": 0.6618, + "falseNegative": 90, + "falsePositive": 2, + "precision": 0.9783, + "predictedPositive": 92, + "recall": 0.5, + "total": 2970, + "trueNegative": 2788, + "truePositive": 90, + "wilsonPrecisionLower95": 0.9242 + }, + "followUpReminder": { + "f1": 0.5977, + "falseNegative": 204, + "falsePositive": 6, + "precision": 0.963, + "predictedPositive": 162, + "recall": 0.4333, + "total": 2970, + "trueNegative": 2604, + "truePositive": 156, + "wilsonPrecisionLower95": 0.9216 + }, + "invitation": { + "f1": 0.662, + "falseNegative": 86, + "falsePositive": 10, + "precision": 0.9038, + "predictedPositive": 104, + "recall": 0.5222, + "total": 2970, + "trueNegative": 2780, + "truePositive": 94, + "wilsonPrecisionLower95": 0.832 + }, + "question": { + "f1": 0.5643, + "falseNegative": 380, + "falsePositive": 6, + "precision": 0.9766, + "predictedPositive": 256, + "recall": 0.3968, + "total": 2970, + "trueNegative": 2334, + "truePositive": 250, + "wilsonPrecisionLower95": 0.9498 + }, + "replyableMessage": { + "f1": 0.7934, + "falseNegative": 621, + "falsePositive": 40, + "precision": 0.9694, + "predictedPositive": 1309, + "recall": 0.6714, + "total": 2970, + "trueNegative": 1040, + "truePositive": 1269, + "wilsonPrecisionLower95": 0.9587 + }, + "scheduleNegotiation": { + "f1": 0.9834, + "falseNegative": 1, + "falsePositive": 2, + "precision": 0.978, + "predictedPositive": 91, + "recall": 0.9889, + "total": 2970, + "trueNegative": 2878, + "truePositive": 89, + "wilsonPrecisionLower95": 0.9234 + }, + "task": { + "f1": 0.7184, + "falseNegative": 308, + "falsePositive": 15, + "precision": 0.9649, + "predictedPositive": 427, + "recall": 0.5722, + "total": 2970, + "trueNegative": 2235, + "truePositive": 412, + "wilsonPrecisionLower95": 0.9429 + } + }, + "recordCount": 2970 + } + } +] diff --git a/ModelTraining/ClipboardSemantics/README.md b/ModelTraining/ClipboardSemantics/README.md index 2f0d529..9f2469b 100644 --- a/ModelTraining/ClipboardSemantics/README.md +++ b/ModelTraining/ClipboardSemantics/README.md @@ -122,6 +122,48 @@ Without human-reviewed gold labels, these results measure agreement with the model committee, not production truth. A passing verifier may enter shadow deployment, but cannot be described as having 95% real-user precision. +## Iterative weakly supervised research + +The portable research harness runs a fixed 20-round matrix over bilingual +character/word n-grams, sparse logistic classifiers, class balancing, +deterministic label-preserving augmentation, hard-example weighting, and +three-round high-confidence self-training consensus: + +```bash +python3 -m pip install -r \ + Scripts/clipboard_semantics/requirements-research.txt +python3 Scripts/clipboard_semantics/generate_open_training_corpus.py \ + --allow-unavailable-sources +python3 Scripts/clipboard_semantics/run_iterative_retraining.py +``` + +The harness fits thresholds only on the generated validation split. Synthetic +test, golden, random, targeted-release, and comprehensive online corpora never +enter training or threshold calibration. Exact normalized overlap with every +frozen holdout is a fatal error. + +The 2026-08-27 study completed the requested 20 rounds and two additional +20-round fine-tuning phases after the first phase missed its release target: + +- The shared-configuration phase selected round 1. Random-holdout macro F1 was + `0.5709`; research-only comprehensive macro F1 was `0.2383`. +- Per-intent selection improved those values to `0.5770` and `0.2645`. +- Explicit evidence gates raised golden macro precision to `0.9877` and + research-only precision to `0.6097`, but reduced recall too severely. +- The current deployed reference remains stronger: random-holdout macro F1 + `0.7669` and research-only comprehensive macro F1 `0.2783`. + +All three phases failed the release gate, so no model was promoted. Detailed +rounds, final evaluations, source breakdowns, and macOS replay instructions are +stored under `IterativeResearch/`. + +This harness is deliberately a Linux surrogate. It cannot emit the +`NLModel`-compatible Create ML artifacts used by the keyboard extension. +Deployable training, Core ML compilation, simulator regression, latency, and +memory checks still require macOS with Xcode 26+. A surrogate result can +nominate a data/threshold policy for macOS replay, but cannot authorize +automatic deployment. + ## Deployment decision Only maxEnt models are trained and deployed because they are self-contained in diff --git a/ModelTraining/ClipboardSemantics/open-training-sources.json b/ModelTraining/ClipboardSemantics/open-training-sources.json index 3689648..025d266 100644 --- a/ModelTraining/ClipboardSemantics/open-training-sources.json +++ b/ModelTraining/ClipboardSemantics/open-training-sources.json @@ -1,9 +1,9 @@ { "baseCorpusRecords": 17692, - "combinedCorpusSHA256": "70d1547b41b08e4417cfa86ad93dd5637b557d174c6503575c44fbf152cc3ee5", - "combinedRecords": 23918, + "combinedCorpusSHA256": "f111bc7425912213082e01038b52dd3d25fe21418d156f532da8f01f27fa16e4", + "combinedRecords": 21537, "excludedBaseOverlap": 6, - "excludedHoldoutOverlap": 151, + "excludedHoldoutOverlap": 72, "excludedSources": [ { "dataset": "MIDAS", @@ -38,72 +38,11 @@ "reason": "No clean official train split independent from the frozen holdout." } ], - "openTrainingRecords": 6226, + "openTrainingRecords": 3845, "policy": "Licensed official training splits only. Per-record knownLabels prevent unannotated intents from becoming false negatives. Exact normalized text overlap with every local *holdout-corpus.jsonl file is excluded.", "schemaVersion": 1, "seed": 20260827, "sources": [ - { - "dataset": "AmazonScience/MASSIVE en-US", - "knownLabelCounts": { - "followUpReminder": 7, - "question": 223, - "sentiment": 4, - "task": 143 - }, - "languages": { - "en": 498 - }, - "license": "CC-BY-4.0", - "records": 498, - "revision": "1.1", - "url": "https://huggingface.co/datasets/AmazonScience/massive" - }, - { - "dataset": "AmazonScience/MASSIVE zh-CN", - "knownLabelCounts": { - "followUpReminder": 3, - "question": 212, - "sentiment": 6, - "task": 165 - }, - "languages": { - "zh-Hans": 487 - }, - "license": "CC-BY-4.0", - "records": 487, - "revision": "1.1", - "url": "https://huggingface.co/datasets/AmazonScience/massive" - }, - { - "dataset": "CFPB via hpe-ai/customer-complaints", - "knownLabelCounts": { - "complaint": 898, - "sentiment": 898 - }, - "languages": { - "en": 898 - }, - "license": "CC0-1.0 source / Apache-2.0 mirror", - "records": 898, - "revision": "e5fec64e1f0688e47699b9cf8c26fe4ed350123a", - "url": "https://www.consumerfinance.gov/data-research/consumer-complaints/search/detail/4696537" - }, - { - "dataset": "CLINC150 UCI", - "knownLabelCounts": { - "followUpReminder": 5, - "question": 308, - "task": 143 - }, - "languages": { - "en": 498 - }, - "license": "CC-BY-3.0", - "records": 498, - "revision": "UCI-570-2020-05-07", - "url": "https://archive.ics.uci.edu/dataset/570/clinc150" - }, { "dataset": "CrossWOZ", "knownLabelCounts": { @@ -121,13 +60,13 @@ { "dataset": "Google Research GoEmotions", "knownLabelCounts": { - "sentiment": 799 + "sentiment": 798 }, "languages": { - "en": 1199 + "en": 1198 }, "license": "Apache-2.0", - "records": 1199, + "records": 1198, "revision": "5d8f4ac97c873bde3a792ba4628f00bb9103d3e6", "url": "https://github.com/google-research/google-research/blob/5d8f4ac97c873bde3a792ba4628f00bb9103d3e6/goemotions/data/train.tsv" }, @@ -164,17 +103,31 @@ { "dataset": "MultiDoGO", "knownLabelCounts": { - "confirmationDecision": 28, + "confirmationDecision": 27, "question": 83, "task": 222 }, "languages": { - "en": 464 + "en": 465 }, "license": "CDLA-Permissive-1.0", - "records": 464, + "records": 465, "revision": "baa30639c4b271f394b81443c842193407cdf26d", - "url": "https://github.com/awslabs/multi-domain-goal-oriented-dialogues-dataset/blob/baa30639c4b271f394b81443c842193407cdf26d/data/paper_splits/splits_annotated_at_turn_level/insurance/train.tsv" + "url": "https://github.com/awslabs/multi-domain-goal-oriented-dialogues-dataset/blob/baa30639c4b271f394b81443c842193407cdf26d/data/paper_splits/splits_annotated_at_turn_level/software/train.tsv" + } + ], + "unavailableSources": [ + { + "dataset": "MASSIVE", + "reason": "URLError: " + }, + { + "dataset": "CLINC150", + "reason": "URLError: " + }, + { + "dataset": "CFPB", + "reason": "URLError: " } ] }