{ "baseTrainingRecords": 11500, "deploymentDecision": "retain-current-deployed-models", "elapsedSeconds": 170.1895, "finalEvaluations": { "golden": { "byLanguage": { "en": { "intentsMeeting95Precision": 1, "macroF1": 0.8445, "macroPrecision": 0.9286, "macroRecall": 0.8066, "minimumPrecision": 0.7692, "perIntent": { "blessing": { "f1": 0.8235, "falseNegative": 3, "falsePositive": 0, "precision": 1.0, "predictedPositive": 7, "recall": 0.7, "total": 126, "trueNegative": 116, "truePositive": 7, "wilsonPrecisionLower95": 0.6457 }, "complaint": { "f1": 1.0, "falseNegative": 0, "falsePositive": 0, "precision": 1.0, "predictedPositive": 6, "recall": 1.0, "total": 126, "trueNegative": 120, "truePositive": 6, "wilsonPrecisionLower95": 0.6097 }, "confirmationDecision": { "f1": 0.8696, "falseNegative": 0, "falsePositive": 3, "precision": 0.7692, "predictedPositive": 13, "recall": 1.0, "total": 126, "trueNegative": 113, "truePositive": 10, "wilsonPrecisionLower95": 0.4974 }, "followUpReminder": { "f1": 0.8696, "falseNegative": 3, "falsePositive": 0, "precision": 1.0, "predictedPositive": 10, "recall": 0.7692, "total": 126, "trueNegative": 113, "truePositive": 10, "wilsonPrecisionLower95": 0.7225 }, "invitation": { "f1": 0.8571, "falseNegative": 2, "falsePositive": 0, "precision": 1.0, "predictedPositive": 6, "recall": 0.75, "total": 126, "trueNegative": 118, "truePositive": 6, "wilsonPrecisionLower95": 0.6097 }, "question": { "f1": 0.8636, "falseNegative": 4, "falsePositive": 2, "precision": 0.9048, "predictedPositive": 21, "recall": 0.8261, "total": 126, "trueNegative": 101, "truePositive": 19, "wilsonPrecisionLower95": 0.7109 }, "replyableMessage": { "f1": 0.8889, "falseNegative": 6, "falsePositive": 10, "precision": 0.8649, "predictedPositive": 74, "recall": 0.9143, "total": 126, "trueNegative": 46, "truePositive": 64, "wilsonPrecisionLower95": 0.7688 }, "scheduleNegotiation": { "f1": 0.8571, "falseNegative": 1, "falsePositive": 2, "precision": 0.8182, "predictedPositive": 11, "recall": 0.9, "total": 126, "trueNegative": 114, "truePositive": 9, "wilsonPrecisionLower95": 0.523 }, "task": { "f1": 0.5714, "falseNegative": 9, "falsePositive": 0, "precision": 1.0, "predictedPositive": 6, "recall": 0.4, "total": 126, "trueNegative": 111, "truePositive": 6, "wilsonPrecisionLower95": 0.6097 } }, "recordCount": 126 }, "zh-Hans": { "intentsMeeting95Precision": 1, "macroF1": 0.6673, "macroPrecision": 0.9541, "macroRecall": 0.5535, "minimumPrecision": 0.7143, "perIntent": { "blessing": { "f1": 0.4615, "falseNegative": 7, "falsePositive": 0, "precision": 1.0, "predictedPositive": 3, "recall": 0.3, "total": 126, "trueNegative": 116, "truePositive": 3, "wilsonPrecisionLower95": 0.4385 }, "complaint": { "f1": 0.6667, "falseNegative": 3, "falsePositive": 0, "precision": 1.0, "predictedPositive": 3, "recall": 0.5, "total": 126, "trueNegative": 120, "truePositive": 3, "wilsonPrecisionLower95": 0.4385 }, "confirmationDecision": { "f1": 0.5882, "falseNegative": 5, "falsePositive": 2, "precision": 0.7143, "predictedPositive": 7, "recall": 0.5, "total": 126, "trueNegative": 114, "truePositive": 5, "wilsonPrecisionLower95": 0.3589 }, "followUpReminder": { "f1": 0.8182, "falseNegative": 4, "falsePositive": 0, "precision": 1.0, "predictedPositive": 9, "recall": 0.6923, "total": 126, "trueNegative": 113, "truePositive": 9, "wilsonPrecisionLower95": 0.7009 }, "invitation": { "f1": 0.5455, "falseNegative": 5, "falsePositive": 0, "precision": 1.0, "predictedPositive": 3, "recall": 0.375, "total": 126, "trueNegative": 118, "truePositive": 3, "wilsonPrecisionLower95": 0.4385 }, "question": { "f1": 0.9787, "falseNegative": 0, "falsePositive": 1, "precision": 0.9583, "predictedPositive": 24, "recall": 1.0, "total": 126, "trueNegative": 102, "truePositive": 23, "wilsonPrecisionLower95": 0.7976 }, "replyableMessage": { "f1": 0.9143, "falseNegative": 6, "falsePositive": 6, "precision": 0.9143, "predictedPositive": 70, "recall": 0.9143, "total": 126, "trueNegative": 50, "truePositive": 64, "wilsonPrecisionLower95": 0.8253 }, "scheduleNegotiation": { "f1": 0.4615, "falseNegative": 7, "falsePositive": 0, "precision": 1.0, "predictedPositive": 3, "recall": 0.3, "total": 126, "trueNegative": 116, "truePositive": 3, "wilsonPrecisionLower95": 0.4385 }, "task": { "f1": 0.5714, "falseNegative": 9, "falsePositive": 0, "precision": 1.0, "predictedPositive": 6, "recall": 0.4, "total": 126, "trueNegative": 111, "truePositive": 6, "wilsonPrecisionLower95": 0.6097 } }, "recordCount": 126 } }, "bySource": { "acknowledgment": { "intentsMeeting95Precision": 0, "macroF1": 0.0, "macroPrecision": 0.0, "macroRecall": 0.0, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 8, "trueNegative": 8, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 8, "trueNegative": 8, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 1, "precision": 0.0, "predictedPositive": 1, "recall": 0.0, "total": 8, "trueNegative": 7, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 8, "trueNegative": 8, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 8, "trueNegative": 8, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 8, "trueNegative": 8, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 8, "trueNegative": 8, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 8, "trueNegative": 8, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 8, "trueNegative": 8, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 8 }, "blessing_boundary": { "intentsMeeting95Precision": 0, "macroF1": 0.1482, "macroPrecision": 0.1667, "macroRecall": 0.1667, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.6667, "falseNegative": 1, "falsePositive": 0, "precision": 1.0, "predictedPositive": 1, "recall": 0.5, "total": 20, "trueNegative": 18, "truePositive": 1, "wilsonPrecisionLower95": 0.2065 }, "replyableMessage": { "f1": 0.6667, "falseNegative": 0, "falsePositive": 4, "precision": 0.5, "predictedPositive": 8, "recall": 1.0, "total": 20, "trueNegative": 12, "truePositive": 4, "wilsonPrecisionLower95": 0.2152 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 2, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 18, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 20 }, "blessing_golden": { "intentsMeeting95Precision": 2, "macroF1": 0.1823, "macroPrecision": 0.2222, "macroRecall": 0.1611, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.6667, "falseNegative": 10, "falsePositive": 0, "precision": 1.0, "predictedPositive": 10, "recall": 0.5, "total": 20, "trueNegative": 0, "truePositive": 10, "wilsonPrecisionLower95": 0.7225 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.9744, "falseNegative": 1, "falsePositive": 0, "precision": 1.0, "predictedPositive": 19, "recall": 0.95, "total": 20, "trueNegative": 0, "truePositive": 19, "wilsonPrecisionLower95": 0.8318 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 20 }, "complaint_request": { "intentsMeeting95Precision": 0, "macroF1": 0.2703, "macroPrecision": 0.3333, "macroRecall": 0.2315, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.6667, "falseNegative": 3, "falsePositive": 0, "precision": 1.0, "predictedPositive": 3, "recall": 0.5, "total": 6, "trueNegative": 0, "truePositive": 3, "wilsonPrecisionLower95": 0.4385 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.8571, "falseNegative": 1, "falsePositive": 0, "precision": 1.0, "predictedPositive": 3, "recall": 0.75, "total": 6, "trueNegative": 2, "truePositive": 3, "wilsonPrecisionLower95": 0.4385 }, "replyableMessage": { "f1": 0.9091, "falseNegative": 1, "falsePositive": 0, "precision": 1.0, "predictedPositive": 5, "recall": 0.8333, "total": 6, "trueNegative": 0, "truePositive": 5, "wilsonPrecisionLower95": 0.5655 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 6 }, "complaint_statement": { "intentsMeeting95Precision": 0, "macroF1": 0.2222, "macroPrecision": 0.2222, "macroRecall": 0.2222, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 1.0, "falseNegative": 0, "falsePositive": 0, "precision": 1.0, "predictedPositive": 6, "recall": 1.0, "total": 6, "trueNegative": 0, "truePositive": 6, "wilsonPrecisionLower95": 0.6097 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 1.0, "falseNegative": 0, "falsePositive": 0, "precision": 1.0, "predictedPositive": 6, "recall": 1.0, "total": 6, "trueNegative": 0, "truePositive": 6, "wilsonPrecisionLower95": 0.6097 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 6 }, "confirmation_decision_boundary": { "intentsMeeting95Precision": 0, "macroF1": 0.0, "macroPrecision": 0.0, "macroRecall": 0.0, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 3, "precision": 0.0, "predictedPositive": 3, "recall": 0.0, "total": 20, "trueNegative": 17, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.0, "falseNegative": 0, "falsePositive": 3, "precision": 0.0, "predictedPositive": 3, "recall": 0.0, "total": 20, "trueNegative": 17, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 20 }, "confirmation_decision_golden": { "intentsMeeting95Precision": 2, "macroF1": 0.2063, "macroPrecision": 0.2222, "macroRecall": 0.1944, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.8571, "falseNegative": 5, "falsePositive": 0, "precision": 1.0, "predictedPositive": 15, "recall": 0.75, "total": 20, "trueNegative": 0, "truePositive": 15, "wilsonPrecisionLower95": 0.7961 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 1.0, "falseNegative": 0, "falsePositive": 0, "precision": 1.0, "predictedPositive": 20, "recall": 1.0, "total": 20, "trueNegative": 0, "truePositive": 20, "wilsonPrecisionLower95": 0.8389 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 20 }, "conversational_message": { "intentsMeeting95Precision": 1, "macroF1": 0.1111, "macroPrecision": 0.1111, "macroRecall": 0.1111, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 10, "trueNegative": 10, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 10, "trueNegative": 10, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 10, "trueNegative": 10, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 10, "trueNegative": 10, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 10, "trueNegative": 10, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 10, "trueNegative": 10, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 1.0, "falseNegative": 0, "falsePositive": 0, "precision": 1.0, "predictedPositive": 10, "recall": 1.0, "total": 10, "trueNegative": 0, "truePositive": 10, "wilsonPrecisionLower95": 0.7225 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 10, "trueNegative": 10, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 10, "trueNegative": 10, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 10 }, "event_statement": { "intentsMeeting95Precision": 0, "macroF1": 0.0, "macroPrecision": 0.0, "macroRecall": 0.0, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 4, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 4, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 4, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 4, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 4, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 4, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 4, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 4, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 4, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 4 }, "follow_up_reminder_boundary": { "intentsMeeting95Precision": 0, "macroF1": 0.0, "macroPrecision": 0.0, "macroRecall": 0.0, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.0, "falseNegative": 0, "falsePositive": 3, "precision": 0.0, "predictedPositive": 3, "recall": 0.0, "total": 20, "trueNegative": 17, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 20 }, "follow_up_reminder_golden": { "intentsMeeting95Precision": 1, "macroF1": 0.2187, "macroPrecision": 0.3333, "macroRecall": 0.1706, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.8571, "falseNegative": 5, "falsePositive": 0, "precision": 1.0, "predictedPositive": 15, "recall": 0.75, "total": 20, "trueNegative": 0, "truePositive": 15, "wilsonPrecisionLower95": 0.7961 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.6667, "falseNegative": 7, "falsePositive": 0, "precision": 1.0, "predictedPositive": 7, "recall": 0.5, "total": 20, "trueNegative": 6, "truePositive": 7, "wilsonPrecisionLower95": 0.6457 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.4444, "falseNegative": 10, "falsePositive": 0, "precision": 1.0, "predictedPositive": 4, "recall": 0.2857, "total": 20, "trueNegative": 6, "truePositive": 4, "wilsonPrecisionLower95": 0.5101 } }, "recordCount": 20 }, "information_question": { "intentsMeeting95Precision": 0, "macroF1": 0.1693, "macroPrecision": 0.2222, "macroRecall": 0.1389, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 4, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 4, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 4, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 4, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 4, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.6667, "falseNegative": 2, "falsePositive": 0, "precision": 1.0, "predictedPositive": 2, "recall": 0.5, "total": 4, "trueNegative": 0, "truePositive": 2, "wilsonPrecisionLower95": 0.3424 }, "replyableMessage": { "f1": 0.8571, "falseNegative": 1, "falsePositive": 0, "precision": 1.0, "predictedPositive": 3, "recall": 0.75, "total": 4, "trueNegative": 0, "truePositive": 3, "wilsonPrecisionLower95": 0.4385 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 4, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 4, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 4 }, "invitation_question": { "intentsMeeting95Precision": 0, "macroF1": 0.3175, "macroPrecision": 0.3333, "macroRecall": 0.3056, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 8, "trueNegative": 8, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 8, "trueNegative": 8, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 8, "trueNegative": 8, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 8, "trueNegative": 8, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.8571, "falseNegative": 2, "falsePositive": 0, "precision": 1.0, "predictedPositive": 6, "recall": 0.75, "total": 8, "trueNegative": 0, "truePositive": 6, "wilsonPrecisionLower95": 0.6097 }, "question": { "f1": 1.0, "falseNegative": 0, "falsePositive": 0, "precision": 1.0, "predictedPositive": 8, "recall": 1.0, "total": 8, "trueNegative": 0, "truePositive": 8, "wilsonPrecisionLower95": 0.6756 }, "replyableMessage": { "f1": 1.0, "falseNegative": 0, "falsePositive": 0, "precision": 1.0, "predictedPositive": 8, "recall": 1.0, "total": 8, "trueNegative": 0, "truePositive": 8, "wilsonPrecisionLower95": 0.6756 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 8, "trueNegative": 8, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 8, "trueNegative": 8, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 8 }, "negative_news": { "intentsMeeting95Precision": 0, "macroF1": 0.0, "macroPrecision": 0.0, "macroRecall": 0.0, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.0, "falseNegative": 0, "falsePositive": 1, "precision": 0.0, "predictedPositive": 1, "recall": 0.0, "total": 6, "trueNegative": 5, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 6 }, "neutral_fact": { "intentsMeeting95Precision": 0, "macroF1": 0.0, "macroPrecision": 0.0, "macroRecall": 0.0, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 10, "trueNegative": 10, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 10, "trueNegative": 10, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 10, "trueNegative": 10, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 10, "trueNegative": 10, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 10, "trueNegative": 10, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 10, "trueNegative": 10, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.0, "falseNegative": 0, "falsePositive": 2, "precision": 0.0, "predictedPositive": 2, "recall": 0.0, "total": 10, "trueNegative": 8, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 1, "precision": 0.0, "predictedPositive": 1, "recall": 0.0, "total": 10, "trueNegative": 9, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 10, "trueNegative": 10, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 10 }, "positive_feedback": { "intentsMeeting95Precision": 0, "macroF1": 0.1111, "macroPrecision": 0.1111, "macroRecall": 0.1111, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 1.0, "falseNegative": 0, "falsePositive": 0, "precision": 1.0, "predictedPositive": 6, "recall": 1.0, "total": 6, "trueNegative": 0, "truePositive": 6, "wilsonPrecisionLower95": 0.6097 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 6 }, "quoted_question": { "intentsMeeting95Precision": 0, "macroF1": 0.0, "macroPrecision": 0.0, "macroRecall": 0.0, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 4, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 4, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 4, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 4, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 4, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 4, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 4, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 4, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 4, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 4 }, "schedule_negotiation_boundary": { "intentsMeeting95Precision": 0, "macroF1": 0.2541, "macroPrecision": 0.284, "macroRecall": 0.2639, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 1, "precision": 0.0, "predictedPositive": 1, "recall": 0.0, "total": 20, "trueNegative": 19, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.5455, "falseNegative": 5, "falsePositive": 0, "precision": 1.0, "predictedPositive": 3, "recall": 0.375, "total": 20, "trueNegative": 12, "truePositive": 3, "wilsonPrecisionLower95": 0.4385 }, "question": { "f1": 0.8, "falseNegative": 0, "falsePositive": 2, "precision": 0.6667, "predictedPositive": 6, "recall": 1.0, "total": 20, "trueNegative": 14, "truePositive": 4, "wilsonPrecisionLower95": 0.3 }, "replyableMessage": { "f1": 0.9412, "falseNegative": 0, "falsePositive": 1, "precision": 0.8889, "predictedPositive": 9, "recall": 1.0, "total": 20, "trueNegative": 11, "truePositive": 8, "wilsonPrecisionLower95": 0.565 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 1, "precision": 0.0, "predictedPositive": 1, "recall": 0.0, "total": 20, "trueNegative": 19, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 20 }, "schedule_negotiation_golden": { "intentsMeeting95Precision": 2, "macroF1": 0.2997, "macroPrecision": 0.3275, "macroRecall": 0.2833, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.973, "falseNegative": 0, "falsePositive": 1, "precision": 0.9474, "predictedPositive": 19, "recall": 1.0, "total": 20, "trueNegative": 1, "truePositive": 18, "wilsonPrecisionLower95": 0.7536 }, "replyableMessage": { "f1": 0.9744, "falseNegative": 1, "falsePositive": 0, "precision": 1.0, "predictedPositive": 19, "recall": 0.95, "total": 20, "trueNegative": 0, "truePositive": 19, "wilsonPrecisionLower95": 0.8318 }, "scheduleNegotiation": { "f1": 0.75, "falseNegative": 8, "falsePositive": 0, "precision": 1.0, "predictedPositive": 12, "recall": 0.6, "total": 20, "trueNegative": 0, "truePositive": 12, "wilsonPrecisionLower95": 0.7575 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 20 }, "self_plan": { "intentsMeeting95Precision": 0, "macroF1": 0.1111, "macroPrecision": 0.1111, "macroRecall": 0.1111, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 1.0, "falseNegative": 0, "falsePositive": 0, "precision": 1.0, "predictedPositive": 2, "recall": 1.0, "total": 6, "trueNegative": 4, "truePositive": 2, "wilsonPrecisionLower95": 0.3424 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.0, "falseNegative": 0, "falsePositive": 2, "precision": 0.0, "predictedPositive": 2, "recall": 0.0, "total": 6, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 6 }, "task_question": { "intentsMeeting95Precision": 0, "macroF1": 0.2963, "macroPrecision": 0.3333, "macroRecall": 0.2778, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 1.0, "falseNegative": 0, "falsePositive": 0, "precision": 1.0, "predictedPositive": 6, "recall": 1.0, "total": 6, "trueNegative": 0, "truePositive": 6, "wilsonPrecisionLower95": 0.6097 }, "replyableMessage": { "f1": 1.0, "falseNegative": 0, "falsePositive": 0, "precision": 1.0, "predictedPositive": 6, "recall": 1.0, "total": 6, "trueNegative": 0, "truePositive": 6, "wilsonPrecisionLower95": 0.6097 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.6667, "falseNegative": 3, "falsePositive": 0, "precision": 1.0, "predictedPositive": 3, "recall": 0.5, "total": 6, "trueNegative": 0, "truePositive": 3, "wilsonPrecisionLower95": 0.4385 } }, "recordCount": 6 }, "task_statement": { "intentsMeeting95Precision": 0, "macroF1": 0.2632, "macroPrecision": 0.3333, "macroRecall": 0.2222, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 8, "trueNegative": 8, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 8, "trueNegative": 8, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 8, "trueNegative": 8, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.6667, "falseNegative": 2, "falsePositive": 0, "precision": 1.0, "predictedPositive": 2, "recall": 0.5, "total": 8, "trueNegative": 4, "truePositive": 2, "wilsonPrecisionLower95": 0.3424 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 8, "trueNegative": 8, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 8, "trueNegative": 8, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.9333, "falseNegative": 1, "falsePositive": 0, "precision": 1.0, "predictedPositive": 7, "recall": 0.875, "total": 8, "trueNegative": 0, "truePositive": 7, "wilsonPrecisionLower95": 0.6457 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 8, "trueNegative": 8, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.7692, "falseNegative": 3, "falsePositive": 0, "precision": 1.0, "predictedPositive": 5, "recall": 0.625, "total": 8, "trueNegative": 0, "truePositive": 5, "wilsonPrecisionLower95": 0.5655 } }, "recordCount": 8 } }, "intentsMeeting95Precision": 3, "macroF1": 0.7711, "macroPrecision": 0.9366, "macroRecall": 0.6801, "minimumPrecision": 0.75, "perIntent": { "blessing": { "f1": 0.6667, "falseNegative": 10, "falsePositive": 0, "precision": 1.0, "predictedPositive": 10, "recall": 0.5, "total": 252, "trueNegative": 232, "truePositive": 10, "wilsonPrecisionLower95": 0.7225 }, "complaint": { "f1": 0.8571, "falseNegative": 3, "falsePositive": 0, "precision": 1.0, "predictedPositive": 9, "recall": 0.75, "total": 252, "trueNegative": 240, "truePositive": 9, "wilsonPrecisionLower95": 0.7009 }, "confirmationDecision": { "f1": 0.75, "falseNegative": 5, "falsePositive": 5, "precision": 0.75, "predictedPositive": 20, "recall": 0.75, "total": 252, "trueNegative": 227, "truePositive": 15, "wilsonPrecisionLower95": 0.5313 }, "followUpReminder": { "f1": 0.8444, "falseNegative": 7, "falsePositive": 0, "precision": 1.0, "predictedPositive": 19, "recall": 0.7308, "total": 252, "trueNegative": 226, "truePositive": 19, "wilsonPrecisionLower95": 0.8318 }, "invitation": { "f1": 0.72, "falseNegative": 7, "falsePositive": 0, "precision": 1.0, "predictedPositive": 9, "recall": 0.5625, "total": 252, "trueNegative": 236, "truePositive": 9, "wilsonPrecisionLower95": 0.7009 }, "question": { "f1": 0.9231, "falseNegative": 4, "falsePositive": 3, "precision": 0.9333, "predictedPositive": 45, "recall": 0.913, "total": 252, "trueNegative": 203, "truePositive": 42, "wilsonPrecisionLower95": 0.8214 }, "replyableMessage": { "f1": 0.9014, "falseNegative": 12, "falsePositive": 16, "precision": 0.8889, "predictedPositive": 144, "recall": 0.9143, "total": 252, "trueNegative": 96, "truePositive": 128, "wilsonPrecisionLower95": 0.8271 }, "scheduleNegotiation": { "f1": 0.7059, "falseNegative": 8, "falsePositive": 2, "precision": 0.8571, "predictedPositive": 14, "recall": 0.6, "total": 252, "trueNegative": 230, "truePositive": 12, "wilsonPrecisionLower95": 0.6006 }, "task": { "f1": 0.5714, "falseNegative": 18, "falsePositive": 0, "precision": 1.0, "predictedPositive": 12, "recall": 0.4, "total": 252, "trueNegative": 222, "truePositive": 12, "wilsonPrecisionLower95": 0.7575 } }, "recordCount": 252 }, "randomHoldout": { "byLanguage": { "en": { "intentsMeeting95Precision": 4, "macroF1": 0.6163, "macroPrecision": 0.8521, "macroRecall": 0.5281, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 340, "trueNegative": 340, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.3333, "falseNegative": 16, "falsePositive": 0, "precision": 1.0, "predictedPositive": 4, "recall": 0.2, "total": 340, "trueNegative": 320, "truePositive": 4, "wilsonPrecisionLower95": 0.5101 }, "confirmationDecision": { "f1": 0.9302, "falseNegative": 0, "falsePositive": 3, "precision": 0.8696, "predictedPositive": 23, "recall": 1.0, "total": 340, "trueNegative": 317, "truePositive": 20, "wilsonPrecisionLower95": 0.6787 }, "followUpReminder": { "f1": 0.9027, "falseNegative": 9, "falsePositive": 2, "precision": 0.9623, "predictedPositive": 53, "recall": 0.85, "total": 340, "trueNegative": 278, "truePositive": 51, "wilsonPrecisionLower95": 0.8725 }, "invitation": { "f1": 0.5714, "falseNegative": 12, "falsePositive": 0, "precision": 1.0, "predictedPositive": 8, "recall": 0.4, "total": 340, "trueNegative": 320, "truePositive": 8, "wilsonPrecisionLower95": 0.6756 }, "question": { "f1": 0.6087, "falseNegative": 45, "falsePositive": 0, "precision": 1.0, "predictedPositive": 35, "recall": 0.4375, "total": 340, "trueNegative": 260, "truePositive": 35, "wilsonPrecisionLower95": 0.9011 }, "replyableMessage": { "f1": 0.8859, "falseNegative": 37, "falsePositive": 5, "precision": 0.9702, "predictedPositive": 168, "recall": 0.815, "total": 340, "trueNegative": 135, "truePositive": 163, "wilsonPrecisionLower95": 0.9322 }, "scheduleNegotiation": { "f1": 0.7429, "falseNegative": 7, "falsePositive": 2, "precision": 0.8667, "predictedPositive": 15, "recall": 0.65, "total": 340, "trueNegative": 318, "truePositive": 13, "wilsonPrecisionLower95": 0.6212 }, "task": { "f1": 0.5714, "falseNegative": 36, "falsePositive": 0, "precision": 1.0, "predictedPositive": 24, "recall": 0.4, "total": 340, "trueNegative": 280, "truePositive": 24, "wilsonPrecisionLower95": 0.862 } }, "recordCount": 340 }, "zh-Hans": { "intentsMeeting95Precision": 4, "macroF1": 0.4934, "macroPrecision": 0.7719, "macroRecall": 0.3925, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 340, "trueNegative": 340, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.3333, "falseNegative": 16, "falsePositive": 0, "precision": 1.0, "predictedPositive": 4, "recall": 0.2, "total": 340, "trueNegative": 320, "truePositive": 4, "wilsonPrecisionLower95": 0.5101 }, "confirmationDecision": { "f1": 0.6667, "falseNegative": 10, "falsePositive": 0, "precision": 1.0, "predictedPositive": 10, "recall": 0.5, "total": 340, "trueNegative": 320, "truePositive": 10, "wilsonPrecisionLower95": 0.7225 }, "followUpReminder": { "f1": 0.7879, "falseNegative": 21, "falsePositive": 0, "precision": 1.0, "predictedPositive": 39, "recall": 0.65, "total": 340, "trueNegative": 280, "truePositive": 39, "wilsonPrecisionLower95": 0.9103 }, "invitation": { "f1": 0.6207, "falseNegative": 11, "falsePositive": 0, "precision": 1.0, "predictedPositive": 9, "recall": 0.45, "total": 340, "trueNegative": 320, "truePositive": 9, "wilsonPrecisionLower95": 0.7009 }, "question": { "f1": 0.7786, "falseNegative": 29, "falsePositive": 0, "precision": 1.0, "predictedPositive": 51, "recall": 0.6375, "total": 340, "trueNegative": 260, "truePositive": 51, "wilsonPrecisionLower95": 0.93 }, "replyableMessage": { "f1": 0.9203, "falseNegative": 21, "falsePositive": 10, "precision": 0.9471, "predictedPositive": 189, "recall": 0.895, "total": 340, "trueNegative": 130, "truePositive": 179, "wilsonPrecisionLower95": 0.9054 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 20, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 340, "trueNegative": 320, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.3333, "falseNegative": 48, "falsePositive": 0, "precision": 1.0, "predictedPositive": 12, "recall": 0.2, "total": 340, "trueNegative": 280, "truePositive": 12, "wilsonPrecisionLower95": 0.7575 } }, "recordCount": 340 } }, "bySource": { "acknowledgment_boundary": { "intentsMeeting95Precision": 0, "macroF1": 0.0, "macroPrecision": 0.0, "macroRecall": 0.0, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 40 }, "complaint_support": { "intentsMeeting95Precision": 1, "macroF1": 0.1783, "macroPrecision": 0.3333, "macroRecall": 0.1389, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.3333, "falseNegative": 32, "falsePositive": 0, "precision": 1.0, "predictedPositive": 8, "recall": 0.2, "total": 40, "trueNegative": 0, "truePositive": 8, "wilsonPrecisionLower95": 0.6756 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.3673, "falseNegative": 31, "falsePositive": 0, "precision": 1.0, "predictedPositive": 9, "recall": 0.225, "total": 40, "trueNegative": 0, "truePositive": 9, "wilsonPrecisionLower95": 0.7009 }, "replyableMessage": { "f1": 0.9041, "falseNegative": 7, "falsePositive": 0, "precision": 1.0, "predictedPositive": 33, "recall": 0.825, "total": 40, "trueNegative": 0, "truePositive": 33, "wilsonPrecisionLower95": 0.8957 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 40 }, "confirmation_decision": { "intentsMeeting95Precision": 2, "macroF1": 0.1957, "macroPrecision": 0.2222, "macroRecall": 0.175, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.8571, "falseNegative": 10, "falsePositive": 0, "precision": 1.0, "predictedPositive": 30, "recall": 0.75, "total": 40, "trueNegative": 0, "truePositive": 30, "wilsonPrecisionLower95": 0.8865 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.9041, "falseNegative": 7, "falsePositive": 0, "precision": 1.0, "predictedPositive": 33, "recall": 0.825, "total": 40, "trueNegative": 0, "truePositive": 33, "wilsonPrecisionLower95": 0.8957 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 40 }, "conversation": { "intentsMeeting95Precision": 1, "macroF1": 0.1053, "macroPrecision": 0.1111, "macroRecall": 0.1, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.9474, "falseNegative": 4, "falsePositive": 0, "precision": 1.0, "predictedPositive": 36, "recall": 0.9, "total": 40, "trueNegative": 0, "truePositive": 36, "wilsonPrecisionLower95": 0.9036 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 40 }, "event_boundary": { "intentsMeeting95Precision": 0, "macroF1": 0.0, "macroPrecision": 0.0, "macroRecall": 0.0, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.0, "falseNegative": 0, "falsePositive": 2, "precision": 0.0, "predictedPositive": 2, "recall": 0.0, "total": 40, "trueNegative": 38, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 40 }, "fixed_invitation": { "intentsMeeting95Precision": 3, "macroF1": 0.2797, "macroPrecision": 0.3333, "macroRecall": 0.2528, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.5965, "falseNegative": 23, "falsePositive": 0, "precision": 1.0, "predictedPositive": 17, "recall": 0.425, "total": 40, "trueNegative": 0, "truePositive": 17, "wilsonPrecisionLower95": 0.8157 }, "question": { "f1": 0.9333, "falseNegative": 5, "falsePositive": 0, "precision": 1.0, "predictedPositive": 35, "recall": 0.875, "total": 40, "trueNegative": 0, "truePositive": 35, "wilsonPrecisionLower95": 0.9011 }, "replyableMessage": { "f1": 0.9873, "falseNegative": 1, "falsePositive": 0, "precision": 1.0, "predictedPositive": 39, "recall": 0.975, "total": 40, "trueNegative": 0, "truePositive": 39, "wilsonPrecisionLower95": 0.9103 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 40 }, "follow_up_instruction": { "intentsMeeting95Precision": 2, "macroF1": 0.2153, "macroPrecision": 0.3333, "macroRecall": 0.1694, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.7302, "falseNegative": 17, "falsePositive": 0, "precision": 1.0, "predictedPositive": 23, "recall": 0.575, "total": 40, "trueNegative": 0, "truePositive": 23, "wilsonPrecisionLower95": 0.8569 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.8406, "falseNegative": 11, "falsePositive": 0, "precision": 1.0, "predictedPositive": 29, "recall": 0.725, "total": 40, "trueNegative": 0, "truePositive": 29, "wilsonPrecisionLower95": 0.883 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.3673, "falseNegative": 31, "falsePositive": 0, "precision": 1.0, "predictedPositive": 9, "recall": 0.225, "total": 40, "trueNegative": 0, "truePositive": 9, "wilsonPrecisionLower95": 0.7009 } }, "recordCount": 40 }, "follow_up_personal_reminder": { "intentsMeeting95Precision": 1, "macroF1": 0.097, "macroPrecision": 0.1111, "macroRecall": 0.0861, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.8732, "falseNegative": 9, "falsePositive": 0, "precision": 1.0, "predictedPositive": 31, "recall": 0.775, "total": 40, "trueNegative": 0, "truePositive": 31, "wilsonPrecisionLower95": 0.8897 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 40 }, "information_question": { "intentsMeeting95Precision": 2, "macroF1": 0.1421, "macroPrecision": 0.2222, "macroRecall": 0.1083, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 3, "precision": 0.0, "predictedPositive": 3, "recall": 0.0, "total": 40, "trueNegative": 37, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 2, "precision": 0.0, "predictedPositive": 2, "recall": 0.0, "total": 40, "trueNegative": 38, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.4906, "falseNegative": 27, "falsePositive": 0, "precision": 1.0, "predictedPositive": 13, "recall": 0.325, "total": 40, "trueNegative": 0, "truePositive": 13, "wilsonPrecisionLower95": 0.7719 }, "replyableMessage": { "f1": 0.7879, "falseNegative": 14, "falsePositive": 0, "precision": 1.0, "predictedPositive": 26, "recall": 0.65, "total": 40, "trueNegative": 0, "truePositive": 26, "wilsonPrecisionLower95": 0.8713 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 40 }, "negative_news": { "intentsMeeting95Precision": 0, "macroF1": 0.0, "macroPrecision": 0.0, "macroRecall": 0.0, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.0, "falseNegative": 0, "falsePositive": 3, "precision": 0.0, "predictedPositive": 3, "recall": 0.0, "total": 40, "trueNegative": 37, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 2, "precision": 0.0, "predictedPositive": 2, "recall": 0.0, "total": 40, "trueNegative": 38, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 40 }, "neutral_fact": { "intentsMeeting95Precision": 0, "macroF1": 0.0, "macroPrecision": 0.0, "macroRecall": 0.0, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.0, "falseNegative": 0, "falsePositive": 5, "precision": 0.0, "predictedPositive": 5, "recall": 0.0, "total": 40, "trueNegative": 35, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 40 }, "positive_feedback": { "intentsMeeting95Precision": 1, "macroF1": 0.1068, "macroPrecision": 0.1111, "macroRecall": 0.1028, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.961, "falseNegative": 3, "falsePositive": 0, "precision": 1.0, "predictedPositive": 37, "recall": 0.925, "total": 40, "trueNegative": 0, "truePositive": 37, "wilsonPrecisionLower95": 0.9059 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 40 }, "quoted_question_boundary": { "intentsMeeting95Precision": 0, "macroF1": 0.0, "macroPrecision": 0.0, "macroRecall": 0.0, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.0, "falseNegative": 0, "falsePositive": 1, "precision": 0.0, "predictedPositive": 1, "recall": 0.0, "total": 40, "trueNegative": 39, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 40 }, "schedule_negotiation": { "intentsMeeting95Precision": 3, "macroF1": 0.259, "macroPrecision": 0.3333, "macroRecall": 0.2278, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.8406, "falseNegative": 11, "falsePositive": 0, "precision": 1.0, "predictedPositive": 29, "recall": 0.725, "total": 40, "trueNegative": 0, "truePositive": 29, "wilsonPrecisionLower95": 0.883 }, "replyableMessage": { "f1": 1.0, "falseNegative": 0, "falsePositive": 0, "precision": 1.0, "predictedPositive": 40, "recall": 1.0, "total": 40, "trueNegative": 0, "truePositive": 40, "wilsonPrecisionLower95": 0.9124 }, "scheduleNegotiation": { "f1": 0.4906, "falseNegative": 27, "falsePositive": 0, "precision": 1.0, "predictedPositive": 13, "recall": 0.325, "total": 40, "trueNegative": 0, "truePositive": 13, "wilsonPrecisionLower95": 0.7719 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 40 }, "task_next_action": { "intentsMeeting95Precision": 3, "macroF1": 0.257, "macroPrecision": 0.3333, "macroRecall": 0.225, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.9474, "falseNegative": 4, "falsePositive": 0, "precision": 1.0, "predictedPositive": 36, "recall": 0.9, "total": 40, "trueNegative": 0, "truePositive": 36, "wilsonPrecisionLower95": 0.9036 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.9041, "falseNegative": 7, "falsePositive": 0, "precision": 1.0, "predictedPositive": 33, "recall": 0.825, "total": 40, "trueNegative": 0, "truePositive": 33, "wilsonPrecisionLower95": 0.8957 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.4615, "falseNegative": 28, "falsePositive": 0, "precision": 1.0, "predictedPositive": 12, "recall": 0.3, "total": 40, "trueNegative": 0, "truePositive": 12, "wilsonPrecisionLower95": 0.7575 } }, "recordCount": 40 }, "task_request": { "intentsMeeting95Precision": 2, "macroF1": 0.1659, "macroPrecision": 0.2222, "macroRecall": 0.1417, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.9474, "falseNegative": 4, "falsePositive": 0, "precision": 1.0, "predictedPositive": 36, "recall": 0.9, "total": 40, "trueNegative": 0, "truePositive": 36, "wilsonPrecisionLower95": 0.9036 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.5455, "falseNegative": 25, "falsePositive": 0, "precision": 1.0, "predictedPositive": 15, "recall": 0.375, "total": 40, "trueNegative": 0, "truePositive": 15, "wilsonPrecisionLower95": 0.7961 } }, "recordCount": 40 }, "vague_future_boundary": { "intentsMeeting95Precision": 0, "macroF1": 0.0, "macroPrecision": 0.0, "macroRecall": 0.0, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.0, "falseNegative": 0, "falsePositive": 4, "precision": 0.0, "predictedPositive": 4, "recall": 0.0, "total": 40, "trueNegative": 36, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 40 } }, "intentsMeeting95Precision": 5, "macroF1": 0.5709, "macroPrecision": 0.8569, "macroRecall": 0.4603, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 680, "trueNegative": 680, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.3333, "falseNegative": 32, "falsePositive": 0, "precision": 1.0, "predictedPositive": 8, "recall": 0.2, "total": 680, "trueNegative": 640, "truePositive": 8, "wilsonPrecisionLower95": 0.6756 }, "confirmationDecision": { "f1": 0.8219, "falseNegative": 10, "falsePositive": 3, "precision": 0.9091, "predictedPositive": 33, "recall": 0.75, "total": 680, "trueNegative": 637, "truePositive": 30, "wilsonPrecisionLower95": 0.7643 }, "followUpReminder": { "f1": 0.8491, "falseNegative": 30, "falsePositive": 2, "precision": 0.9783, "predictedPositive": 92, "recall": 0.75, "total": 680, "trueNegative": 558, "truePositive": 90, "wilsonPrecisionLower95": 0.9242 }, "invitation": { "f1": 0.5965, "falseNegative": 23, "falsePositive": 0, "precision": 1.0, "predictedPositive": 17, "recall": 0.425, "total": 680, "trueNegative": 640, "truePositive": 17, "wilsonPrecisionLower95": 0.8157 }, "question": { "f1": 0.6992, "falseNegative": 74, "falsePositive": 0, "precision": 1.0, "predictedPositive": 86, "recall": 0.5375, "total": 680, "trueNegative": 520, "truePositive": 86, "wilsonPrecisionLower95": 0.9572 }, "replyableMessage": { "f1": 0.9036, "falseNegative": 58, "falsePositive": 15, "precision": 0.958, "predictedPositive": 357, "recall": 0.855, "total": 680, "trueNegative": 265, "truePositive": 342, "wilsonPrecisionLower95": 0.9318 }, "scheduleNegotiation": { "f1": 0.4727, "falseNegative": 27, "falsePositive": 2, "precision": 0.8667, "predictedPositive": 15, "recall": 0.325, "total": 680, "trueNegative": 638, "truePositive": 13, "wilsonPrecisionLower95": 0.6212 }, "task": { "f1": 0.4615, "falseNegative": 84, "falsePositive": 0, "precision": 1.0, "predictedPositive": 36, "recall": 0.3, "total": 680, "trueNegative": 560, "truePositive": 36, "wilsonPrecisionLower95": 0.9036 } }, "recordCount": 680 }, "researchOnlyComprehensive": { "byLanguage": { "en": { "intentsMeeting95Precision": 0, "macroF1": 0.2616, "macroPrecision": 0.3966, "macroRecall": 0.2659, "minimumPrecision": 0.0012, "perIntent": { "blessing": { "f1": 0.2308, "falseNegative": 60, "falsePositive": 0, "precision": 1.0, "predictedPositive": 9, "recall": 0.1304, "total": 13176, "trueNegative": 13107, "truePositive": 9, "wilsonPrecisionLower95": 0.7009 }, "complaint": { "f1": 0.048, "falseNegative": 421, "falsePositive": 95, "precision": 0.1204, "predictedPositive": 108, "recall": 0.03, "total": 13176, "trueNegative": 12647, "truePositive": 13, "wilsonPrecisionLower95": 0.0717 }, "confirmationDecision": { "f1": 0.0024, "falseNegative": 10, "falsePositive": 1668, "precision": 0.0012, "predictedPositive": 1670, "recall": 0.1667, "total": 13176, "trueNegative": 11496, "truePositive": 2, "wilsonPrecisionLower95": 0.0003 }, "followUpReminder": { "f1": 0.1134, "falseNegative": 114, "falsePositive": 58, "precision": 0.1594, "predictedPositive": 69, "recall": 0.088, "total": 13176, "trueNegative": 12993, "truePositive": 11, "wilsonPrecisionLower95": 0.0914 }, "invitation": { "f1": 0.336, "falseNegative": 40, "falsePositive": 43, "precision": 0.3281, "predictedPositive": 64, "recall": 0.3443, "total": 13176, "trueNegative": 13072, "truePositive": 21, "wilsonPrecisionLower95": 0.2257 }, "question": { "f1": 0.4288, "falseNegative": 1828, "falsePositive": 378, "precision": 0.6866, "predictedPositive": 1206, "recall": 0.3117, "total": 13176, "trueNegative": 10142, "truePositive": 828, "wilsonPrecisionLower95": 0.6598 }, "replyableMessage": { "f1": 0.7642, "falseNegative": 2967, "falsePositive": 1917, "precision": 0.805, "predictedPositive": 9831, "recall": 0.7273, "total": 13176, "trueNegative": 378, "truePositive": 7914, "wilsonPrecisionLower95": 0.7971 }, "scheduleNegotiation": { "f1": 0.1683, "falseNegative": 41, "falsePositive": 216, "precision": 0.1074, "predictedPositive": 242, "recall": 0.3881, "total": 13176, "trueNegative": 12893, "truePositive": 26, "wilsonPrecisionLower95": 0.0744 }, "task": { "f1": 0.2628, "falseNegative": 888, "falsePositive": 408, "precision": 0.3615, "predictedPositive": 639, "recall": 0.2064, "total": 13176, "trueNegative": 11649, "truePositive": 231, "wilsonPrecisionLower95": 0.3252 } }, "recordCount": 13176 }, "zh-Hans": { "intentsMeeting95Precision": 0, "macroF1": 0.1774, "macroPrecision": 0.3011, "macroRecall": 0.149, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 17, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 28019, "trueNegative": 28002, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.4077, "falseNegative": 154, "falsePositive": 183, "precision": 0.388, "predictedPositive": 299, "recall": 0.4296, "total": 28019, "trueNegative": 27566, "truePositive": 116, "wilsonPrecisionLower95": 0.3345 }, "confirmationDecision": { "f1": 0.0909, "falseNegative": 53, "falsePositive": 27, "precision": 0.129, "predictedPositive": 31, "recall": 0.0702, "total": 28019, "trueNegative": 27935, "truePositive": 4, "wilsonPrecisionLower95": 0.0513 }, "followUpReminder": { "f1": 0.1856, "falseNegative": 234, "falsePositive": 38, "precision": 0.4493, "predictedPositive": 69, "recall": 0.117, "total": 28019, "trueNegative": 27716, "truePositive": 31, "wilsonPrecisionLower95": 0.3377 }, "invitation": { "f1": 0.0, "falseNegative": 73, "falsePositive": 9, "precision": 0.0, "predictedPositive": 9, "recall": 0.0, "total": 28019, "trueNegative": 27937, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "question": { "f1": 0.1419, "falseNegative": 6619, "falsePositive": 216, "precision": 0.7234, "predictedPositive": 781, "recall": 0.0786, "total": 28019, "trueNegative": 20619, "truePositive": 565, "wilsonPrecisionLower95": 0.691 }, "replyableMessage": { "f1": 0.7622, "falseNegative": 9480, "falsePositive": 1096, "precision": 0.9393, "predictedPositive": 18042, "recall": 0.6413, "total": 28019, "trueNegative": 497, "truePositive": 16946, "wilsonPrecisionLower95": 0.9357 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 4, "falsePositive": 2, "precision": 0.0, "predictedPositive": 2, "recall": 0.0, "total": 28019, "trueNegative": 28013, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "task": { "f1": 0.0086, "falseNegative": 656, "falsePositive": 34, "precision": 0.0811, "predictedPositive": 37, "recall": 0.0046, "total": 28019, "trueNegative": 27326, "truePositive": 3, "wilsonPrecisionLower95": 0.028 } }, "recordCount": 28019 } }, "bySource": { "AmazonScience/MASSIVE en-US test": { "intentsMeeting95Precision": 2, "macroF1": 0.1614, "macroPrecision": 0.2661, "macroRecall": 0.1233, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 2967, "trueNegative": 2967, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 17, "precision": 0.0, "predictedPositive": 17, "recall": 0.0, "total": 2967, "trueNegative": 2950, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 347, "precision": 0.0, "predictedPositive": 347, "recall": 0.0, "total": 2967, "trueNegative": 2620, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "followUpReminder": { "f1": 0.1447, "falseNegative": 86, "falsePositive": 44, "precision": 0.2, "predictedPositive": 55, "recall": 0.1134, "total": 2967, "trueNegative": 2826, "truePositive": 11, "wilsonPrecisionLower95": 0.1155 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 7, "precision": 0.0, "predictedPositive": 7, "recall": 0.0, "total": 2967, "trueNegative": 2960, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "question": { "f1": 0.3532, "falseNegative": 1266, "falsePositive": 16, "precision": 0.9563, "predictedPositive": 366, "recall": 0.2166, "total": 2967, "trueNegative": 1335, "truePositive": 350, "wilsonPrecisionLower95": 0.9302 }, "replyableMessage": { "f1": 0.7951, "falseNegative": 1009, "falsePositive": 0, "precision": 1.0, "predictedPositive": 1958, "recall": 0.6599, "total": 2967, "trueNegative": 0, "truePositive": 1958, "wilsonPrecisionLower95": 0.998 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 101, "precision": 0.0, "predictedPositive": 101, "recall": 0.0, "total": 2967, "trueNegative": 2866, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "task": { "f1": 0.1599, "falseNegative": 476, "falsePositive": 207, "precision": 0.239, "predictedPositive": 272, "recall": 0.1201, "total": 2967, "trueNegative": 2219, "truePositive": 65, "wilsonPrecisionLower95": 0.1921 } }, "recordCount": 2967 }, "AmazonScience/MASSIVE zh-CN test": { "intentsMeeting95Precision": 1, "macroF1": 0.1368, "macroPrecision": 0.2943, "macroRecall": 0.0998, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 2850, "trueNegative": 2850, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 2, "precision": 0.0, "predictedPositive": 2, "recall": 0.0, "total": 2850, "trueNegative": 2848, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 4, "precision": 0.0, "predictedPositive": 4, "recall": 0.0, "total": 2850, "trueNegative": 2846, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "followUpReminder": { "f1": 0.2537, "falseNegative": 140, "falsePositive": 13, "precision": 0.6667, "predictedPositive": 39, "recall": 0.1566, "total": 2850, "trueNegative": 2671, "truePositive": 26, "wilsonPrecisionLower95": 0.5098 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 7, "precision": 0.0, "predictedPositive": 7, "recall": 0.0, "total": 2850, "trueNegative": 2843, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "question": { "f1": 0.2054, "falseNegative": 1381, "falsePositive": 19, "precision": 0.905, "predictedPositive": 200, "recall": 0.1159, "total": 2850, "trueNegative": 1269, "truePositive": 181, "wilsonPrecisionLower95": 0.8564 }, "replyableMessage": { "f1": 0.7684, "falseNegative": 1072, "falsePositive": 0, "precision": 1.0, "predictedPositive": 1778, "recall": 0.6239, "total": 2850, "trueNegative": 0, "truePositive": 1778, "wilsonPrecisionLower95": 0.9978 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 2, "precision": 0.0, "predictedPositive": 2, "recall": 0.0, "total": 2850, "trueNegative": 2848, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "task": { "f1": 0.0041, "falseNegative": 477, "falsePositive": 12, "precision": 0.0769, "predictedPositive": 13, "recall": 0.0021, "total": 2850, "trueNegative": 2360, "truePositive": 1, "wilsonPrecisionLower95": 0.0137 } }, "recordCount": 2850 }, "CFPB via hpe-ai/customer-complaints": { "intentsMeeting95Precision": 1, "macroF1": 0.0837, "macroPrecision": 0.2593, "macroRecall": 0.063, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 250, "trueNegative": 250, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0159, "falseNegative": 248, "falsePositive": 0, "precision": 1.0, "predictedPositive": 2, "recall": 0.008, "total": 250, "trueNegative": 0, "truePositive": 2, "wilsonPrecisionLower95": 0.3424 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 7, "precision": 0.0, "predictedPositive": 7, "recall": 0.0, "total": 250, "trueNegative": 243, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 250, "trueNegative": 250, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 250, "trueNegative": 250, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 17, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 250, "trueNegative": 233, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.7013, "falseNegative": 115, "falsePositive": 0, "precision": 1.0, "predictedPositive": 135, "recall": 0.54, "total": 250, "trueNegative": 0, "truePositive": 135, "wilsonPrecisionLower95": 0.9723 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 250, "trueNegative": 250, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0357, "falseNegative": 52, "falsePositive": 2, "precision": 0.3333, "predictedPositive": 3, "recall": 0.0189, "total": 250, "trueNegative": 195, "truePositive": 1, "wilsonPrecisionLower95": 0.0615 } }, "recordCount": 250 }, "CPED test split": { "intentsMeeting95Precision": 0, "macroF1": 0.1219, "macroPrecision": 0.2231, "macroRecall": 0.096, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 17, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 25049, "trueNegative": 25032, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.018, "falseNegative": 147, "falsePositive": 181, "precision": 0.0163, "predictedPositive": 184, "recall": 0.02, "total": 25049, "trueNegative": 24718, "truePositive": 3, "wilsonPrecisionLower95": 0.0056 }, "confirmationDecision": { "f1": 0.0952, "falseNegative": 53, "falsePositive": 23, "precision": 0.1481, "predictedPositive": 27, "recall": 0.0702, "total": 25049, "trueNegative": 24969, "truePositive": 4, "wilsonPrecisionLower95": 0.0592 }, "followUpReminder": { "f1": 0.0775, "falseNegative": 94, "falsePositive": 25, "precision": 0.1667, "predictedPositive": 30, "recall": 0.0505, "total": 25049, "trueNegative": 24925, "truePositive": 5, "wilsonPrecisionLower95": 0.0734 }, "invitation": { "f1": 0.0, "falseNegative": 73, "falsePositive": 2, "precision": 0.0, "predictedPositive": 2, "recall": 0.0, "total": 25049, "trueNegative": 24974, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "question": { "f1": 0.1245, "falseNegative": 5202, "falsePositive": 197, "precision": 0.6609, "predictedPositive": 581, "recall": 0.0687, "total": 25049, "trueNegative": 19266, "truePositive": 384, "wilsonPrecisionLower95": 0.6215 }, "replyableMessage": { "f1": 0.7609, "falseNegative": 8378, "falsePositive": 1096, "precision": 0.9322, "predictedPositive": 16174, "recall": 0.6428, "total": 25049, "trueNegative": 497, "truePositive": 15078, "wilsonPrecisionLower95": 0.9283 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 4, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 25049, "trueNegative": 25045, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0208, "falseNegative": 166, "falsePositive": 22, "precision": 0.0833, "predictedPositive": 24, "recall": 0.0119, "total": 25049, "trueNegative": 24859, "truePositive": 2, "wilsonPrecisionLower95": 0.0232 } }, "recordCount": 25049 }, "Google Research GoEmotions test": { "intentsMeeting95Precision": 1, "macroF1": 0.1942, "macroPrecision": 0.34, "macroRecall": 0.1531, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.1918, "falseNegative": 59, "falsePositive": 0, "precision": 1.0, "predictedPositive": 7, "recall": 0.1061, "total": 5418, "trueNegative": 5352, "truePositive": 7, "wilsonPrecisionLower95": 0.6457 }, "complaint": { "f1": 0.0, "falseNegative": 20, "falsePositive": 27, "precision": 0.0, "predictedPositive": 27, "recall": 0.0, "total": 5418, "trueNegative": 5371, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 881, "precision": 0.0, "predictedPositive": 881, "recall": 0.0, "total": 5418, "trueNegative": 4537, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 6, "precision": 0.0, "predictedPositive": 6, "recall": 0.0, "total": 5418, "trueNegative": 5412, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 8, "precision": 0.0, "predictedPositive": 8, "recall": 0.0, "total": 5418, "trueNegative": 5410, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "question": { "f1": 0.3991, "falseNegative": 363, "falsePositive": 146, "precision": 0.5365, "predictedPositive": 315, "recall": 0.3177, "total": 5418, "trueNegative": 4740, "truePositive": 169, "wilsonPrecisionLower95": 0.4813 }, "replyableMessage": { "f1": 0.8467, "falseNegative": 1440, "falsePositive": 0, "precision": 1.0, "predictedPositive": 3978, "recall": 0.7342, "total": 5418, "trueNegative": 0, "truePositive": 3978, "wilsonPrecisionLower95": 0.999 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 33, "precision": 0.0, "predictedPositive": 33, "recall": 0.0, "total": 5418, "trueNegative": 5385, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "task": { "f1": 0.3099, "falseNegative": 39, "falsePositive": 10, "precision": 0.5238, "predictedPositive": 21, "recall": 0.22, "total": 5418, "trueNegative": 5358, "truePositive": 11, "wilsonPrecisionLower95": 0.3237 } }, "recordCount": 5418 }, "Meituan-Dianping/ASAP test": { "intentsMeeting95Precision": 2, "macroF1": 0.203, "macroPrecision": 0.2222, "macroRecall": 0.188, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 120, "trueNegative": 120, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.97, "falseNegative": 7, "falsePositive": 0, "precision": 1.0, "predictedPositive": 113, "recall": 0.9417, "total": 120, "trueNegative": 0, "truePositive": 113, "wilsonPrecisionLower95": 0.9671 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 120, "trueNegative": 120, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 120, "trueNegative": 120, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 120, "trueNegative": 120, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 36, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 120, "trueNegative": 84, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.8571, "falseNegative": 30, "falsePositive": 0, "precision": 1.0, "predictedPositive": 90, "recall": 0.75, "total": 120, "trueNegative": 0, "truePositive": 90, "wilsonPrecisionLower95": 0.9591 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 120, "trueNegative": 120, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 13, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 120, "trueNegative": 107, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 120 }, "Sulak2020 GitHub issues multi-repo dataset": { "intentsMeeting95Precision": 1, "macroF1": 0.28, "macroPrecision": 0.415, "macroRecall": 0.216, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.6667, "falseNegative": 1, "falsePositive": 0, "precision": 1.0, "predictedPositive": 1, "recall": 0.5, "total": 1004, "trueNegative": 1002, "truePositive": 1, "wilsonPrecisionLower95": 0.2065 }, "complaint": { "f1": 0.0995, "falseNegative": 153, "falsePositive": 46, "precision": 0.193, "predictedPositive": 57, "recall": 0.0671, "total": 1004, "trueNegative": 794, "truePositive": 11, "wilsonPrecisionLower95": 0.1113 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 2, "falsePositive": 61, "precision": 0.0, "predictedPositive": 61, "recall": 0.0, "total": 1004, "trueNegative": 941, "truePositive": 0, "wilsonPrecisionLower95": -0.0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 2, "falsePositive": 1, "precision": 0.0, "predictedPositive": 1, "recall": 0.0, "total": 1004, "trueNegative": 1001, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 2, "precision": 0.0, "predictedPositive": 2, "recall": 0.0, "total": 1004, "trueNegative": 1002, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "question": { "f1": 0.4474, "falseNegative": 39, "falsePositive": 3, "precision": 0.85, "predictedPositive": 20, "recall": 0.3036, "total": 1004, "trueNegative": 945, "truePositive": 17, "wilsonPrecisionLower95": 0.6396 }, "replyableMessage": { "f1": 0.8326, "falseNegative": 288, "falsePositive": 0, "precision": 1.0, "predictedPositive": 716, "recall": 0.7131, "total": 1004, "trueNegative": 0, "truePositive": 716, "wilsonPrecisionLower95": 0.9947 }, "scheduleNegotiation": { "f1": 0.4737, "falseNegative": 16, "falsePositive": 4, "precision": 0.6923, "predictedPositive": 13, "recall": 0.36, "total": 1004, "trueNegative": 975, "truePositive": 9, "wilsonPrecisionLower95": 0.4237 }, "task": { "f1": 0.0, "falseNegative": 90, "falsePositive": 3, "precision": 0.0, "predictedPositive": 3, "recall": 0.0, "total": 1004, "trueNegative": 911, "truePositive": 0, "wilsonPrecisionLower95": 0.0 } }, "recordCount": 1004 }, "Verified Enron Intent Dataset": { "intentsMeeting95Precision": 0, "macroF1": 0.3577, "macroPrecision": 0.3369, "macroRecall": 0.4364, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 1.0, "falseNegative": 0, "falsePositive": 0, "precision": 1.0, "predictedPositive": 1, "recall": 1.0, "total": 3537, "trueNegative": 3536, "truePositive": 1, "wilsonPrecisionLower95": 0.2065 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 5, "precision": 0.0, "predictedPositive": 5, "recall": 0.0, "total": 3537, "trueNegative": 3532, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "confirmationDecision": { "f1": 0.0104, "falseNegative": 8, "falsePositive": 372, "precision": 0.0053, "predictedPositive": 374, "recall": 0.2, "total": 3537, "trueNegative": 3155, "truePositive": 2, "wilsonPrecisionLower95": 0.0015 }, "followUpReminder": { "f1": 0.0, "falseNegative": 26, "falsePositive": 7, "precision": 0.0, "predictedPositive": 7, "recall": 0.0, "total": 3537, "trueNegative": 3504, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "invitation": { "f1": 0.3889, "falseNegative": 40, "falsePositive": 26, "precision": 0.4468, "predictedPositive": 47, "recall": 0.3443, "total": 3537, "trueNegative": 3450, "truePositive": 21, "wilsonPrecisionLower95": 0.3141 }, "question": { "f1": 0.6213, "falseNegative": 143, "falsePositive": 213, "precision": 0.5782, "predictedPositive": 505, "recall": 0.6713, "total": 3537, "trueNegative": 2889, "truePositive": 292, "wilsonPrecisionLower95": 0.5347 }, "replyableMessage": { "f1": 0.5259, "falseNegative": 115, "falsePositive": 1917, "precision": 0.3702, "predictedPositive": 3044, "recall": 0.9074, "total": 3537, "trueNegative": 378, "truePositive": 1127, "wilsonPrecisionLower95": 0.3533 }, "scheduleNegotiation": { "f1": 0.2482, "falseNegative": 25, "falsePositive": 78, "precision": 0.1789, "predictedPositive": 95, "recall": 0.4048, "total": 3537, "trueNegative": 3417, "truePositive": 17, "wilsonPrecisionLower95": 0.1148 }, "task": { "f1": 0.4248, "falseNegative": 231, "falsePositive": 186, "precision": 0.4529, "predictedPositive": 340, "recall": 0.4, "total": 3537, "trueNegative": 2966, "truePositive": 154, "wilsonPrecisionLower95": 0.4008 } }, "recordCount": 3537 } }, "intentsMeeting95Precision": 0, "macroF1": 0.2383, "macroPrecision": 0.4398, "macroRecall": 0.2161, "minimumPrecision": 0.0035, "perIntent": { "blessing": { "f1": 0.1895, "falseNegative": 77, "falsePositive": 0, "precision": 1.0, "predictedPositive": 9, "recall": 0.1047, "total": 41195, "trueNegative": 41109, "truePositive": 9, "wilsonPrecisionLower95": 0.7009 }, "complaint": { "f1": 0.2322, "falseNegative": 575, "falsePositive": 278, "precision": 0.317, "predictedPositive": 407, "recall": 0.1832, "total": 41195, "trueNegative": 40213, "truePositive": 129, "wilsonPrecisionLower95": 0.2736 }, "confirmationDecision": { "f1": 0.0068, "falseNegative": 63, "falsePositive": 1695, "precision": 0.0035, "predictedPositive": 1701, "recall": 0.087, "total": 41195, "trueNegative": 39431, "truePositive": 6, "wilsonPrecisionLower95": 0.0016 }, "followUpReminder": { "f1": 0.1591, "falseNegative": 348, "falsePositive": 96, "precision": 0.3043, "predictedPositive": 138, "recall": 0.1077, "total": 41195, "trueNegative": 40709, "truePositive": 42, "wilsonPrecisionLower95": 0.2337 }, "invitation": { "f1": 0.2029, "falseNegative": 113, "falsePositive": 52, "precision": 0.2877, "predictedPositive": 73, "recall": 0.1567, "total": 41195, "trueNegative": 41009, "truePositive": 21, "wilsonPrecisionLower95": 0.1965 }, "question": { "f1": 0.2356, "falseNegative": 8447, "falsePositive": 594, "precision": 0.7011, "predictedPositive": 1987, "recall": 0.1416, "total": 41195, "trueNegative": 30761, "truePositive": 1393, "wilsonPrecisionLower95": 0.6806 }, "replyableMessage": { "f1": 0.7628, "falseNegative": 12447, "falsePositive": 3013, "precision": 0.8919, "predictedPositive": 27873, "recall": 0.6664, "total": 41195, "trueNegative": 875, "truePositive": 24860, "wilsonPrecisionLower95": 0.8882 }, "scheduleNegotiation": { "f1": 0.1651, "falseNegative": 45, "falsePositive": 218, "precision": 0.1066, "predictedPositive": 244, "recall": 0.3662, "total": 41195, "trueNegative": 40906, "truePositive": 26, "wilsonPrecisionLower95": 0.0738 }, "task": { "f1": 0.1907, "falseNegative": 1544, "falsePositive": 442, "precision": 0.3462, "predictedPositive": 676, "recall": 0.1316, "total": 41195, "trueNegative": 38975, "truePositive": 234, "wilsonPrecisionLower95": 0.3113 } }, "recordCount": 41195 }, "targetedRelease": { "byLanguage": { "en": { "intentsMeeting95Precision": 4, "macroF1": 0.4138, "macroPrecision": 0.5476, "macroRecall": 0.3512, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 320, "trueNegative": 320, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.6957, "falseNegative": 56, "falsePositive": 0, "precision": 1.0, "predictedPositive": 64, "recall": 0.5333, "total": 320, "trueNegative": 200, "truePositive": 64, "wilsonPrecisionLower95": 0.9434 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 4, "precision": 0.0, "predictedPositive": 4, "recall": 0.0, "total": 320, "trueNegative": 316, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "followUpReminder": { "f1": 0.961, "falseNegative": 3, "falsePositive": 0, "precision": 1.0, "predictedPositive": 37, "recall": 0.925, "total": 320, "trueNegative": 280, "truePositive": 37, "wilsonPrecisionLower95": 0.9059 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 320, "trueNegative": 320, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.4762, "falseNegative": 55, "falsePositive": 0, "precision": 1.0, "predictedPositive": 25, "recall": 0.3125, "total": 320, "trueNegative": 240, "truePositive": 25, "wilsonPrecisionLower95": 0.8668 }, "replyableMessage": { "f1": 0.8819, "falseNegative": 32, "falsePositive": 13, "precision": 0.9282, "predictedPositive": 181, "recall": 0.84, "total": 320, "trueNegative": 107, "truePositive": 168, "wilsonPrecisionLower95": 0.881 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 320, "trueNegative": 320, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.7097, "falseNegative": 36, "falsePositive": 0, "precision": 1.0, "predictedPositive": 44, "recall": 0.55, "total": 320, "trueNegative": 240, "truePositive": 44, "wilsonPrecisionLower95": 0.9197 } }, "recordCount": 320 }, "zh-Hans": { "intentsMeeting95Precision": 4, "macroF1": 0.3345, "macroPrecision": 0.5501, "macroRecall": 0.2766, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 320, "trueNegative": 320, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.4516, "falseNegative": 85, "falsePositive": 0, "precision": 1.0, "predictedPositive": 35, "recall": 0.2917, "total": 320, "trueNegative": 200, "truePositive": 35, "wilsonPrecisionLower95": 0.9011 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 320, "trueNegative": 320, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.2609, "falseNegative": 34, "falsePositive": 0, "precision": 1.0, "predictedPositive": 6, "recall": 0.15, "total": 320, "trueNegative": 280, "truePositive": 6, "wilsonPrecisionLower95": 0.6097 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 320, "trueNegative": 320, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.8811, "falseNegative": 17, "falsePositive": 0, "precision": 1.0, "predictedPositive": 63, "recall": 0.7875, "total": 320, "trueNegative": 240, "truePositive": 63, "wilsonPrecisionLower95": 0.9425 }, "replyableMessage": { "f1": 0.9552, "falseNegative": 8, "falsePositive": 10, "precision": 0.9505, "predictedPositive": 202, "recall": 0.96, "total": 320, "trueNegative": 110, "truePositive": 192, "wilsonPrecisionLower95": 0.9113 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 320, "trueNegative": 320, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.4615, "falseNegative": 56, "falsePositive": 0, "precision": 1.0, "predictedPositive": 24, "recall": 0.3, "total": 320, "trueNegative": 240, "truePositive": 24, "wilsonPrecisionLower95": 0.862 } }, "recordCount": 320 } }, "bySource": { "release_complaint_brief": { "intentsMeeting95Precision": 2, "macroF1": 0.1893, "macroPrecision": 0.2222, "macroRecall": 0.1708, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.7097, "falseNegative": 36, "falsePositive": 0, "precision": 1.0, "predictedPositive": 44, "recall": 0.55, "total": 80, "trueNegative": 0, "truePositive": 44, "wilsonPrecisionLower95": 0.9197 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.9937, "falseNegative": 1, "falsePositive": 0, "precision": 1.0, "predictedPositive": 79, "recall": 0.9875, "total": 80, "trueNegative": 0, "truePositive": 79, "wilsonPrecisionLower95": 0.9536 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 80 }, "release_complaint_question": { "intentsMeeting95Precision": 3, "macroF1": 0.2152, "macroPrecision": 0.3333, "macroRecall": 0.1667, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.5586, "falseNegative": 49, "falsePositive": 0, "precision": 1.0, "predictedPositive": 31, "recall": 0.3875, "total": 80, "trueNegative": 0, "truePositive": 31, "wilsonPrecisionLower95": 0.8897 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.5047, "falseNegative": 53, "falsePositive": 0, "precision": 1.0, "predictedPositive": 27, "recall": 0.3375, "total": 80, "trueNegative": 0, "truePositive": 27, "wilsonPrecisionLower95": 0.8754 }, "replyableMessage": { "f1": 0.8732, "falseNegative": 18, "falsePositive": 0, "precision": 1.0, "predictedPositive": 62, "recall": 0.775, "total": 80, "trueNegative": 0, "truePositive": 62, "wilsonPrecisionLower95": 0.9417 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 80 }, "release_complaint_unspoken": { "intentsMeeting95Precision": 2, "macroF1": 0.1565, "macroPrecision": 0.2222, "macroRecall": 0.1333, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.4615, "falseNegative": 56, "falsePositive": 0, "precision": 1.0, "predictedPositive": 24, "recall": 0.3, "total": 80, "trueNegative": 0, "truePositive": 24, "wilsonPrecisionLower95": 0.862 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.9474, "falseNegative": 8, "falsePositive": 0, "precision": 1.0, "predictedPositive": 72, "recall": 0.9, "total": 80, "trueNegative": 0, "truePositive": 72, "wilsonPrecisionLower95": 0.9493 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 80 }, "release_negative_boundary": { "intentsMeeting95Precision": 0, "macroF1": 0.0, "macroPrecision": 0.0, "macroRecall": 0.0, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 80 }, "release_personal_boundary": { "intentsMeeting95Precision": 1, "macroF1": 0.0777, "macroPrecision": 0.1111, "macroRecall": 0.0597, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.6992, "falseNegative": 37, "falsePositive": 0, "precision": 1.0, "predictedPositive": 43, "recall": 0.5375, "total": 80, "trueNegative": 0, "truePositive": 43, "wilsonPrecisionLower95": 0.918 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.0, "falseNegative": 0, "falsePositive": 4, "precision": 0.0, "predictedPositive": 4, "recall": 0.0, "total": 80, "trueNegative": 76, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 80 }, "release_resolved_boundary": { "intentsMeeting95Precision": 0, "macroF1": 0.0, "macroPrecision": 0.0, "macroRecall": 0.0, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.0, "falseNegative": 0, "falsePositive": 19, "precision": 0.0, "predictedPositive": 19, "recall": 0.0, "total": 80, "trueNegative": 61, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 80 }, "release_task_owner": { "intentsMeeting95Precision": 2, "macroF1": 0.171, "macroPrecision": 0.2222, "macroRecall": 0.1514, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 4, "precision": 0.0, "predictedPositive": 4, "recall": 0.0, "total": 80, "trueNegative": 76, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.9937, "falseNegative": 1, "falsePositive": 0, "precision": 1.0, "predictedPositive": 79, "recall": 0.9875, "total": 80, "trueNegative": 0, "truePositive": 79, "wilsonPrecisionLower95": 0.9536 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.5455, "falseNegative": 50, "falsePositive": 0, "precision": 1.0, "predictedPositive": 30, "recall": 0.375, "total": 80, "trueNegative": 0, "truePositive": 30, "wilsonPrecisionLower95": 0.8865 } }, "recordCount": 80 }, "release_task_polite_request": { "intentsMeeting95Precision": 3, "macroF1": 0.2698, "macroPrecision": 0.3333, "macroRecall": 0.2319, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.8652, "falseNegative": 19, "falsePositive": 0, "precision": 1.0, "predictedPositive": 61, "recall": 0.7625, "total": 80, "trueNegative": 0, "truePositive": 61, "wilsonPrecisionLower95": 0.9408 }, "replyableMessage": { "f1": 0.9189, "falseNegative": 12, "falsePositive": 0, "precision": 1.0, "predictedPositive": 68, "recall": 0.85, "total": 80, "trueNegative": 0, "truePositive": 68, "wilsonPrecisionLower95": 0.9465 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.6441, "falseNegative": 42, "falsePositive": 0, "precision": 1.0, "predictedPositive": 38, "recall": 0.475, "total": 80, "trueNegative": 0, "truePositive": 38, "wilsonPrecisionLower95": 0.9082 } }, "recordCount": 80 } }, "intentsMeeting95Precision": 4, "macroF1": 0.3899, "macroPrecision": 0.5489, "macroRecall": 0.3139, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 640, "trueNegative": 640, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.5841, "falseNegative": 141, "falsePositive": 0, "precision": 1.0, "predictedPositive": 99, "recall": 0.4125, "total": 640, "trueNegative": 400, "truePositive": 99, "wilsonPrecisionLower95": 0.9626 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 4, "precision": 0.0, "predictedPositive": 4, "recall": 0.0, "total": 640, "trueNegative": 636, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "followUpReminder": { "f1": 0.6992, "falseNegative": 37, "falsePositive": 0, "precision": 1.0, "predictedPositive": 43, "recall": 0.5375, "total": 640, "trueNegative": 560, "truePositive": 43, "wilsonPrecisionLower95": 0.918 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 640, "trueNegative": 640, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.7097, "falseNegative": 72, "falsePositive": 0, "precision": 1.0, "predictedPositive": 88, "recall": 0.55, "total": 640, "trueNegative": 480, "truePositive": 88, "wilsonPrecisionLower95": 0.9582 }, "replyableMessage": { "f1": 0.9195, "falseNegative": 40, "falsePositive": 23, "precision": 0.9399, "predictedPositive": 383, "recall": 0.9, "total": 640, "trueNegative": 217, "truePositive": 360, "wilsonPrecisionLower95": 0.9115 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 640, "trueNegative": 640, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.5965, "falseNegative": 92, "falsePositive": 0, "precision": 1.0, "predictedPositive": 68, "recall": 0.425, "total": 640, "trueNegative": 480, "truePositive": 68, "wilsonPrecisionLower95": 0.9465 } }, "recordCount": 640 } }, "generatedAtUnix": 1787825530, "openTrainingRecords": 3845, "overlapChecks": { "ModelTraining/ClipboardSemantics/comprehensive-online-holdout-corpus.jsonl": 0, "ModelTraining/ClipboardSemantics/random-holdout-corpus.jsonl": 0, "ModelTraining/ClipboardSemantics/targeted-release-holdout-corpus.jsonl": 0 }, "platformLimitation": "Linux surrogate research only. Apple Create ML/Core ML artifacts must be retrained and tested on macOS before deployment.", "releaseGatePassed": false, "roundCount": 20, "schemaVersion": 1, "seed": 20260827, "selectedConfiguration": { "alpha": 2e-05, "augmentation": "none", "char_max": 5, "char_min": 2, "external_weight": 0.5700000000000001, "hard_example_weight": 1.0, "l1_ratio": 0.0, "max_char_features": 45000, "max_word_features": 22000, "min_df": 2, "pseudo_weight": 0.16499999999999998, "round": 1, "word_max": 2 }, "selectedRound": 1, "selectedThresholds": { "blessing": { "abstained": false, "byLanguage": { "en": { "abstained": false, "metrics": { "f1": 0.9318, "falseNegative": 4, "falsePositive": 2, "precision": 0.9535, "predictedPositive": 43, "recall": 0.9111, "total": 1485, "trueNegative": 1438, "truePositive": 41, "wilsonPrecisionLower95": 0.8454 }, "threshold": 0.915 }, "zh-Hans": { "abstained": false, "metrics": { "f1": 0.7568, "falseNegative": 17, "falsePositive": 1, "precision": 0.9655, "predictedPositive": 29, "recall": 0.6222, "total": 1485, "trueNegative": 1439, "truePositive": 28, "wilsonPrecisionLower95": 0.8282 }, "threshold": 0.9 } }, "metrics": { "f1": 0.8375, "falseNegative": 23, "falsePositive": 3, "precision": 0.9571, "predictedPositive": 70, "recall": 0.7444, "total": 2970, "trueNegative": 2877, "truePositive": 67, "wilsonPrecisionLower95": 0.8814 }, "threshold": 0.915 }, "complaint": { "abstained": false, "byLanguage": { "en": { "abstained": false, "metrics": { "f1": 1.0, "falseNegative": 0, "falsePositive": 0, "precision": 1.0, "predictedPositive": 180, "recall": 1.0, "total": 1485, "trueNegative": 1305, "truePositive": 180, "wilsonPrecisionLower95": 0.9791 }, "threshold": 0.5 }, "zh-Hans": { "abstained": false, "metrics": { "f1": 0.9526, "falseNegative": 9, "falsePositive": 8, "precision": 0.9553, "predictedPositive": 179, "recall": 0.95, "total": 1485, "trueNegative": 1297, "truePositive": 171, "wilsonPrecisionLower95": 0.9143 }, "threshold": 0.54 } }, "metrics": { "f1": 0.9764, "falseNegative": 9, "falsePositive": 8, "precision": 0.9777, "predictedPositive": 359, "recall": 0.975, "total": 2970, "trueNegative": 2602, "truePositive": 351, "wilsonPrecisionLower95": 0.9567 }, "threshold": 0.54 }, "confirmationDecision": { "abstained": false, "byLanguage": { "en": { "abstained": false, "metrics": { "f1": 0.8848, "falseNegative": 17, "falsePositive": 2, "precision": 0.9733, "predictedPositive": 75, "recall": 0.8111, "total": 1485, "trueNegative": 1393, "truePositive": 73, "wilsonPrecisionLower95": 0.9079 }, "threshold": 0.5 }, "zh-Hans": { "abstained": false, "metrics": { "f1": 0.7947, "falseNegative": 30, "falsePositive": 1, "precision": 0.9836, "predictedPositive": 61, "recall": 0.6667, "total": 1485, "trueNegative": 1394, "truePositive": 60, "wilsonPrecisionLower95": 0.9128 }, "threshold": 0.5 } }, "metrics": { "f1": 0.8418, "falseNegative": 47, "falsePositive": 3, "precision": 0.9779, "predictedPositive": 136, "recall": 0.7389, "total": 2970, "trueNegative": 2787, "truePositive": 133, "wilsonPrecisionLower95": 0.9372 }, "threshold": 0.5 }, "followUpReminder": { "abstained": false, "byLanguage": { "en": { "abstained": false, "metrics": { "f1": 0.8554, "falseNegative": 41, "falsePositive": 6, "precision": 0.9586, "predictedPositive": 145, "recall": 0.7722, "total": 1485, "trueNegative": 1299, "truePositive": 139, "wilsonPrecisionLower95": 0.9127 }, "threshold": 0.83 }, "zh-Hans": { "abstained": false, "metrics": { "f1": 0.8354, "falseNegative": 48, "falsePositive": 4, "precision": 0.9706, "predictedPositive": 136, "recall": 0.7333, "total": 1485, "trueNegative": 1301, "truePositive": 132, "wilsonPrecisionLower95": 0.9268 }, "threshold": 0.505 } }, "metrics": { "f1": 0.7994, "falseNegative": 113, "falsePositive": 11, "precision": 0.9574, "predictedPositive": 258, "recall": 0.6861, "total": 2970, "trueNegative": 2599, "truePositive": 247, "wilsonPrecisionLower95": 0.9253 }, "threshold": 0.815 }, "invitation": { "abstained": false, "byLanguage": { "en": { "abstained": false, "metrics": { "f1": 0.8302, "falseNegative": 24, "falsePositive": 3, "precision": 0.9565, "predictedPositive": 69, "recall": 0.7333, "total": 1485, "trueNegative": 1392, "truePositive": 66, "wilsonPrecisionLower95": 0.8798 }, "threshold": 0.935 }, "zh-Hans": { "abstained": false, "metrics": { "f1": 0.6765, "falseNegative": 44, "falsePositive": 0, "precision": 1.0, "predictedPositive": 46, "recall": 0.5111, "total": 1485, "trueNegative": 1395, "truePositive": 46, "wilsonPrecisionLower95": 0.9229 }, "threshold": 0.505 } }, "metrics": { "f1": 0.5476, "falseNegative": 111, "falsePositive": 3, "precision": 0.9583, "predictedPositive": 72, "recall": 0.3833, "total": 2970, "trueNegative": 2787, "truePositive": 69, "wilsonPrecisionLower95": 0.8845 }, "threshold": 0.935 }, "question": { "abstained": false, "byLanguage": { "en": { "abstained": false, "metrics": { "f1": 0.8445, "falseNegative": 76, "falsePositive": 12, "precision": 0.9522, "predictedPositive": 251, "recall": 0.7587, "total": 1485, "trueNegative": 1158, "truePositive": 239, "wilsonPrecisionLower95": 0.9183 }, "threshold": 0.725 }, "zh-Hans": { "abstained": false, "metrics": { "f1": 0.8536, "falseNegative": 76, "falsePositive": 6, "precision": 0.9755, "predictedPositive": 245, "recall": 0.7587, "total": 1485, "trueNegative": 1164, "truePositive": 239, "wilsonPrecisionLower95": 0.9476 }, "threshold": 0.5 } }, "metrics": { "f1": 0.8729, "falseNegative": 122, "falsePositive": 26, "precision": 0.9513, "predictedPositive": 534, "recall": 0.8063, "total": 2970, "trueNegative": 2314, "truePositive": 508, "wilsonPrecisionLower95": 0.9296 }, "threshold": 0.515 }, "replyableMessage": { "abstained": false, "byLanguage": { "en": { "abstained": false, "metrics": { "f1": 0.948, "falseNegative": 52, "falsePositive": 46, "precision": 0.951, "predictedPositive": 939, "recall": 0.945, "total": 1485, "trueNegative": 494, "truePositive": 893, "wilsonPrecisionLower95": 0.9353 }, "threshold": 0.7 }, "zh-Hans": { "abstained": false, "metrics": { "f1": 0.9424, "falseNegative": 62, "falsePositive": 46, "precision": 0.9505, "predictedPositive": 929, "recall": 0.9344, "total": 1485, "trueNegative": 494, "truePositive": 883, "wilsonPrecisionLower95": 0.9346 }, "threshold": 0.695 } }, "metrics": { "f1": 0.9454, "falseNegative": 115, "falsePositive": 90, "precision": 0.9517, "predictedPositive": 1865, "recall": 0.9392, "total": 2970, "trueNegative": 990, "truePositive": 1775, "wilsonPrecisionLower95": 0.9411 }, "threshold": 0.7 }, "scheduleNegotiation": { "abstained": false, "byLanguage": { "en": { "abstained": false, "metrics": { "f1": 1.0, "falseNegative": 0, "falsePositive": 0, "precision": 1.0, "predictedPositive": 45, "recall": 1.0, "total": 1485, "trueNegative": 1440, "truePositive": 45, "wilsonPrecisionLower95": 0.9213 }, "threshold": 0.5 }, "zh-Hans": { "abstained": false, "metrics": { "f1": 0.967, "falseNegative": 1, "falsePositive": 2, "precision": 0.9565, "predictedPositive": 46, "recall": 0.9778, "total": 1485, "trueNegative": 1438, "truePositive": 44, "wilsonPrecisionLower95": 0.8547 }, "threshold": 0.93 } }, "metrics": { "f1": 0.978, "falseNegative": 1, "falsePositive": 3, "precision": 0.9674, "predictedPositive": 92, "recall": 0.9889, "total": 2970, "trueNegative": 2877, "truePositive": 89, "wilsonPrecisionLower95": 0.9085 }, "threshold": 0.91 }, "task": { "abstained": false, "byLanguage": { "en": { "abstained": false, "metrics": { "f1": 0.8403, "falseNegative": 89, "falsePositive": 14, "precision": 0.9509, "predictedPositive": 285, "recall": 0.7528, "total": 1485, "trueNegative": 1111, "truePositive": 271, "wilsonPrecisionLower95": 0.9192 }, "threshold": 0.835 }, "zh-Hans": { "abstained": false, "metrics": { "f1": 0.8278, "falseNegative": 98, "falsePositive": 11, "precision": 0.9597, "predictedPositive": 273, "recall": 0.7278, "total": 1485, "trueNegative": 1114, "truePositive": 262, "wilsonPrecisionLower95": 0.9293 }, "threshold": 0.745 } }, "metrics": { "f1": 0.8637, "falseNegative": 153, "falsePositive": 26, "precision": 0.9562, "predictedPositive": 593, "recall": 0.7875, "total": 2970, "trueNegative": 2224, "truePositive": 567, "wilsonPrecisionLower95": 0.9365 }, "threshold": 0.755 } }, "selectionObjective": 0.8669, "trainingPolicy": "No user clipboard data. Official partial labels plus deterministic label-preserving augmentation and three-round high-confidence self-training consensus. Frozen holdouts never enter training or threshold calibration." }