{ "baseTrainingRecords": 11500, "deploymentDecision": "retain-current-deployed-models", "elapsedSeconds": 188.7732, "finalEvaluations": { "golden": { "byLanguage": { "en": { "intentsMeeting95Precision": 1, "macroF1": 0.847, "macroPrecision": 0.9225, "macroRecall": 0.8029, "minimumPrecision": 0.7692, "perIntent": { "blessing": { "f1": 0.8235, "falseNegative": 3, "falsePositive": 0, "precision": 1.0, "predictedPositive": 7, "recall": 0.7, "total": 126, "trueNegative": 116, "truePositive": 7, "wilsonPrecisionLower95": 0.6457 }, "complaint": { "f1": 1.0, "falseNegative": 0, "falsePositive": 0, "precision": 1.0, "predictedPositive": 6, "recall": 1.0, "total": 126, "trueNegative": 120, "truePositive": 6, "wilsonPrecisionLower95": 0.6097 }, "confirmationDecision": { "f1": 0.8696, "falseNegative": 0, "falsePositive": 3, "precision": 0.7692, "predictedPositive": 13, "recall": 1.0, "total": 126, "trueNegative": 113, "truePositive": 10, "wilsonPrecisionLower95": 0.4974 }, "followUpReminder": { "f1": 0.8696, "falseNegative": 3, "falsePositive": 0, "precision": 1.0, "predictedPositive": 10, "recall": 0.7692, "total": 126, "trueNegative": 113, "truePositive": 10, "wilsonPrecisionLower95": 0.7225 }, "invitation": { "f1": 0.8571, "falseNegative": 2, "falsePositive": 0, "precision": 1.0, "predictedPositive": 6, "recall": 0.75, "total": 126, "trueNegative": 118, "truePositive": 6, "wilsonPrecisionLower95": 0.6097 }, "question": { "f1": 0.8636, "falseNegative": 4, "falsePositive": 2, "precision": 0.9048, "predictedPositive": 21, "recall": 0.8261, "total": 126, "trueNegative": 101, "truePositive": 19, "wilsonPrecisionLower95": 0.7109 }, "replyableMessage": { "f1": 0.8889, "falseNegative": 6, "falsePositive": 10, "precision": 0.8649, "predictedPositive": 74, "recall": 0.9143, "total": 126, "trueNegative": 46, "truePositive": 64, "wilsonPrecisionLower95": 0.7688 }, "scheduleNegotiation": { "f1": 0.8421, "falseNegative": 2, "falsePositive": 1, "precision": 0.8889, "predictedPositive": 9, "recall": 0.8, "total": 126, "trueNegative": 115, "truePositive": 8, "wilsonPrecisionLower95": 0.565 }, "task": { "f1": 0.6087, "falseNegative": 8, "falsePositive": 1, "precision": 0.875, "predictedPositive": 8, "recall": 0.4667, "total": 126, "trueNegative": 110, "truePositive": 7, "wilsonPrecisionLower95": 0.5291 } }, "recordCount": 126 }, "zh-Hans": { "intentsMeeting95Precision": 1, "macroF1": 0.652, "macroPrecision": 0.9656, "macroRecall": 0.5609, "minimumPrecision": 0.8182, "perIntent": { "blessing": { "f1": 0.3333, "falseNegative": 8, "falsePositive": 0, "precision": 1.0, "predictedPositive": 2, "recall": 0.2, "total": 126, "trueNegative": 116, "truePositive": 2, "wilsonPrecisionLower95": 0.3424 }, "complaint": { "f1": 0.6667, "falseNegative": 3, "falsePositive": 0, "precision": 1.0, "predictedPositive": 3, "recall": 0.5, "total": 126, "trueNegative": 120, "truePositive": 3, "wilsonPrecisionLower95": 0.4385 }, "confirmationDecision": { "f1": 0.8571, "falseNegative": 1, "falsePositive": 2, "precision": 0.8182, "predictedPositive": 11, "recall": 0.9, "total": 126, "trueNegative": 114, "truePositive": 9, "wilsonPrecisionLower95": 0.523 }, "followUpReminder": { "f1": 0.8182, "falseNegative": 4, "falsePositive": 0, "precision": 1.0, "predictedPositive": 9, "recall": 0.6923, "total": 126, "trueNegative": 113, "truePositive": 9, "wilsonPrecisionLower95": 0.7009 }, "invitation": { "f1": 0.5455, "falseNegative": 5, "falsePositive": 0, "precision": 1.0, "predictedPositive": 3, "recall": 0.375, "total": 126, "trueNegative": 118, "truePositive": 3, "wilsonPrecisionLower95": 0.4385 }, "question": { "f1": 0.9787, "falseNegative": 0, "falsePositive": 1, "precision": 0.9583, "predictedPositive": 24, "recall": 1.0, "total": 126, "trueNegative": 102, "truePositive": 23, "wilsonPrecisionLower95": 0.7976 }, "replyableMessage": { "f1": 0.9143, "falseNegative": 6, "falsePositive": 6, "precision": 0.9143, "predictedPositive": 70, "recall": 0.9143, "total": 126, "trueNegative": 50, "truePositive": 64, "wilsonPrecisionLower95": 0.8253 }, "scheduleNegotiation": { "f1": 0.3333, "falseNegative": 8, "falsePositive": 0, "precision": 1.0, "predictedPositive": 2, "recall": 0.2, "total": 126, "trueNegative": 116, "truePositive": 2, "wilsonPrecisionLower95": 0.3424 }, "task": { "f1": 0.4211, "falseNegative": 11, "falsePositive": 0, "precision": 1.0, "predictedPositive": 4, "recall": 0.2667, "total": 126, "trueNegative": 111, "truePositive": 4, "wilsonPrecisionLower95": 0.5101 } }, "recordCount": 126 } }, "bySource": { "acknowledgment": { "intentsMeeting95Precision": 0, "macroF1": 0.0, "macroPrecision": 0.0, "macroRecall": 0.0, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 8, "trueNegative": 8, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 8, "trueNegative": 8, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 1, "precision": 0.0, "predictedPositive": 1, "recall": 0.0, "total": 8, "trueNegative": 7, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 8, "trueNegative": 8, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 8, "trueNegative": 8, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 8, "trueNegative": 8, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 8, "trueNegative": 8, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 8, "trueNegative": 8, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 8, "trueNegative": 8, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 8 }, "blessing_boundary": { "intentsMeeting95Precision": 0, "macroF1": 0.1482, "macroPrecision": 0.1667, "macroRecall": 0.1667, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.6667, "falseNegative": 1, "falsePositive": 0, "precision": 1.0, "predictedPositive": 1, "recall": 0.5, "total": 20, "trueNegative": 18, "truePositive": 1, "wilsonPrecisionLower95": 0.2065 }, "replyableMessage": { "f1": 0.6667, "falseNegative": 0, "falsePositive": 4, "precision": 0.5, "predictedPositive": 8, "recall": 1.0, "total": 20, "trueNegative": 12, "truePositive": 4, "wilsonPrecisionLower95": 0.2152 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 2, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 18, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 20 }, "blessing_golden": { "intentsMeeting95Precision": 1, "macroF1": 0.1772, "macroPrecision": 0.2222, "macroRecall": 0.1556, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.6207, "falseNegative": 11, "falsePositive": 0, "precision": 1.0, "predictedPositive": 9, "recall": 0.45, "total": 20, "trueNegative": 0, "truePositive": 9, "wilsonPrecisionLower95": 0.7009 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.9744, "falseNegative": 1, "falsePositive": 0, "precision": 1.0, "predictedPositive": 19, "recall": 0.95, "total": 20, "trueNegative": 0, "truePositive": 19, "wilsonPrecisionLower95": 0.8318 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 20 }, "complaint_request": { "intentsMeeting95Precision": 0, "macroF1": 0.2703, "macroPrecision": 0.3333, "macroRecall": 0.2315, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.6667, "falseNegative": 3, "falsePositive": 0, "precision": 1.0, "predictedPositive": 3, "recall": 0.5, "total": 6, "trueNegative": 0, "truePositive": 3, "wilsonPrecisionLower95": 0.4385 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.8571, "falseNegative": 1, "falsePositive": 0, "precision": 1.0, "predictedPositive": 3, "recall": 0.75, "total": 6, "trueNegative": 2, "truePositive": 3, "wilsonPrecisionLower95": 0.4385 }, "replyableMessage": { "f1": 0.9091, "falseNegative": 1, "falsePositive": 0, "precision": 1.0, "predictedPositive": 5, "recall": 0.8333, "total": 6, "trueNegative": 0, "truePositive": 5, "wilsonPrecisionLower95": 0.5655 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 6 }, "complaint_statement": { "intentsMeeting95Precision": 0, "macroF1": 0.2222, "macroPrecision": 0.2222, "macroRecall": 0.2222, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 1.0, "falseNegative": 0, "falsePositive": 0, "precision": 1.0, "predictedPositive": 6, "recall": 1.0, "total": 6, "trueNegative": 0, "truePositive": 6, "wilsonPrecisionLower95": 0.6097 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 1.0, "falseNegative": 0, "falsePositive": 0, "precision": 1.0, "predictedPositive": 6, "recall": 1.0, "total": 6, "trueNegative": 0, "truePositive": 6, "wilsonPrecisionLower95": 0.6097 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 6 }, "confirmation_decision_boundary": { "intentsMeeting95Precision": 0, "macroF1": 0.0, "macroPrecision": 0.0, "macroRecall": 0.0, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 2, "precision": 0.0, "predictedPositive": 2, "recall": 0.0, "total": 20, "trueNegative": 18, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.0, "falseNegative": 0, "falsePositive": 3, "precision": 0.0, "predictedPositive": 3, "recall": 0.0, "total": 20, "trueNegative": 17, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 20 }, "confirmation_decision_golden": { "intentsMeeting95Precision": 2, "macroF1": 0.2194, "macroPrecision": 0.2222, "macroRecall": 0.2167, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.9744, "falseNegative": 1, "falsePositive": 0, "precision": 1.0, "predictedPositive": 19, "recall": 0.95, "total": 20, "trueNegative": 0, "truePositive": 19, "wilsonPrecisionLower95": 0.8318 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 1.0, "falseNegative": 0, "falsePositive": 0, "precision": 1.0, "predictedPositive": 20, "recall": 1.0, "total": 20, "trueNegative": 0, "truePositive": 20, "wilsonPrecisionLower95": 0.8389 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 20 }, "conversational_message": { "intentsMeeting95Precision": 1, "macroF1": 0.1111, "macroPrecision": 0.1111, "macroRecall": 0.1111, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 10, "trueNegative": 10, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 10, "trueNegative": 10, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 10, "trueNegative": 10, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 10, "trueNegative": 10, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 10, "trueNegative": 10, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 10, "trueNegative": 10, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 1.0, "falseNegative": 0, "falsePositive": 0, "precision": 1.0, "predictedPositive": 10, "recall": 1.0, "total": 10, "trueNegative": 0, "truePositive": 10, "wilsonPrecisionLower95": 0.7225 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 10, "trueNegative": 10, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 10, "trueNegative": 10, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 10 }, "event_statement": { "intentsMeeting95Precision": 0, "macroF1": 0.0, "macroPrecision": 0.0, "macroRecall": 0.0, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 4, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 4, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 4, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 4, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 4, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 4, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 4, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 4, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 4, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 4 }, "follow_up_reminder_boundary": { "intentsMeeting95Precision": 0, "macroF1": 0.0, "macroPrecision": 0.0, "macroRecall": 0.0, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 1, "precision": 0.0, "predictedPositive": 1, "recall": 0.0, "total": 20, "trueNegative": 19, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.0, "falseNegative": 0, "falsePositive": 3, "precision": 0.0, "predictedPositive": 3, "recall": 0.0, "total": 20, "trueNegative": 17, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 20 }, "follow_up_reminder_golden": { "intentsMeeting95Precision": 1, "macroF1": 0.2187, "macroPrecision": 0.3333, "macroRecall": 0.1706, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.8571, "falseNegative": 5, "falsePositive": 0, "precision": 1.0, "predictedPositive": 15, "recall": 0.75, "total": 20, "trueNegative": 0, "truePositive": 15, "wilsonPrecisionLower95": 0.7961 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.6667, "falseNegative": 7, "falsePositive": 0, "precision": 1.0, "predictedPositive": 7, "recall": 0.5, "total": 20, "trueNegative": 6, "truePositive": 7, "wilsonPrecisionLower95": 0.6457 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.4444, "falseNegative": 10, "falsePositive": 0, "precision": 1.0, "predictedPositive": 4, "recall": 0.2857, "total": 20, "trueNegative": 6, "truePositive": 4, "wilsonPrecisionLower95": 0.5101 } }, "recordCount": 20 }, "information_question": { "intentsMeeting95Precision": 0, "macroF1": 0.1693, "macroPrecision": 0.2222, "macroRecall": 0.1389, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 4, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 4, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 4, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 4, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 4, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.6667, "falseNegative": 2, "falsePositive": 0, "precision": 1.0, "predictedPositive": 2, "recall": 0.5, "total": 4, "trueNegative": 0, "truePositive": 2, "wilsonPrecisionLower95": 0.3424 }, "replyableMessage": { "f1": 0.8571, "falseNegative": 1, "falsePositive": 0, "precision": 1.0, "predictedPositive": 3, "recall": 0.75, "total": 4, "trueNegative": 0, "truePositive": 3, "wilsonPrecisionLower95": 0.4385 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 4, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 4, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 4 }, "invitation_question": { "intentsMeeting95Precision": 0, "macroF1": 0.3175, "macroPrecision": 0.3333, "macroRecall": 0.3056, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 8, "trueNegative": 8, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 8, "trueNegative": 8, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 8, "trueNegative": 8, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 8, "trueNegative": 8, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.8571, "falseNegative": 2, "falsePositive": 0, "precision": 1.0, "predictedPositive": 6, "recall": 0.75, "total": 8, "trueNegative": 0, "truePositive": 6, "wilsonPrecisionLower95": 0.6097 }, "question": { "f1": 1.0, "falseNegative": 0, "falsePositive": 0, "precision": 1.0, "predictedPositive": 8, "recall": 1.0, "total": 8, "trueNegative": 0, "truePositive": 8, "wilsonPrecisionLower95": 0.6756 }, "replyableMessage": { "f1": 1.0, "falseNegative": 0, "falsePositive": 0, "precision": 1.0, "predictedPositive": 8, "recall": 1.0, "total": 8, "trueNegative": 0, "truePositive": 8, "wilsonPrecisionLower95": 0.6756 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 8, "trueNegative": 8, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 8, "trueNegative": 8, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 8 }, "negative_news": { "intentsMeeting95Precision": 0, "macroF1": 0.0, "macroPrecision": 0.0, "macroRecall": 0.0, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.0, "falseNegative": 0, "falsePositive": 1, "precision": 0.0, "predictedPositive": 1, "recall": 0.0, "total": 6, "trueNegative": 5, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 6 }, "neutral_fact": { "intentsMeeting95Precision": 0, "macroF1": 0.0, "macroPrecision": 0.0, "macroRecall": 0.0, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 10, "trueNegative": 10, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 10, "trueNegative": 10, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 10, "trueNegative": 10, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 10, "trueNegative": 10, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 10, "trueNegative": 10, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 10, "trueNegative": 10, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.0, "falseNegative": 0, "falsePositive": 2, "precision": 0.0, "predictedPositive": 2, "recall": 0.0, "total": 10, "trueNegative": 8, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 10, "trueNegative": 10, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 10, "trueNegative": 10, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 10 }, "positive_feedback": { "intentsMeeting95Precision": 0, "macroF1": 0.1111, "macroPrecision": 0.1111, "macroRecall": 0.1111, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 1.0, "falseNegative": 0, "falsePositive": 0, "precision": 1.0, "predictedPositive": 6, "recall": 1.0, "total": 6, "trueNegative": 0, "truePositive": 6, "wilsonPrecisionLower95": 0.6097 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 6 }, "quoted_question": { "intentsMeeting95Precision": 0, "macroF1": 0.0, "macroPrecision": 0.0, "macroRecall": 0.0, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 4, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 4, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 4, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 4, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 4, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 4, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 4, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 4, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 1, "precision": 0.0, "predictedPositive": 1, "recall": 0.0, "total": 4, "trueNegative": 3, "truePositive": 0, "wilsonPrecisionLower95": 0.0 } }, "recordCount": 4 }, "schedule_negotiation_boundary": { "intentsMeeting95Precision": 0, "macroF1": 0.2541, "macroPrecision": 0.284, "macroRecall": 0.2639, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.5455, "falseNegative": 5, "falsePositive": 0, "precision": 1.0, "predictedPositive": 3, "recall": 0.375, "total": 20, "trueNegative": 12, "truePositive": 3, "wilsonPrecisionLower95": 0.4385 }, "question": { "f1": 0.8, "falseNegative": 0, "falsePositive": 2, "precision": 0.6667, "predictedPositive": 6, "recall": 1.0, "total": 20, "trueNegative": 14, "truePositive": 4, "wilsonPrecisionLower95": 0.3 }, "replyableMessage": { "f1": 0.9412, "falseNegative": 0, "falsePositive": 1, "precision": 0.8889, "predictedPositive": 9, "recall": 1.0, "total": 20, "trueNegative": 11, "truePositive": 8, "wilsonPrecisionLower95": 0.565 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 1, "precision": 0.0, "predictedPositive": 1, "recall": 0.0, "total": 20, "trueNegative": 19, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 20 }, "schedule_negotiation_golden": { "intentsMeeting95Precision": 2, "macroF1": 0.2905, "macroPrecision": 0.3275, "macroRecall": 0.2722, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 1, "precision": 0.0, "predictedPositive": 1, "recall": 0.0, "total": 20, "trueNegative": 19, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.973, "falseNegative": 0, "falsePositive": 1, "precision": 0.9474, "predictedPositive": 19, "recall": 1.0, "total": 20, "trueNegative": 1, "truePositive": 18, "wilsonPrecisionLower95": 0.7536 }, "replyableMessage": { "f1": 0.9744, "falseNegative": 1, "falsePositive": 0, "precision": 1.0, "predictedPositive": 19, "recall": 0.95, "total": 20, "trueNegative": 0, "truePositive": 19, "wilsonPrecisionLower95": 0.8318 }, "scheduleNegotiation": { "f1": 0.6667, "falseNegative": 10, "falsePositive": 0, "precision": 1.0, "predictedPositive": 10, "recall": 0.5, "total": 20, "trueNegative": 0, "truePositive": 10, "wilsonPrecisionLower95": 0.7225 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 20, "trueNegative": 20, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 20 }, "self_plan": { "intentsMeeting95Precision": 0, "macroF1": 0.1111, "macroPrecision": 0.1111, "macroRecall": 0.1111, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 1.0, "falseNegative": 0, "falsePositive": 0, "precision": 1.0, "predictedPositive": 2, "recall": 1.0, "total": 6, "trueNegative": 4, "truePositive": 2, "wilsonPrecisionLower95": 0.3424 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.0, "falseNegative": 0, "falsePositive": 2, "precision": 0.0, "predictedPositive": 2, "recall": 0.0, "total": 6, "trueNegative": 4, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 6 }, "task_question": { "intentsMeeting95Precision": 0, "macroF1": 0.2963, "macroPrecision": 0.3333, "macroRecall": 0.2778, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 1.0, "falseNegative": 0, "falsePositive": 0, "precision": 1.0, "predictedPositive": 6, "recall": 1.0, "total": 6, "trueNegative": 0, "truePositive": 6, "wilsonPrecisionLower95": 0.6097 }, "replyableMessage": { "f1": 1.0, "falseNegative": 0, "falsePositive": 0, "precision": 1.0, "predictedPositive": 6, "recall": 1.0, "total": 6, "trueNegative": 0, "truePositive": 6, "wilsonPrecisionLower95": 0.6097 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 6, "trueNegative": 6, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.6667, "falseNegative": 3, "falsePositive": 0, "precision": 1.0, "predictedPositive": 3, "recall": 0.5, "total": 6, "trueNegative": 0, "truePositive": 3, "wilsonPrecisionLower95": 0.4385 } }, "recordCount": 6 }, "task_statement": { "intentsMeeting95Precision": 0, "macroF1": 0.2519, "macroPrecision": 0.3333, "macroRecall": 0.2083, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 8, "trueNegative": 8, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 8, "trueNegative": 8, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 8, "trueNegative": 8, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.6667, "falseNegative": 2, "falsePositive": 0, "precision": 1.0, "predictedPositive": 2, "recall": 0.5, "total": 8, "trueNegative": 4, "truePositive": 2, "wilsonPrecisionLower95": 0.3424 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 8, "trueNegative": 8, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 8, "trueNegative": 8, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.9333, "falseNegative": 1, "falsePositive": 0, "precision": 1.0, "predictedPositive": 7, "recall": 0.875, "total": 8, "trueNegative": 0, "truePositive": 7, "wilsonPrecisionLower95": 0.6457 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 8, "trueNegative": 8, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.6667, "falseNegative": 4, "falsePositive": 0, "precision": 1.0, "predictedPositive": 4, "recall": 0.5, "total": 8, "trueNegative": 0, "truePositive": 4, "wilsonPrecisionLower95": 0.5101 } }, "recordCount": 8 } }, "intentsMeeting95Precision": 1, "macroF1": 0.7666, "macroPrecision": 0.9377, "macroRecall": 0.6819, "minimumPrecision": 0.7917, "perIntent": { "blessing": { "f1": 0.6207, "falseNegative": 11, "falsePositive": 0, "precision": 1.0, "predictedPositive": 9, "recall": 0.45, "total": 252, "trueNegative": 232, "truePositive": 9, "wilsonPrecisionLower95": 0.7009 }, "complaint": { "f1": 0.8571, "falseNegative": 3, "falsePositive": 0, "precision": 1.0, "predictedPositive": 9, "recall": 0.75, "total": 252, "trueNegative": 240, "truePositive": 9, "wilsonPrecisionLower95": 0.7009 }, "confirmationDecision": { "f1": 0.8636, "falseNegative": 1, "falsePositive": 5, "precision": 0.7917, "predictedPositive": 24, "recall": 0.95, "total": 252, "trueNegative": 227, "truePositive": 19, "wilsonPrecisionLower95": 0.5953 }, "followUpReminder": { "f1": 0.8444, "falseNegative": 7, "falsePositive": 0, "precision": 1.0, "predictedPositive": 19, "recall": 0.7308, "total": 252, "trueNegative": 226, "truePositive": 19, "wilsonPrecisionLower95": 0.8318 }, "invitation": { "f1": 0.72, "falseNegative": 7, "falsePositive": 0, "precision": 1.0, "predictedPositive": 9, "recall": 0.5625, "total": 252, "trueNegative": 236, "truePositive": 9, "wilsonPrecisionLower95": 0.7009 }, "question": { "f1": 0.9231, "falseNegative": 4, "falsePositive": 3, "precision": 0.9333, "predictedPositive": 45, "recall": 0.913, "total": 252, "trueNegative": 203, "truePositive": 42, "wilsonPrecisionLower95": 0.8214 }, "replyableMessage": { "f1": 0.9014, "falseNegative": 12, "falsePositive": 16, "precision": 0.8889, "predictedPositive": 144, "recall": 0.9143, "total": 252, "trueNegative": 96, "truePositive": 128, "wilsonPrecisionLower95": 0.8271 }, "scheduleNegotiation": { "f1": 0.6452, "falseNegative": 10, "falsePositive": 1, "precision": 0.9091, "predictedPositive": 11, "recall": 0.5, "total": 252, "trueNegative": 231, "truePositive": 10, "wilsonPrecisionLower95": 0.6226 }, "task": { "f1": 0.5238, "falseNegative": 19, "falsePositive": 1, "precision": 0.9167, "predictedPositive": 12, "recall": 0.3667, "total": 252, "trueNegative": 221, "truePositive": 11, "wilsonPrecisionLower95": 0.6461 } }, "recordCount": 252 }, "randomHoldout": { "byLanguage": { "en": { "intentsMeeting95Precision": 5, "macroF1": 0.6248, "macroPrecision": 0.8629, "macroRecall": 0.5355, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 340, "trueNegative": 340, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.3333, "falseNegative": 16, "falsePositive": 0, "precision": 1.0, "predictedPositive": 4, "recall": 0.2, "total": 340, "trueNegative": 320, "truePositive": 4, "wilsonPrecisionLower95": 0.5101 }, "confirmationDecision": { "f1": 0.9091, "falseNegative": 0, "falsePositive": 4, "precision": 0.8333, "predictedPositive": 24, "recall": 1.0, "total": 340, "trueNegative": 316, "truePositive": 20, "wilsonPrecisionLower95": 0.6415 }, "followUpReminder": { "f1": 0.9027, "falseNegative": 9, "falsePositive": 2, "precision": 0.9623, "predictedPositive": 53, "recall": 0.85, "total": 340, "trueNegative": 278, "truePositive": 51, "wilsonPrecisionLower95": 0.8725 }, "invitation": { "f1": 0.5714, "falseNegative": 12, "falsePositive": 0, "precision": 1.0, "predictedPositive": 8, "recall": 0.4, "total": 340, "trueNegative": 320, "truePositive": 8, "wilsonPrecisionLower95": 0.6756 }, "question": { "f1": 0.6087, "falseNegative": 45, "falsePositive": 0, "precision": 1.0, "predictedPositive": 35, "recall": 0.4375, "total": 340, "trueNegative": 260, "truePositive": 35, "wilsonPrecisionLower95": 0.9011 }, "replyableMessage": { "f1": 0.8859, "falseNegative": 37, "falsePositive": 5, "precision": 0.9702, "predictedPositive": 168, "recall": 0.815, "total": 340, "trueNegative": 135, "truePositive": 163, "wilsonPrecisionLower95": 0.9322 }, "scheduleNegotiation": { "f1": 0.8235, "falseNegative": 6, "falsePositive": 0, "precision": 1.0, "predictedPositive": 14, "recall": 0.7, "total": 340, "trueNegative": 320, "truePositive": 14, "wilsonPrecisionLower95": 0.7847 }, "task": { "f1": 0.5882, "falseNegative": 35, "falsePositive": 0, "precision": 1.0, "predictedPositive": 25, "recall": 0.4167, "total": 340, "trueNegative": 280, "truePositive": 25, "wilsonPrecisionLower95": 0.8668 } }, "recordCount": 340 }, "zh-Hans": { "intentsMeeting95Precision": 3, "macroF1": 0.497, "macroPrecision": 0.7719, "macroRecall": 0.4055, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 340, "trueNegative": 340, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.3333, "falseNegative": 16, "falsePositive": 0, "precision": 1.0, "predictedPositive": 4, "recall": 0.2, "total": 340, "trueNegative": 320, "truePositive": 4, "wilsonPrecisionLower95": 0.5101 }, "confirmationDecision": { "f1": 0.8235, "falseNegative": 6, "falsePositive": 0, "precision": 1.0, "predictedPositive": 14, "recall": 0.7, "total": 340, "trueNegative": 320, "truePositive": 14, "wilsonPrecisionLower95": 0.7847 }, "followUpReminder": { "f1": 0.7879, "falseNegative": 21, "falsePositive": 0, "precision": 1.0, "predictedPositive": 39, "recall": 0.65, "total": 340, "trueNegative": 280, "truePositive": 39, "wilsonPrecisionLower95": 0.9103 }, "invitation": { "f1": 0.6207, "falseNegative": 11, "falsePositive": 0, "precision": 1.0, "predictedPositive": 9, "recall": 0.45, "total": 340, "trueNegative": 320, "truePositive": 9, "wilsonPrecisionLower95": 0.7009 }, "question": { "f1": 0.7786, "falseNegative": 29, "falsePositive": 0, "precision": 1.0, "predictedPositive": 51, "recall": 0.6375, "total": 340, "trueNegative": 260, "truePositive": 51, "wilsonPrecisionLower95": 0.93 }, "replyableMessage": { "f1": 0.9203, "falseNegative": 21, "falsePositive": 10, "precision": 0.9471, "predictedPositive": 189, "recall": 0.895, "total": 340, "trueNegative": 130, "truePositive": 179, "wilsonPrecisionLower95": 0.9054 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 20, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 340, "trueNegative": 320, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.209, "falseNegative": 53, "falsePositive": 0, "precision": 1.0, "predictedPositive": 7, "recall": 0.1167, "total": 340, "trueNegative": 280, "truePositive": 7, "wilsonPrecisionLower95": 0.6457 } }, "recordCount": 340 } }, "bySource": { "acknowledgment_boundary": { "intentsMeeting95Precision": 0, "macroF1": 0.0, "macroPrecision": 0.0, "macroRecall": 0.0, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 40 }, "complaint_support": { "intentsMeeting95Precision": 1, "macroF1": 0.1783, "macroPrecision": 0.3333, "macroRecall": 0.1389, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.3333, "falseNegative": 32, "falsePositive": 0, "precision": 1.0, "predictedPositive": 8, "recall": 0.2, "total": 40, "trueNegative": 0, "truePositive": 8, "wilsonPrecisionLower95": 0.6756 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.3673, "falseNegative": 31, "falsePositive": 0, "precision": 1.0, "predictedPositive": 9, "recall": 0.225, "total": 40, "trueNegative": 0, "truePositive": 9, "wilsonPrecisionLower95": 0.7009 }, "replyableMessage": { "f1": 0.9041, "falseNegative": 7, "falsePositive": 0, "precision": 1.0, "predictedPositive": 33, "recall": 0.825, "total": 40, "trueNegative": 0, "truePositive": 33, "wilsonPrecisionLower95": 0.8957 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 40 }, "confirmation_decision": { "intentsMeeting95Precision": 2, "macroF1": 0.2026, "macroPrecision": 0.2222, "macroRecall": 0.1861, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.9189, "falseNegative": 6, "falsePositive": 0, "precision": 1.0, "predictedPositive": 34, "recall": 0.85, "total": 40, "trueNegative": 0, "truePositive": 34, "wilsonPrecisionLower95": 0.8985 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.9041, "falseNegative": 7, "falsePositive": 0, "precision": 1.0, "predictedPositive": 33, "recall": 0.825, "total": 40, "trueNegative": 0, "truePositive": 33, "wilsonPrecisionLower95": 0.8957 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 40 }, "conversation": { "intentsMeeting95Precision": 1, "macroF1": 0.1053, "macroPrecision": 0.1111, "macroRecall": 0.1, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.9474, "falseNegative": 4, "falsePositive": 0, "precision": 1.0, "predictedPositive": 36, "recall": 0.9, "total": 40, "trueNegative": 0, "truePositive": 36, "wilsonPrecisionLower95": 0.9036 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 40 }, "event_boundary": { "intentsMeeting95Precision": 0, "macroF1": 0.0, "macroPrecision": 0.0, "macroRecall": 0.0, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.0, "falseNegative": 0, "falsePositive": 2, "precision": 0.0, "predictedPositive": 2, "recall": 0.0, "total": 40, "trueNegative": 38, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 40 }, "fixed_invitation": { "intentsMeeting95Precision": 3, "macroF1": 0.2797, "macroPrecision": 0.3333, "macroRecall": 0.2528, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.5965, "falseNegative": 23, "falsePositive": 0, "precision": 1.0, "predictedPositive": 17, "recall": 0.425, "total": 40, "trueNegative": 0, "truePositive": 17, "wilsonPrecisionLower95": 0.8157 }, "question": { "f1": 0.9333, "falseNegative": 5, "falsePositive": 0, "precision": 1.0, "predictedPositive": 35, "recall": 0.875, "total": 40, "trueNegative": 0, "truePositive": 35, "wilsonPrecisionLower95": 0.9011 }, "replyableMessage": { "f1": 0.9873, "falseNegative": 1, "falsePositive": 0, "precision": 1.0, "predictedPositive": 39, "recall": 0.975, "total": 40, "trueNegative": 0, "truePositive": 39, "wilsonPrecisionLower95": 0.9103 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 40 }, "follow_up_instruction": { "intentsMeeting95Precision": 3, "macroF1": 0.2225, "macroPrecision": 0.3333, "macroRecall": 0.175, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.7302, "falseNegative": 17, "falsePositive": 0, "precision": 1.0, "predictedPositive": 23, "recall": 0.575, "total": 40, "trueNegative": 0, "truePositive": 23, "wilsonPrecisionLower95": 0.8569 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.8406, "falseNegative": 11, "falsePositive": 0, "precision": 1.0, "predictedPositive": 29, "recall": 0.725, "total": 40, "trueNegative": 0, "truePositive": 29, "wilsonPrecisionLower95": 0.883 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.4314, "falseNegative": 29, "falsePositive": 0, "precision": 1.0, "predictedPositive": 11, "recall": 0.275, "total": 40, "trueNegative": 0, "truePositive": 11, "wilsonPrecisionLower95": 0.7412 } }, "recordCount": 40 }, "follow_up_personal_reminder": { "intentsMeeting95Precision": 1, "macroF1": 0.097, "macroPrecision": 0.1111, "macroRecall": 0.0861, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.8732, "falseNegative": 9, "falsePositive": 0, "precision": 1.0, "predictedPositive": 31, "recall": 0.775, "total": 40, "trueNegative": 0, "truePositive": 31, "wilsonPrecisionLower95": 0.8897 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 40 }, "information_question": { "intentsMeeting95Precision": 2, "macroF1": 0.1421, "macroPrecision": 0.2222, "macroRecall": 0.1083, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 2, "precision": 0.0, "predictedPositive": 2, "recall": 0.0, "total": 40, "trueNegative": 38, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 2, "precision": 0.0, "predictedPositive": 2, "recall": 0.0, "total": 40, "trueNegative": 38, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.4906, "falseNegative": 27, "falsePositive": 0, "precision": 1.0, "predictedPositive": 13, "recall": 0.325, "total": 40, "trueNegative": 0, "truePositive": 13, "wilsonPrecisionLower95": 0.7719 }, "replyableMessage": { "f1": 0.7879, "falseNegative": 14, "falsePositive": 0, "precision": 1.0, "predictedPositive": 26, "recall": 0.65, "total": 40, "trueNegative": 0, "truePositive": 26, "wilsonPrecisionLower95": 0.8713 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 40 }, "negative_news": { "intentsMeeting95Precision": 0, "macroF1": 0.0, "macroPrecision": 0.0, "macroRecall": 0.0, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.0, "falseNegative": 0, "falsePositive": 3, "precision": 0.0, "predictedPositive": 3, "recall": 0.0, "total": 40, "trueNegative": 37, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 40 }, "neutral_fact": { "intentsMeeting95Precision": 0, "macroF1": 0.0, "macroPrecision": 0.0, "macroRecall": 0.0, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.0, "falseNegative": 0, "falsePositive": 5, "precision": 0.0, "predictedPositive": 5, "recall": 0.0, "total": 40, "trueNegative": 35, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 40 }, "positive_feedback": { "intentsMeeting95Precision": 1, "macroF1": 0.1068, "macroPrecision": 0.1111, "macroRecall": 0.1028, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.961, "falseNegative": 3, "falsePositive": 0, "precision": 1.0, "predictedPositive": 37, "recall": 0.925, "total": 40, "trueNegative": 0, "truePositive": 37, "wilsonPrecisionLower95": 0.9059 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 40 }, "quoted_question_boundary": { "intentsMeeting95Precision": 0, "macroF1": 0.0, "macroPrecision": 0.0, "macroRecall": 0.0, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.0, "falseNegative": 0, "falsePositive": 1, "precision": 0.0, "predictedPositive": 1, "recall": 0.0, "total": 40, "trueNegative": 39, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 40 }, "schedule_negotiation": { "intentsMeeting95Precision": 3, "macroF1": 0.2621, "macroPrecision": 0.3333, "macroRecall": 0.2306, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.8406, "falseNegative": 11, "falsePositive": 0, "precision": 1.0, "predictedPositive": 29, "recall": 0.725, "total": 40, "trueNegative": 0, "truePositive": 29, "wilsonPrecisionLower95": 0.883 }, "replyableMessage": { "f1": 1.0, "falseNegative": 0, "falsePositive": 0, "precision": 1.0, "predictedPositive": 40, "recall": 1.0, "total": 40, "trueNegative": 0, "truePositive": 40, "wilsonPrecisionLower95": 0.9124 }, "scheduleNegotiation": { "f1": 0.5185, "falseNegative": 26, "falsePositive": 0, "precision": 1.0, "predictedPositive": 14, "recall": 0.35, "total": 40, "trueNegative": 0, "truePositive": 14, "wilsonPrecisionLower95": 0.7847 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 40 }, "task_next_action": { "intentsMeeting95Precision": 2, "macroF1": 0.2304, "macroPrecision": 0.3333, "macroRecall": 0.2056, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 1, "precision": 0.0, "predictedPositive": 1, "recall": 0.0, "total": 40, "trueNegative": 39, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "followUpReminder": { "f1": 0.9474, "falseNegative": 4, "falsePositive": 0, "precision": 1.0, "predictedPositive": 36, "recall": 0.9, "total": 40, "trueNegative": 0, "truePositive": 36, "wilsonPrecisionLower95": 0.9036 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.9041, "falseNegative": 7, "falsePositive": 0, "precision": 1.0, "predictedPositive": 33, "recall": 0.825, "total": 40, "trueNegative": 0, "truePositive": 33, "wilsonPrecisionLower95": 0.8957 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.2222, "falseNegative": 35, "falsePositive": 0, "precision": 1.0, "predictedPositive": 5, "recall": 0.125, "total": 40, "trueNegative": 0, "truePositive": 5, "wilsonPrecisionLower95": 0.5655 } }, "recordCount": 40 }, "task_request": { "intentsMeeting95Precision": 2, "macroF1": 0.1688, "macroPrecision": 0.2222, "macroRecall": 0.1444, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 1, "precision": 0.0, "predictedPositive": 1, "recall": 0.0, "total": 40, "trueNegative": 39, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.9474, "falseNegative": 4, "falsePositive": 0, "precision": 1.0, "predictedPositive": 36, "recall": 0.9, "total": 40, "trueNegative": 0, "truePositive": 36, "wilsonPrecisionLower95": 0.9036 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.5714, "falseNegative": 24, "falsePositive": 0, "precision": 1.0, "predictedPositive": 16, "recall": 0.4, "total": 40, "trueNegative": 0, "truePositive": 16, "wilsonPrecisionLower95": 0.8064 } }, "recordCount": 40 }, "vague_future_boundary": { "intentsMeeting95Precision": 0, "macroF1": 0.0, "macroPrecision": 0.0, "macroRecall": 0.0, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.0, "falseNegative": 0, "falsePositive": 4, "precision": 0.0, "predictedPositive": 4, "recall": 0.0, "total": 40, "trueNegative": 36, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 40, "trueNegative": 40, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 40 } }, "intentsMeeting95Precision": 6, "macroF1": 0.577, "macroPrecision": 0.8701, "macroRecall": 0.4705, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 680, "trueNegative": 680, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.3333, "falseNegative": 32, "falsePositive": 0, "precision": 1.0, "predictedPositive": 8, "recall": 0.2, "total": 680, "trueNegative": 640, "truePositive": 8, "wilsonPrecisionLower95": 0.6756 }, "confirmationDecision": { "f1": 0.8718, "falseNegative": 6, "falsePositive": 4, "precision": 0.8947, "predictedPositive": 38, "recall": 0.85, "total": 680, "trueNegative": 636, "truePositive": 34, "wilsonPrecisionLower95": 0.7587 }, "followUpReminder": { "f1": 0.8491, "falseNegative": 30, "falsePositive": 2, "precision": 0.9783, "predictedPositive": 92, "recall": 0.75, "total": 680, "trueNegative": 558, "truePositive": 90, "wilsonPrecisionLower95": 0.9242 }, "invitation": { "f1": 0.5965, "falseNegative": 23, "falsePositive": 0, "precision": 1.0, "predictedPositive": 17, "recall": 0.425, "total": 680, "trueNegative": 640, "truePositive": 17, "wilsonPrecisionLower95": 0.8157 }, "question": { "f1": 0.6992, "falseNegative": 74, "falsePositive": 0, "precision": 1.0, "predictedPositive": 86, "recall": 0.5375, "total": 680, "trueNegative": 520, "truePositive": 86, "wilsonPrecisionLower95": 0.9572 }, "replyableMessage": { "f1": 0.9036, "falseNegative": 58, "falsePositive": 15, "precision": 0.958, "predictedPositive": 357, "recall": 0.855, "total": 680, "trueNegative": 265, "truePositive": 342, "wilsonPrecisionLower95": 0.9318 }, "scheduleNegotiation": { "f1": 0.5185, "falseNegative": 26, "falsePositive": 0, "precision": 1.0, "predictedPositive": 14, "recall": 0.35, "total": 680, "trueNegative": 640, "truePositive": 14, "wilsonPrecisionLower95": 0.7847 }, "task": { "f1": 0.4211, "falseNegative": 88, "falsePositive": 0, "precision": 1.0, "predictedPositive": 32, "recall": 0.2667, "total": 680, "trueNegative": 560, "truePositive": 32, "wilsonPrecisionLower95": 0.8928 } }, "recordCount": 680 }, "researchOnlyComprehensive": { "byLanguage": { "en": { "intentsMeeting95Precision": 1, "macroF1": 0.2884, "macroPrecision": 0.3871, "macroRecall": 0.2731, "minimumPrecision": 0.0006, "perIntent": { "blessing": { "f1": 0.4667, "falseNegative": 48, "falsePositive": 0, "precision": 1.0, "predictedPositive": 21, "recall": 0.3043, "total": 13176, "trueNegative": 13107, "truePositive": 21, "wilsonPrecisionLower95": 0.8454 }, "complaint": { "f1": 0.048, "falseNegative": 421, "falsePositive": 95, "precision": 0.1204, "predictedPositive": 108, "recall": 0.03, "total": 13176, "trueNegative": 12647, "truePositive": 13, "wilsonPrecisionLower95": 0.0717 }, "confirmationDecision": { "f1": 0.0012, "falseNegative": 11, "falsePositive": 1649, "precision": 0.0006, "predictedPositive": 1650, "recall": 0.0833, "total": 13176, "trueNegative": 11515, "truePositive": 1, "wilsonPrecisionLower95": 0.0001 }, "followUpReminder": { "f1": 0.1134, "falseNegative": 114, "falsePositive": 58, "precision": 0.1594, "predictedPositive": 69, "recall": 0.088, "total": 13176, "trueNegative": 12993, "truePositive": 11, "wilsonPrecisionLower95": 0.0914 }, "invitation": { "f1": 0.336, "falseNegative": 40, "falsePositive": 43, "precision": 0.3281, "predictedPositive": 64, "recall": 0.3443, "total": 13176, "trueNegative": 13072, "truePositive": 21, "wilsonPrecisionLower95": 0.2257 }, "question": { "f1": 0.4288, "falseNegative": 1828, "falsePositive": 378, "precision": 0.6866, "predictedPositive": 1206, "recall": 0.3117, "total": 13176, "trueNegative": 10142, "truePositive": 828, "wilsonPrecisionLower95": 0.6598 }, "replyableMessage": { "f1": 0.7642, "falseNegative": 2967, "falsePositive": 1917, "precision": 0.805, "predictedPositive": 9831, "recall": 0.7273, "total": 13176, "trueNegative": 378, "truePositive": 7914, "wilsonPrecisionLower95": 0.7971 }, "scheduleNegotiation": { "f1": 0.1333, "falseNegative": 50, "falsePositive": 171, "precision": 0.0904, "predictedPositive": 188, "recall": 0.2537, "total": 13176, "trueNegative": 12938, "truePositive": 17, "wilsonPrecisionLower95": 0.0572 }, "task": { "f1": 0.3039, "falseNegative": 766, "falsePositive": 851, "precision": 0.2932, "predictedPositive": 1204, "recall": 0.3155, "total": 13176, "trueNegative": 11206, "truePositive": 353, "wilsonPrecisionLower95": 0.2682 } }, "recordCount": 13176 }, "zh-Hans": { "intentsMeeting95Precision": 0, "macroF1": 0.1992, "macroPrecision": 0.4046, "macroRecall": 0.1613, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.2105, "falseNegative": 15, "falsePositive": 0, "precision": 1.0, "predictedPositive": 2, "recall": 0.1176, "total": 28019, "trueNegative": 28002, "truePositive": 2, "wilsonPrecisionLower95": 0.3424 }, "complaint": { "f1": 0.4077, "falseNegative": 154, "falsePositive": 183, "precision": 0.388, "predictedPositive": 299, "recall": 0.4296, "total": 28019, "trueNegative": 27566, "truePositive": 116, "wilsonPrecisionLower95": 0.3345 }, "confirmationDecision": { "f1": 0.0594, "falseNegative": 54, "falsePositive": 41, "precision": 0.0682, "predictedPositive": 44, "recall": 0.0526, "total": 28019, "trueNegative": 27921, "truePositive": 3, "wilsonPrecisionLower95": 0.0235 }, "followUpReminder": { "f1": 0.1856, "falseNegative": 234, "falsePositive": 38, "precision": 0.4493, "predictedPositive": 69, "recall": 0.117, "total": 28019, "trueNegative": 27716, "truePositive": 31, "wilsonPrecisionLower95": 0.3377 }, "invitation": { "f1": 0.0, "falseNegative": 73, "falsePositive": 9, "precision": 0.0, "predictedPositive": 9, "recall": 0.0, "total": 28019, "trueNegative": 27937, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "question": { "f1": 0.1419, "falseNegative": 6619, "falsePositive": 216, "precision": 0.7234, "predictedPositive": 781, "recall": 0.0786, "total": 28019, "trueNegative": 20619, "truePositive": 565, "wilsonPrecisionLower95": 0.691 }, "replyableMessage": { "f1": 0.7622, "falseNegative": 9480, "falsePositive": 1096, "precision": 0.9393, "predictedPositive": 18042, "recall": 0.6413, "total": 28019, "trueNegative": 497, "truePositive": 16946, "wilsonPrecisionLower95": 0.9357 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 4, "falsePositive": 5, "precision": 0.0, "predictedPositive": 5, "recall": 0.0, "total": 28019, "trueNegative": 28010, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "task": { "f1": 0.0251, "falseNegative": 649, "falsePositive": 127, "precision": 0.073, "predictedPositive": 137, "recall": 0.0152, "total": 28019, "trueNegative": 27233, "truePositive": 10, "wilsonPrecisionLower95": 0.0401 } }, "recordCount": 28019 } }, "bySource": { "AmazonScience/MASSIVE en-US test": { "intentsMeeting95Precision": 2, "macroF1": 0.1663, "macroPrecision": 0.2646, "macroRecall": 0.1307, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 2967, "trueNegative": 2967, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 17, "precision": 0.0, "predictedPositive": 17, "recall": 0.0, "total": 2967, "trueNegative": 2950, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 441, "precision": 0.0, "predictedPositive": 441, "recall": 0.0, "total": 2967, "trueNegative": 2526, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "followUpReminder": { "f1": 0.1447, "falseNegative": 86, "falsePositive": 44, "precision": 0.2, "predictedPositive": 55, "recall": 0.1134, "total": 2967, "trueNegative": 2826, "truePositive": 11, "wilsonPrecisionLower95": 0.1155 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 7, "precision": 0.0, "predictedPositive": 7, "recall": 0.0, "total": 2967, "trueNegative": 2960, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "question": { "f1": 0.3532, "falseNegative": 1266, "falsePositive": 16, "precision": 0.9563, "predictedPositive": 366, "recall": 0.2166, "total": 2967, "trueNegative": 1335, "truePositive": 350, "wilsonPrecisionLower95": 0.9302 }, "replyableMessage": { "f1": 0.7951, "falseNegative": 1009, "falsePositive": 0, "precision": 1.0, "predictedPositive": 1958, "recall": 0.6599, "total": 2967, "trueNegative": 0, "truePositive": 1958, "wilsonPrecisionLower95": 0.998 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 69, "precision": 0.0, "predictedPositive": 69, "recall": 0.0, "total": 2967, "trueNegative": 2898, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "task": { "f1": 0.204, "falseNegative": 440, "falsePositive": 348, "precision": 0.2249, "predictedPositive": 449, "recall": 0.1867, "total": 2967, "trueNegative": 2078, "truePositive": 101, "wilsonPrecisionLower95": 0.1887 } }, "recordCount": 2967 }, "AmazonScience/MASSIVE zh-CN test": { "intentsMeeting95Precision": 1, "macroF1": 0.1386, "macroPrecision": 0.3031, "macroRecall": 0.1008, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 2850, "trueNegative": 2850, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 2, "precision": 0.0, "predictedPositive": 2, "recall": 0.0, "total": 2850, "trueNegative": 2848, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 5, "precision": 0.0, "predictedPositive": 5, "recall": 0.0, "total": 2850, "trueNegative": 2845, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "followUpReminder": { "f1": 0.2537, "falseNegative": 140, "falsePositive": 13, "precision": 0.6667, "predictedPositive": 39, "recall": 0.1566, "total": 2850, "trueNegative": 2671, "truePositive": 26, "wilsonPrecisionLower95": 0.5098 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 7, "precision": 0.0, "predictedPositive": 7, "recall": 0.0, "total": 2850, "trueNegative": 2843, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "question": { "f1": 0.2054, "falseNegative": 1381, "falsePositive": 19, "precision": 0.905, "predictedPositive": 200, "recall": 0.1159, "total": 2850, "trueNegative": 1269, "truePositive": 181, "wilsonPrecisionLower95": 0.8564 }, "replyableMessage": { "f1": 0.7684, "falseNegative": 1072, "falsePositive": 0, "precision": 1.0, "predictedPositive": 1778, "recall": 0.6239, "total": 2850, "trueNegative": 0, "truePositive": 1778, "wilsonPrecisionLower95": 0.9978 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 4, "precision": 0.0, "predictedPositive": 4, "recall": 0.0, "total": 2850, "trueNegative": 2846, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "task": { "f1": 0.0196, "falseNegative": 473, "falsePositive": 27, "precision": 0.1562, "predictedPositive": 32, "recall": 0.0105, "total": 2850, "trueNegative": 2345, "truePositive": 5, "wilsonPrecisionLower95": 0.0686 } }, "recordCount": 2850 }, "CFPB via hpe-ai/customer-complaints": { "intentsMeeting95Precision": 1, "macroF1": 0.0968, "macroPrecision": 0.2685, "macroRecall": 0.0714, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 250, "trueNegative": 250, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0159, "falseNegative": 248, "falsePositive": 0, "precision": 1.0, "predictedPositive": 2, "recall": 0.008, "total": 250, "trueNegative": 0, "truePositive": 2, "wilsonPrecisionLower95": 0.3424 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 4, "precision": 0.0, "predictedPositive": 4, "recall": 0.0, "total": 250, "trueNegative": 246, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 250, "trueNegative": 250, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 250, "trueNegative": 250, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 17, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 250, "trueNegative": 233, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.7013, "falseNegative": 115, "falsePositive": 0, "precision": 1.0, "predictedPositive": 135, "recall": 0.54, "total": 250, "trueNegative": 0, "truePositive": 135, "wilsonPrecisionLower95": 0.9723 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 250, "trueNegative": 250, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.1538, "falseNegative": 48, "falsePositive": 7, "precision": 0.4167, "predictedPositive": 12, "recall": 0.0943, "total": 250, "trueNegative": 190, "truePositive": 5, "wilsonPrecisionLower95": 0.1933 } }, "recordCount": 250 }, "CPED test split": { "intentsMeeting95Precision": 0, "macroF1": 0.1434, "macroPrecision": 0.3223, "macroRecall": 0.1091, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.2105, "falseNegative": 15, "falsePositive": 0, "precision": 1.0, "predictedPositive": 2, "recall": 0.1176, "total": 25049, "trueNegative": 25032, "truePositive": 2, "wilsonPrecisionLower95": 0.3424 }, "complaint": { "f1": 0.018, "falseNegative": 147, "falsePositive": 181, "precision": 0.0163, "predictedPositive": 184, "recall": 0.02, "total": 25049, "trueNegative": 24718, "truePositive": 3, "wilsonPrecisionLower95": 0.0056 }, "confirmationDecision": { "f1": 0.0625, "falseNegative": 54, "falsePositive": 36, "precision": 0.0769, "predictedPositive": 39, "recall": 0.0526, "total": 25049, "trueNegative": 24956, "truePositive": 3, "wilsonPrecisionLower95": 0.0265 }, "followUpReminder": { "f1": 0.0775, "falseNegative": 94, "falsePositive": 25, "precision": 0.1667, "predictedPositive": 30, "recall": 0.0505, "total": 25049, "trueNegative": 24925, "truePositive": 5, "wilsonPrecisionLower95": 0.0734 }, "invitation": { "f1": 0.0, "falseNegative": 73, "falsePositive": 2, "precision": 0.0, "predictedPositive": 2, "recall": 0.0, "total": 25049, "trueNegative": 24974, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "question": { "f1": 0.1245, "falseNegative": 5202, "falsePositive": 197, "precision": 0.6609, "predictedPositive": 581, "recall": 0.0687, "total": 25049, "trueNegative": 19266, "truePositive": 384, "wilsonPrecisionLower95": 0.6215 }, "replyableMessage": { "f1": 0.7609, "falseNegative": 8378, "falsePositive": 1096, "precision": 0.9322, "predictedPositive": 16174, "recall": 0.6428, "total": 25049, "trueNegative": 497, "truePositive": 15078, "wilsonPrecisionLower95": 0.9283 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 4, "falsePositive": 1, "precision": 0.0, "predictedPositive": 1, "recall": 0.0, "total": 25049, "trueNegative": 25044, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "task": { "f1": 0.0366, "falseNegative": 163, "falsePositive": 100, "precision": 0.0476, "predictedPositive": 105, "recall": 0.0298, "total": 25049, "trueNegative": 24781, "truePositive": 5, "wilsonPrecisionLower95": 0.0205 } }, "recordCount": 25049 }, "Google Research GoEmotions test": { "intentsMeeting95Precision": 2, "macroF1": 0.2218, "macroPrecision": 0.3073, "macroRecall": 0.1928, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.4651, "falseNegative": 46, "falsePositive": 0, "precision": 1.0, "predictedPositive": 20, "recall": 0.303, "total": 5418, "trueNegative": 5352, "truePositive": 20, "wilsonPrecisionLower95": 0.8389 }, "complaint": { "f1": 0.0, "falseNegative": 20, "falsePositive": 27, "precision": 0.0, "predictedPositive": 27, "recall": 0.0, "total": 5418, "trueNegative": 5371, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 742, "precision": 0.0, "predictedPositive": 742, "recall": 0.0, "total": 5418, "trueNegative": 4676, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 6, "precision": 0.0, "predictedPositive": 6, "recall": 0.0, "total": 5418, "trueNegative": 5412, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 8, "precision": 0.0, "predictedPositive": 8, "recall": 0.0, "total": 5418, "trueNegative": 5410, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "question": { "f1": 0.3991, "falseNegative": 363, "falsePositive": 146, "precision": 0.5365, "predictedPositive": 315, "recall": 0.3177, "total": 5418, "trueNegative": 4740, "truePositive": 169, "wilsonPrecisionLower95": 0.4813 }, "replyableMessage": { "f1": 0.8467, "falseNegative": 1440, "falsePositive": 0, "precision": 1.0, "predictedPositive": 3978, "recall": 0.7342, "total": 5418, "trueNegative": 0, "truePositive": 3978, "wilsonPrecisionLower95": 0.999 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 23, "precision": 0.0, "predictedPositive": 23, "recall": 0.0, "total": 5418, "trueNegative": 5395, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "task": { "f1": 0.2857, "falseNegative": 31, "falsePositive": 64, "precision": 0.2289, "predictedPositive": 83, "recall": 0.38, "total": 5418, "trueNegative": 5304, "truePositive": 19, "wilsonPrecisionLower95": 0.1517 } }, "recordCount": 5418 }, "Meituan-Dianping/ASAP test": { "intentsMeeting95Precision": 2, "macroF1": 0.203, "macroPrecision": 0.2222, "macroRecall": 0.188, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 120, "trueNegative": 120, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.97, "falseNegative": 7, "falsePositive": 0, "precision": 1.0, "predictedPositive": 113, "recall": 0.9417, "total": 120, "trueNegative": 0, "truePositive": 113, "wilsonPrecisionLower95": 0.9671 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 120, "trueNegative": 120, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 120, "trueNegative": 120, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 120, "trueNegative": 120, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 36, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 120, "trueNegative": 84, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.8571, "falseNegative": 30, "falsePositive": 0, "precision": 1.0, "predictedPositive": 90, "recall": 0.75, "total": 120, "trueNegative": 0, "truePositive": 90, "wilsonPrecisionLower95": 0.9591 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 120, "trueNegative": 120, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 13, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 120, "trueNegative": 107, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 120 }, "Sulak2020 GitHub issues multi-repo dataset": { "intentsMeeting95Precision": 1, "macroF1": 0.1573, "macroPrecision": 0.2376, "macroRecall": 0.1229, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 2, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 1004, "trueNegative": 1002, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0995, "falseNegative": 153, "falsePositive": 46, "precision": 0.193, "predictedPositive": 57, "recall": 0.0671, "total": 1004, "trueNegative": 794, "truePositive": 11, "wilsonPrecisionLower95": 0.1113 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 2, "falsePositive": 84, "precision": 0.0, "predictedPositive": 84, "recall": 0.0, "total": 1004, "trueNegative": 918, "truePositive": 0, "wilsonPrecisionLower95": -0.0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 2, "falsePositive": 1, "precision": 0.0, "predictedPositive": 1, "recall": 0.0, "total": 1004, "trueNegative": 1001, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 2, "precision": 0.0, "predictedPositive": 2, "recall": 0.0, "total": 1004, "trueNegative": 1002, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "question": { "f1": 0.4474, "falseNegative": 39, "falsePositive": 3, "precision": 0.85, "predictedPositive": 20, "recall": 0.3036, "total": 1004, "trueNegative": 945, "truePositive": 17, "wilsonPrecisionLower95": 0.6396 }, "replyableMessage": { "f1": 0.8326, "falseNegative": 288, "falsePositive": 0, "precision": 1.0, "predictedPositive": 716, "recall": 0.7131, "total": 1004, "trueNegative": 0, "truePositive": 716, "wilsonPrecisionLower95": 0.9947 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 25, "falsePositive": 4, "precision": 0.0, "predictedPositive": 4, "recall": 0.0, "total": 1004, "trueNegative": 975, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "task": { "f1": 0.036, "falseNegative": 88, "falsePositive": 19, "precision": 0.0952, "predictedPositive": 21, "recall": 0.0222, "total": 1004, "trueNegative": 895, "truePositive": 2, "wilsonPrecisionLower95": 0.0265 } }, "recordCount": 1004 }, "Verified Enron Intent Dataset": { "intentsMeeting95Precision": 0, "macroF1": 0.3596, "macroPrecision": 0.3263, "macroRecall": 0.4461, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 1.0, "falseNegative": 0, "falsePositive": 0, "precision": 1.0, "predictedPositive": 1, "recall": 1.0, "total": 3537, "trueNegative": 3536, "truePositive": 1, "wilsonPrecisionLower95": 0.2065 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 5, "precision": 0.0, "predictedPositive": 5, "recall": 0.0, "total": 3537, "trueNegative": 3532, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "confirmationDecision": { "f1": 0.0051, "falseNegative": 9, "falsePositive": 378, "precision": 0.0026, "predictedPositive": 379, "recall": 0.1, "total": 3537, "trueNegative": 3149, "truePositive": 1, "wilsonPrecisionLower95": 0.0005 }, "followUpReminder": { "f1": 0.0, "falseNegative": 26, "falsePositive": 7, "precision": 0.0, "predictedPositive": 7, "recall": 0.0, "total": 3537, "trueNegative": 3504, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "invitation": { "f1": 0.3889, "falseNegative": 40, "falsePositive": 26, "precision": 0.4468, "predictedPositive": 47, "recall": 0.3443, "total": 3537, "trueNegative": 3450, "truePositive": 21, "wilsonPrecisionLower95": 0.3141 }, "question": { "f1": 0.6213, "falseNegative": 143, "falsePositive": 213, "precision": 0.5782, "predictedPositive": 505, "recall": 0.6713, "total": 3537, "trueNegative": 2889, "truePositive": 292, "wilsonPrecisionLower95": 0.5347 }, "replyableMessage": { "f1": 0.5259, "falseNegative": 115, "falsePositive": 1917, "precision": 0.3702, "predictedPositive": 3044, "recall": 0.9074, "total": 3537, "trueNegative": 378, "truePositive": 1127, "wilsonPrecisionLower95": 0.3533 }, "scheduleNegotiation": { "f1": 0.2537, "falseNegative": 25, "falsePositive": 75, "precision": 0.1848, "predictedPositive": 92, "recall": 0.4048, "total": 3537, "trueNegative": 3420, "truePositive": 17, "wilsonPrecisionLower95": 0.1187 }, "task": { "f1": 0.4414, "falseNegative": 159, "falsePositive": 413, "precision": 0.3537, "predictedPositive": 639, "recall": 0.587, "total": 3537, "trueNegative": 2739, "truePositive": 226, "wilsonPrecisionLower95": 0.3176 } }, "recordCount": 3537 } }, "intentsMeeting95Precision": 1, "macroF1": 0.2645, "macroPrecision": 0.4292, "macroRecall": 0.225, "minimumPrecision": 0.0024, "perIntent": { "blessing": { "f1": 0.422, "falseNegative": 63, "falsePositive": 0, "precision": 1.0, "predictedPositive": 23, "recall": 0.2674, "total": 41195, "trueNegative": 41109, "truePositive": 23, "wilsonPrecisionLower95": 0.8569 }, "complaint": { "f1": 0.2322, "falseNegative": 575, "falsePositive": 278, "precision": 0.317, "predictedPositive": 407, "recall": 0.1832, "total": 41195, "trueNegative": 40213, "truePositive": 129, "wilsonPrecisionLower95": 0.2736 }, "confirmationDecision": { "f1": 0.0045, "falseNegative": 65, "falsePositive": 1690, "precision": 0.0024, "predictedPositive": 1694, "recall": 0.058, "total": 41195, "trueNegative": 39436, "truePositive": 4, "wilsonPrecisionLower95": 0.0009 }, "followUpReminder": { "f1": 0.1591, "falseNegative": 348, "falsePositive": 96, "precision": 0.3043, "predictedPositive": 138, "recall": 0.1077, "total": 41195, "trueNegative": 40709, "truePositive": 42, "wilsonPrecisionLower95": 0.2337 }, "invitation": { "f1": 0.2029, "falseNegative": 113, "falsePositive": 52, "precision": 0.2877, "predictedPositive": 73, "recall": 0.1567, "total": 41195, "trueNegative": 41009, "truePositive": 21, "wilsonPrecisionLower95": 0.1965 }, "question": { "f1": 0.2356, "falseNegative": 8447, "falsePositive": 594, "precision": 0.7011, "predictedPositive": 1987, "recall": 0.1416, "total": 41195, "trueNegative": 30761, "truePositive": 1393, "wilsonPrecisionLower95": 0.6806 }, "replyableMessage": { "f1": 0.7628, "falseNegative": 12447, "falsePositive": 3013, "precision": 0.8919, "predictedPositive": 27873, "recall": 0.6664, "total": 41195, "trueNegative": 875, "truePositive": 24860, "wilsonPrecisionLower95": 0.8882 }, "scheduleNegotiation": { "f1": 0.1288, "falseNegative": 54, "falsePositive": 176, "precision": 0.0881, "predictedPositive": 193, "recall": 0.2394, "total": 41195, "trueNegative": 40948, "truePositive": 17, "wilsonPrecisionLower95": 0.0557 }, "task": { "f1": 0.2328, "falseNegative": 1415, "falsePositive": 978, "precision": 0.2707, "predictedPositive": 1341, "recall": 0.2042, "total": 41195, "trueNegative": 38439, "truePositive": 363, "wilsonPrecisionLower95": 0.2476 } }, "recordCount": 41195 }, "targetedRelease": { "byLanguage": { "en": { "intentsMeeting95Precision": 4, "macroF1": 0.4252, "macroPrecision": 0.5437, "macroRecall": 0.3679, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 320, "trueNegative": 320, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.6957, "falseNegative": 56, "falsePositive": 0, "precision": 1.0, "predictedPositive": 64, "recall": 0.5333, "total": 320, "trueNegative": 200, "truePositive": 64, "wilsonPrecisionLower95": 0.9434 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 8, "precision": 0.0, "predictedPositive": 8, "recall": 0.0, "total": 320, "trueNegative": 312, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "followUpReminder": { "f1": 0.961, "falseNegative": 3, "falsePositive": 0, "precision": 1.0, "predictedPositive": 37, "recall": 0.925, "total": 320, "trueNegative": 280, "truePositive": 37, "wilsonPrecisionLower95": 0.9059 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 320, "trueNegative": 320, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.4762, "falseNegative": 55, "falsePositive": 0, "precision": 1.0, "predictedPositive": 25, "recall": 0.3125, "total": 320, "trueNegative": 240, "truePositive": 25, "wilsonPrecisionLower95": 0.8668 }, "replyableMessage": { "f1": 0.8819, "falseNegative": 32, "falsePositive": 13, "precision": 0.9282, "predictedPositive": 181, "recall": 0.84, "total": 320, "trueNegative": 107, "truePositive": 168, "wilsonPrecisionLower95": 0.881 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 320, "trueNegative": 320, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.8116, "falseNegative": 24, "falsePositive": 2, "precision": 0.9655, "predictedPositive": 58, "recall": 0.7, "total": 320, "trueNegative": 238, "truePositive": 56, "wilsonPrecisionLower95": 0.8827 } }, "recordCount": 320 }, "zh-Hans": { "intentsMeeting95Precision": 4, "macroF1": 0.3643, "macroPrecision": 0.5501, "macroRecall": 0.3071, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 320, "trueNegative": 320, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.4516, "falseNegative": 85, "falsePositive": 0, "precision": 1.0, "predictedPositive": 35, "recall": 0.2917, "total": 320, "trueNegative": 200, "truePositive": 35, "wilsonPrecisionLower95": 0.9011 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 320, "trueNegative": 320, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.2609, "falseNegative": 34, "falsePositive": 0, "precision": 1.0, "predictedPositive": 6, "recall": 0.15, "total": 320, "trueNegative": 280, "truePositive": 6, "wilsonPrecisionLower95": 0.6097 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 320, "trueNegative": 320, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.8811, "falseNegative": 17, "falsePositive": 0, "precision": 1.0, "predictedPositive": 63, "recall": 0.7875, "total": 320, "trueNegative": 240, "truePositive": 63, "wilsonPrecisionLower95": 0.9425 }, "replyableMessage": { "f1": 0.9552, "falseNegative": 8, "falsePositive": 10, "precision": 0.9505, "predictedPositive": 202, "recall": 0.96, "total": 320, "trueNegative": 110, "truePositive": 192, "wilsonPrecisionLower95": 0.9113 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 320, "trueNegative": 320, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.7302, "falseNegative": 34, "falsePositive": 0, "precision": 1.0, "predictedPositive": 46, "recall": 0.575, "total": 320, "trueNegative": 240, "truePositive": 46, "wilsonPrecisionLower95": 0.9229 } }, "recordCount": 320 } }, "bySource": { "release_complaint_brief": { "intentsMeeting95Precision": 2, "macroF1": 0.1893, "macroPrecision": 0.2222, "macroRecall": 0.1708, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.7097, "falseNegative": 36, "falsePositive": 0, "precision": 1.0, "predictedPositive": 44, "recall": 0.55, "total": 80, "trueNegative": 0, "truePositive": 44, "wilsonPrecisionLower95": 0.9197 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.9937, "falseNegative": 1, "falsePositive": 0, "precision": 1.0, "predictedPositive": 79, "recall": 0.9875, "total": 80, "trueNegative": 0, "truePositive": 79, "wilsonPrecisionLower95": 0.9536 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 80 }, "release_complaint_question": { "intentsMeeting95Precision": 3, "macroF1": 0.2152, "macroPrecision": 0.3333, "macroRecall": 0.1667, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.5586, "falseNegative": 49, "falsePositive": 0, "precision": 1.0, "predictedPositive": 31, "recall": 0.3875, "total": 80, "trueNegative": 0, "truePositive": 31, "wilsonPrecisionLower95": 0.8897 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.5047, "falseNegative": 53, "falsePositive": 0, "precision": 1.0, "predictedPositive": 27, "recall": 0.3375, "total": 80, "trueNegative": 0, "truePositive": 27, "wilsonPrecisionLower95": 0.8754 }, "replyableMessage": { "f1": 0.8732, "falseNegative": 18, "falsePositive": 0, "precision": 1.0, "predictedPositive": 62, "recall": 0.775, "total": 80, "trueNegative": 0, "truePositive": 62, "wilsonPrecisionLower95": 0.9417 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 80 }, "release_complaint_unspoken": { "intentsMeeting95Precision": 2, "macroF1": 0.1565, "macroPrecision": 0.2222, "macroRecall": 0.1333, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.4615, "falseNegative": 56, "falsePositive": 0, "precision": 1.0, "predictedPositive": 24, "recall": 0.3, "total": 80, "trueNegative": 0, "truePositive": 24, "wilsonPrecisionLower95": 0.862 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.9474, "falseNegative": 8, "falsePositive": 0, "precision": 1.0, "predictedPositive": 72, "recall": 0.9, "total": 80, "trueNegative": 0, "truePositive": 72, "wilsonPrecisionLower95": 0.9493 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 80 }, "release_negative_boundary": { "intentsMeeting95Precision": 0, "macroF1": 0.0, "macroPrecision": 0.0, "macroRecall": 0.0, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 80 }, "release_personal_boundary": { "intentsMeeting95Precision": 1, "macroF1": 0.0777, "macroPrecision": 0.1111, "macroRecall": 0.0597, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.6992, "falseNegative": 37, "falsePositive": 0, "precision": 1.0, "predictedPositive": 43, "recall": 0.5375, "total": 80, "trueNegative": 0, "truePositive": 43, "wilsonPrecisionLower95": 0.918 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.0, "falseNegative": 0, "falsePositive": 4, "precision": 0.0, "predictedPositive": 4, "recall": 0.0, "total": 80, "trueNegative": 76, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 } }, "recordCount": 80 }, "release_resolved_boundary": { "intentsMeeting95Precision": 0, "macroF1": 0.0, "macroPrecision": 0.0, "macroRecall": 0.0, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.0, "falseNegative": 0, "falsePositive": 19, "precision": 0.0, "predictedPositive": 19, "recall": 0.0, "total": 80, "trueNegative": 61, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.0, "falseNegative": 0, "falsePositive": 2, "precision": 0.0, "predictedPositive": 2, "recall": 0.0, "total": 80, "trueNegative": 78, "truePositive": 0, "wilsonPrecisionLower95": 0.0 } }, "recordCount": 80 }, "release_task_owner": { "intentsMeeting95Precision": 2, "macroF1": 0.2, "macroPrecision": 0.2222, "macroRecall": 0.1847, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 8, "precision": 0.0, "predictedPositive": 8, "recall": 0.0, "total": 80, "trueNegative": 72, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "replyableMessage": { "f1": 0.9937, "falseNegative": 1, "falsePositive": 0, "precision": 1.0, "predictedPositive": 79, "recall": 0.9875, "total": 80, "trueNegative": 0, "truePositive": 79, "wilsonPrecisionLower95": 0.9536 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.806, "falseNegative": 26, "falsePositive": 0, "precision": 1.0, "predictedPositive": 54, "recall": 0.675, "total": 80, "trueNegative": 0, "truePositive": 54, "wilsonPrecisionLower95": 0.9336 } }, "recordCount": 80 }, "release_task_polite_request": { "intentsMeeting95Precision": 3, "macroF1": 0.2816, "macroPrecision": 0.3333, "macroRecall": 0.2458, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "followUpReminder": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.8652, "falseNegative": 19, "falsePositive": 0, "precision": 1.0, "predictedPositive": 61, "recall": 0.7625, "total": 80, "trueNegative": 0, "truePositive": 61, "wilsonPrecisionLower95": 0.9408 }, "replyableMessage": { "f1": 0.9189, "falseNegative": 12, "falsePositive": 0, "precision": 1.0, "predictedPositive": 68, "recall": 0.85, "total": 80, "trueNegative": 0, "truePositive": 68, "wilsonPrecisionLower95": 0.9465 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 80, "trueNegative": 80, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.75, "falseNegative": 32, "falsePositive": 0, "precision": 1.0, "predictedPositive": 48, "recall": 0.6, "total": 80, "trueNegative": 0, "truePositive": 48, "wilsonPrecisionLower95": 0.9259 } }, "recordCount": 80 } }, "intentsMeeting95Precision": 4, "macroF1": 0.4095, "macroPrecision": 0.5467, "macroRecall": 0.3375, "minimumPrecision": 0.0, "perIntent": { "blessing": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 640, "trueNegative": 640, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "complaint": { "f1": 0.5841, "falseNegative": 141, "falsePositive": 0, "precision": 1.0, "predictedPositive": 99, "recall": 0.4125, "total": 640, "trueNegative": 400, "truePositive": 99, "wilsonPrecisionLower95": 0.9626 }, "confirmationDecision": { "f1": 0.0, "falseNegative": 0, "falsePositive": 8, "precision": 0.0, "predictedPositive": 8, "recall": 0.0, "total": 640, "trueNegative": 632, "truePositive": 0, "wilsonPrecisionLower95": 0.0 }, "followUpReminder": { "f1": 0.6992, "falseNegative": 37, "falsePositive": 0, "precision": 1.0, "predictedPositive": 43, "recall": 0.5375, "total": 640, "trueNegative": 560, "truePositive": 43, "wilsonPrecisionLower95": 0.918 }, "invitation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 640, "trueNegative": 640, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "question": { "f1": 0.7097, "falseNegative": 72, "falsePositive": 0, "precision": 1.0, "predictedPositive": 88, "recall": 0.55, "total": 640, "trueNegative": 480, "truePositive": 88, "wilsonPrecisionLower95": 0.9582 }, "replyableMessage": { "f1": 0.9195, "falseNegative": 40, "falsePositive": 23, "precision": 0.9399, "predictedPositive": 383, "recall": 0.9, "total": 640, "trueNegative": 217, "truePositive": 360, "wilsonPrecisionLower95": 0.9115 }, "scheduleNegotiation": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 640, "trueNegative": 640, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "task": { "f1": 0.7727, "falseNegative": 58, "falsePositive": 2, "precision": 0.9808, "predictedPositive": 104, "recall": 0.6375, "total": 640, "trueNegative": 478, "truePositive": 102, "wilsonPrecisionLower95": 0.9326 } }, "recordCount": 640 } }, "generatedAtUnix": 1787825777, "openTrainingRecords": 3845, "overlapChecks": { "ModelTraining/ClipboardSemantics/comprehensive-online-holdout-corpus.jsonl": 0, "ModelTraining/ClipboardSemantics/random-holdout-corpus.jsonl": 0, "ModelTraining/ClipboardSemantics/targeted-release-holdout-corpus.jsonl": 0 }, "platformLimitation": "Linux surrogate research only. Apple Create ML/Core ML artifacts must be retrained and tested on macOS before deployment.", "releaseGatePassed": false, "roundCount": 20, "schemaVersion": 1, "seed": 20260827, "selectedConfiguration": { "alpha": 2e-05, "augmentation": "none", "char_max": 5, "char_min": 2, "external_weight": 0.5700000000000001, "hard_example_weight": 1.0, "l1_ratio": 0.0, "max_char_features": 45000, "max_word_features": 22000, "min_df": 2, "pseudo_weight": 0.16499999999999998, "round": 1, "word_max": 2 }, "selectedRound": 1, "selectedRoundsByIntent": { "blessing": 19, "complaint": 1, "confirmationDecision": 3, "followUpReminder": 1, "invitation": 1, "question": 1, "replyableMessage": 1, "scheduleNegotiation": 7, "task": 15 }, "selectedThresholds": { "blessing": { "abstained": true, "byLanguage": { "en": { "abstained": true, "metrics": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 0, "trueNegative": 0, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "threshold": 1.0 }, "zh-Hans": { "abstained": true, "metrics": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 0, "trueNegative": 0, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "threshold": 1.0 } }, "metrics": { "f1": 0.0, "falseNegative": 0, "falsePositive": 0, "precision": 0.0, "predictedPositive": 0, "recall": 0.0, "total": 0, "trueNegative": 0, "truePositive": 0, "wilsonPrecisionLower95": 0 }, "threshold": 1.0 }, "complaint": { "abstained": false, "byLanguage": { "en": { "abstained": false, "metrics": { "f1": 1.0, "falseNegative": 0, "falsePositive": 0, "precision": 1.0, "predictedPositive": 180, "recall": 1.0, "total": 1485, "trueNegative": 1305, "truePositive": 180, "wilsonPrecisionLower95": 0.9791 }, "threshold": 0.5 }, "zh-Hans": { "abstained": false, "metrics": { "f1": 0.9526, "falseNegative": 9, "falsePositive": 8, "precision": 0.9553, "predictedPositive": 179, "recall": 0.95, "total": 1485, "trueNegative": 1297, "truePositive": 171, "wilsonPrecisionLower95": 0.9143 }, "threshold": 0.54 } }, "metrics": { "f1": 0.9764, "falseNegative": 9, "falsePositive": 8, "precision": 0.9777, "predictedPositive": 359, "recall": 0.975, "total": 2970, "trueNegative": 2602, "truePositive": 351, "wilsonPrecisionLower95": 0.9567 }, "threshold": 0.54 }, "confirmationDecision": { "abstained": false, "byLanguage": { "en": { "abstained": false, "metrics": { "f1": 0.9727, "falseNegative": 1, "falsePositive": 4, "precision": 0.957, "predictedPositive": 93, "recall": 0.9889, "total": 1485, "trueNegative": 1391, "truePositive": 89, "wilsonPrecisionLower95": 0.8946 }, "threshold": 0.585 }, "zh-Hans": { "abstained": false, "metrics": { "f1": 0.7815, "falseNegative": 31, "falsePositive": 2, "precision": 0.9672, "predictedPositive": 61, "recall": 0.6556, "total": 1485, "trueNegative": 1393, "truePositive": 59, "wilsonPrecisionLower95": 0.8881 }, "threshold": 0.5 } }, "metrics": { "f1": 0.8701, "falseNegative": 36, "falsePositive": 7, "precision": 0.9536, "predictedPositive": 151, "recall": 0.8, "total": 2970, "trueNegative": 2783, "truePositive": 144, "wilsonPrecisionLower95": 0.9074 }, "threshold": 0.555 }, "followUpReminder": { "abstained": false, "byLanguage": { "en": { "abstained": false, "metrics": { "f1": 0.8554, "falseNegative": 41, "falsePositive": 6, "precision": 0.9586, "predictedPositive": 145, "recall": 0.7722, "total": 1485, "trueNegative": 1299, "truePositive": 139, "wilsonPrecisionLower95": 0.9127 }, "threshold": 0.83 }, "zh-Hans": { "abstained": false, "metrics": { "f1": 0.8354, "falseNegative": 48, "falsePositive": 4, "precision": 0.9706, "predictedPositive": 136, "recall": 0.7333, "total": 1485, "trueNegative": 1301, "truePositive": 132, "wilsonPrecisionLower95": 0.9268 }, "threshold": 0.505 } }, "metrics": { "f1": 0.7994, "falseNegative": 113, "falsePositive": 11, "precision": 0.9574, "predictedPositive": 258, "recall": 0.6861, "total": 2970, "trueNegative": 2599, "truePositive": 247, "wilsonPrecisionLower95": 0.9253 }, "threshold": 0.815 }, "invitation": { "abstained": false, "byLanguage": { "en": { "abstained": false, "metrics": { "f1": 0.8302, "falseNegative": 24, "falsePositive": 3, "precision": 0.9565, "predictedPositive": 69, "recall": 0.7333, "total": 1485, "trueNegative": 1392, "truePositive": 66, "wilsonPrecisionLower95": 0.8798 }, "threshold": 0.935 }, "zh-Hans": { "abstained": false, "metrics": { "f1": 0.6765, "falseNegative": 44, "falsePositive": 0, "precision": 1.0, "predictedPositive": 46, "recall": 0.5111, "total": 1485, "trueNegative": 1395, "truePositive": 46, "wilsonPrecisionLower95": 0.9229 }, "threshold": 0.505 } }, "metrics": { "f1": 0.5476, "falseNegative": 111, "falsePositive": 3, "precision": 0.9583, "predictedPositive": 72, "recall": 0.3833, "total": 2970, "trueNegative": 2787, "truePositive": 69, "wilsonPrecisionLower95": 0.8845 }, "threshold": 0.935 }, "question": { "abstained": false, "byLanguage": { "en": { "abstained": false, "metrics": { "f1": 0.8445, "falseNegative": 76, "falsePositive": 12, "precision": 0.9522, "predictedPositive": 251, "recall": 0.7587, "total": 1485, "trueNegative": 1158, "truePositive": 239, "wilsonPrecisionLower95": 0.9183 }, "threshold": 0.725 }, "zh-Hans": { "abstained": false, "metrics": { "f1": 0.8536, "falseNegative": 76, "falsePositive": 6, "precision": 0.9755, "predictedPositive": 245, "recall": 0.7587, "total": 1485, "trueNegative": 1164, "truePositive": 239, "wilsonPrecisionLower95": 0.9476 }, "threshold": 0.5 } }, "metrics": { "f1": 0.8729, "falseNegative": 122, "falsePositive": 26, "precision": 0.9513, "predictedPositive": 534, "recall": 0.8063, "total": 2970, "trueNegative": 2314, "truePositive": 508, "wilsonPrecisionLower95": 0.9296 }, "threshold": 0.515 }, "replyableMessage": { "abstained": false, "byLanguage": { "en": { "abstained": false, "metrics": { "f1": 0.948, "falseNegative": 52, "falsePositive": 46, "precision": 0.951, "predictedPositive": 939, "recall": 0.945, "total": 1485, "trueNegative": 494, "truePositive": 893, "wilsonPrecisionLower95": 0.9353 }, "threshold": 0.7 }, "zh-Hans": { "abstained": false, "metrics": { "f1": 0.9424, "falseNegative": 62, "falsePositive": 46, "precision": 0.9505, "predictedPositive": 929, "recall": 0.9344, "total": 1485, "trueNegative": 494, "truePositive": 883, "wilsonPrecisionLower95": 0.9346 }, "threshold": 0.695 } }, "metrics": { "f1": 0.9454, "falseNegative": 115, "falsePositive": 90, "precision": 0.9517, "predictedPositive": 1865, "recall": 0.9392, "total": 2970, "trueNegative": 990, "truePositive": 1775, "wilsonPrecisionLower95": 0.9411 }, "threshold": 0.7 }, "scheduleNegotiation": { "abstained": false, "byLanguage": { "en": { "abstained": false, "metrics": { "f1": 1.0, "falseNegative": 0, "falsePositive": 0, "precision": 1.0, "predictedPositive": 45, "recall": 1.0, "total": 1485, "trueNegative": 1440, "truePositive": 45, "wilsonPrecisionLower95": 0.9213 }, "threshold": 0.66 }, "zh-Hans": { "abstained": false, "metrics": { "f1": 1.0, "falseNegative": 0, "falsePositive": 0, "precision": 1.0, "predictedPositive": 45, "recall": 1.0, "total": 1485, "trueNegative": 1440, "truePositive": 45, "wilsonPrecisionLower95": 0.9213 }, "threshold": 0.55 } }, "metrics": { "f1": 1.0, "falseNegative": 0, "falsePositive": 0, "precision": 1.0, "predictedPositive": 90, "recall": 1.0, "total": 2970, "trueNegative": 2880, "truePositive": 90, "wilsonPrecisionLower95": 0.9591 }, "threshold": 0.66 }, "task": { "abstained": false, "byLanguage": { "en": { "abstained": false, "metrics": { "f1": 0.8593, "falseNegative": 79, "falsePositive": 13, "precision": 0.9558, "predictedPositive": 294, "recall": 0.7806, "total": 1485, "trueNegative": 1112, "truePositive": 281, "wilsonPrecisionLower95": 0.9258 }, "threshold": 0.98 }, "zh-Hans": { "abstained": false, "metrics": { "f1": 0.8166, "falseNegative": 104, "falsePositive": 11, "precision": 0.9588, "predictedPositive": 267, "recall": 0.7111, "total": 1485, "trueNegative": 1114, "truePositive": 256, "wilsonPrecisionLower95": 0.9277 }, "threshold": 0.96 } }, "metrics": { "f1": 0.8434, "falseNegative": 176, "falsePositive": 26, "precision": 0.9544, "predictedPositive": 570, "recall": 0.7556, "total": 2970, "trueNegative": 2224, "truePositive": 544, "wilsonPrecisionLower95": 0.934 }, "threshold": 0.965 } }, "selectionObjective": 0.8669, "trainingPolicy": "No user clipboard data. Official partial labels plus deterministic label-preserving augmentation and three-round high-confidence self-training consensus. Frozen holdouts never enter training or threshold calibration." }