Cursor Agent
14832 lines
402 KiB
JSON
14832 lines
402 KiB
JSON
[
|
|
{
|
|
"configuration": {
|
|
"alpha": 2e-05,
|
|
"augmentation": "none",
|
|
"char_max": 5,
|
|
"char_min": 2,
|
|
"external_weight": 0.5700000000000001,
|
|
"hard_example_weight": 1.0,
|
|
"l1_ratio": 0.0,
|
|
"max_char_features": 45000,
|
|
"max_word_features": 22000,
|
|
"min_df": 2,
|
|
"pseudo_weight": 0.16499999999999998,
|
|
"round": 1,
|
|
"word_max": 2
|
|
},
|
|
"featureCount": 56451,
|
|
"pseudoLabelCounts": {},
|
|
"round": 1,
|
|
"selectionObjective": 0.8669,
|
|
"syntheticRegression": {
|
|
"intentsMeeting95Precision": 6,
|
|
"macroF1": 0.8088,
|
|
"macroPrecision": 0.9661,
|
|
"macroRecall": 0.7541,
|
|
"minimumPrecision": 0.8589,
|
|
"perIntent": {
|
|
"blessing": {
|
|
"f1": 0.125,
|
|
"falseNegative": 84,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 6,
|
|
"recall": 0.0667,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 6,
|
|
"wilsonPrecisionLower95": 0.6097
|
|
},
|
|
"complaint": {
|
|
"f1": 0.9773,
|
|
"falseNegative": 16,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 344,
|
|
"recall": 0.9556,
|
|
"total": 2970,
|
|
"trueNegative": 2610,
|
|
"truePositive": 344,
|
|
"wilsonPrecisionLower95": 0.989
|
|
},
|
|
"confirmationDecision": {
|
|
"f1": 0.8163,
|
|
"falseNegative": 40,
|
|
"falsePositive": 23,
|
|
"precision": 0.8589,
|
|
"predictedPositive": 163,
|
|
"recall": 0.7778,
|
|
"total": 2970,
|
|
"trueNegative": 2767,
|
|
"truePositive": 140,
|
|
"wilsonPrecisionLower95": 0.7972
|
|
},
|
|
"followUpReminder": {
|
|
"f1": 0.8705,
|
|
"falseNegative": 81,
|
|
"falsePositive": 2,
|
|
"precision": 0.9929,
|
|
"predictedPositive": 281,
|
|
"recall": 0.775,
|
|
"total": 2970,
|
|
"trueNegative": 2608,
|
|
"truePositive": 279,
|
|
"wilsonPrecisionLower95": 0.9744
|
|
},
|
|
"invitation": {
|
|
"f1": 0.8545,
|
|
"falseNegative": 42,
|
|
"falsePositive": 5,
|
|
"precision": 0.965,
|
|
"predictedPositive": 143,
|
|
"recall": 0.7667,
|
|
"total": 2970,
|
|
"trueNegative": 2785,
|
|
"truePositive": 138,
|
|
"wilsonPrecisionLower95": 0.9208
|
|
},
|
|
"question": {
|
|
"f1": 0.9092,
|
|
"falseNegative": 104,
|
|
"falsePositive": 1,
|
|
"precision": 0.9981,
|
|
"predictedPositive": 527,
|
|
"recall": 0.8349,
|
|
"total": 2970,
|
|
"trueNegative": 2339,
|
|
"truePositive": 526,
|
|
"wilsonPrecisionLower95": 0.9893
|
|
},
|
|
"replyableMessage": {
|
|
"f1": 0.9465,
|
|
"falseNegative": 156,
|
|
"falsePositive": 40,
|
|
"precision": 0.9775,
|
|
"predictedPositive": 1774,
|
|
"recall": 0.9175,
|
|
"total": 2970,
|
|
"trueNegative": 1040,
|
|
"truePositive": 1734,
|
|
"wilsonPrecisionLower95": 0.9694
|
|
},
|
|
"scheduleNegotiation": {
|
|
"f1": 0.9727,
|
|
"falseNegative": 1,
|
|
"falsePositive": 4,
|
|
"precision": 0.957,
|
|
"predictedPositive": 93,
|
|
"recall": 0.9889,
|
|
"total": 2970,
|
|
"trueNegative": 2876,
|
|
"truePositive": 89,
|
|
"wilsonPrecisionLower95": 0.8946
|
|
},
|
|
"task": {
|
|
"f1": 0.8073,
|
|
"falseNegative": 213,
|
|
"falsePositive": 29,
|
|
"precision": 0.9459,
|
|
"predictedPositive": 536,
|
|
"recall": 0.7042,
|
|
"total": 2970,
|
|
"trueNegative": 2221,
|
|
"truePositive": 507,
|
|
"wilsonPrecisionLower95": 0.9234
|
|
}
|
|
},
|
|
"recordCount": 2970
|
|
},
|
|
"thresholds": {
|
|
"blessing": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9318,
|
|
"falseNegative": 4,
|
|
"falsePositive": 2,
|
|
"precision": 0.9535,
|
|
"predictedPositive": 43,
|
|
"recall": 0.9111,
|
|
"total": 1485,
|
|
"trueNegative": 1438,
|
|
"truePositive": 41,
|
|
"wilsonPrecisionLower95": 0.8454
|
|
},
|
|
"threshold": 0.915
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.7568,
|
|
"falseNegative": 17,
|
|
"falsePositive": 1,
|
|
"precision": 0.9655,
|
|
"predictedPositive": 29,
|
|
"recall": 0.6222,
|
|
"total": 1485,
|
|
"trueNegative": 1439,
|
|
"truePositive": 28,
|
|
"wilsonPrecisionLower95": 0.8282
|
|
},
|
|
"threshold": 0.9
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.8375,
|
|
"falseNegative": 23,
|
|
"falsePositive": 3,
|
|
"precision": 0.9571,
|
|
"predictedPositive": 70,
|
|
"recall": 0.7444,
|
|
"total": 2970,
|
|
"trueNegative": 2877,
|
|
"truePositive": 67,
|
|
"wilsonPrecisionLower95": 0.8814
|
|
},
|
|
"threshold": 0.915
|
|
},
|
|
"complaint": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 1.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 180,
|
|
"recall": 1.0,
|
|
"total": 1485,
|
|
"trueNegative": 1305,
|
|
"truePositive": 180,
|
|
"wilsonPrecisionLower95": 0.9791
|
|
},
|
|
"threshold": 0.5
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9526,
|
|
"falseNegative": 9,
|
|
"falsePositive": 8,
|
|
"precision": 0.9553,
|
|
"predictedPositive": 179,
|
|
"recall": 0.95,
|
|
"total": 1485,
|
|
"trueNegative": 1297,
|
|
"truePositive": 171,
|
|
"wilsonPrecisionLower95": 0.9143
|
|
},
|
|
"threshold": 0.54
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.9764,
|
|
"falseNegative": 9,
|
|
"falsePositive": 8,
|
|
"precision": 0.9777,
|
|
"predictedPositive": 359,
|
|
"recall": 0.975,
|
|
"total": 2970,
|
|
"trueNegative": 2602,
|
|
"truePositive": 351,
|
|
"wilsonPrecisionLower95": 0.9567
|
|
},
|
|
"threshold": 0.54
|
|
},
|
|
"confirmationDecision": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8848,
|
|
"falseNegative": 17,
|
|
"falsePositive": 2,
|
|
"precision": 0.9733,
|
|
"predictedPositive": 75,
|
|
"recall": 0.8111,
|
|
"total": 1485,
|
|
"trueNegative": 1393,
|
|
"truePositive": 73,
|
|
"wilsonPrecisionLower95": 0.9079
|
|
},
|
|
"threshold": 0.5
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.7947,
|
|
"falseNegative": 30,
|
|
"falsePositive": 1,
|
|
"precision": 0.9836,
|
|
"predictedPositive": 61,
|
|
"recall": 0.6667,
|
|
"total": 1485,
|
|
"trueNegative": 1394,
|
|
"truePositive": 60,
|
|
"wilsonPrecisionLower95": 0.9128
|
|
},
|
|
"threshold": 0.5
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.8418,
|
|
"falseNegative": 47,
|
|
"falsePositive": 3,
|
|
"precision": 0.9779,
|
|
"predictedPositive": 136,
|
|
"recall": 0.7389,
|
|
"total": 2970,
|
|
"trueNegative": 2787,
|
|
"truePositive": 133,
|
|
"wilsonPrecisionLower95": 0.9372
|
|
},
|
|
"threshold": 0.5
|
|
},
|
|
"followUpReminder": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8554,
|
|
"falseNegative": 41,
|
|
"falsePositive": 6,
|
|
"precision": 0.9586,
|
|
"predictedPositive": 145,
|
|
"recall": 0.7722,
|
|
"total": 1485,
|
|
"trueNegative": 1299,
|
|
"truePositive": 139,
|
|
"wilsonPrecisionLower95": 0.9127
|
|
},
|
|
"threshold": 0.83
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8354,
|
|
"falseNegative": 48,
|
|
"falsePositive": 4,
|
|
"precision": 0.9706,
|
|
"predictedPositive": 136,
|
|
"recall": 0.7333,
|
|
"total": 1485,
|
|
"trueNegative": 1301,
|
|
"truePositive": 132,
|
|
"wilsonPrecisionLower95": 0.9268
|
|
},
|
|
"threshold": 0.505
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.7994,
|
|
"falseNegative": 113,
|
|
"falsePositive": 11,
|
|
"precision": 0.9574,
|
|
"predictedPositive": 258,
|
|
"recall": 0.6861,
|
|
"total": 2970,
|
|
"trueNegative": 2599,
|
|
"truePositive": 247,
|
|
"wilsonPrecisionLower95": 0.9253
|
|
},
|
|
"threshold": 0.815
|
|
},
|
|
"invitation": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8302,
|
|
"falseNegative": 24,
|
|
"falsePositive": 3,
|
|
"precision": 0.9565,
|
|
"predictedPositive": 69,
|
|
"recall": 0.7333,
|
|
"total": 1485,
|
|
"trueNegative": 1392,
|
|
"truePositive": 66,
|
|
"wilsonPrecisionLower95": 0.8798
|
|
},
|
|
"threshold": 0.935
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.6765,
|
|
"falseNegative": 44,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 46,
|
|
"recall": 0.5111,
|
|
"total": 1485,
|
|
"trueNegative": 1395,
|
|
"truePositive": 46,
|
|
"wilsonPrecisionLower95": 0.9229
|
|
},
|
|
"threshold": 0.505
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.5476,
|
|
"falseNegative": 111,
|
|
"falsePositive": 3,
|
|
"precision": 0.9583,
|
|
"predictedPositive": 72,
|
|
"recall": 0.3833,
|
|
"total": 2970,
|
|
"trueNegative": 2787,
|
|
"truePositive": 69,
|
|
"wilsonPrecisionLower95": 0.8845
|
|
},
|
|
"threshold": 0.935
|
|
},
|
|
"question": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8445,
|
|
"falseNegative": 76,
|
|
"falsePositive": 12,
|
|
"precision": 0.9522,
|
|
"predictedPositive": 251,
|
|
"recall": 0.7587,
|
|
"total": 1485,
|
|
"trueNegative": 1158,
|
|
"truePositive": 239,
|
|
"wilsonPrecisionLower95": 0.9183
|
|
},
|
|
"threshold": 0.725
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8536,
|
|
"falseNegative": 76,
|
|
"falsePositive": 6,
|
|
"precision": 0.9755,
|
|
"predictedPositive": 245,
|
|
"recall": 0.7587,
|
|
"total": 1485,
|
|
"trueNegative": 1164,
|
|
"truePositive": 239,
|
|
"wilsonPrecisionLower95": 0.9476
|
|
},
|
|
"threshold": 0.5
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.8729,
|
|
"falseNegative": 122,
|
|
"falsePositive": 26,
|
|
"precision": 0.9513,
|
|
"predictedPositive": 534,
|
|
"recall": 0.8063,
|
|
"total": 2970,
|
|
"trueNegative": 2314,
|
|
"truePositive": 508,
|
|
"wilsonPrecisionLower95": 0.9296
|
|
},
|
|
"threshold": 0.515
|
|
},
|
|
"replyableMessage": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.948,
|
|
"falseNegative": 52,
|
|
"falsePositive": 46,
|
|
"precision": 0.951,
|
|
"predictedPositive": 939,
|
|
"recall": 0.945,
|
|
"total": 1485,
|
|
"trueNegative": 494,
|
|
"truePositive": 893,
|
|
"wilsonPrecisionLower95": 0.9353
|
|
},
|
|
"threshold": 0.7
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9424,
|
|
"falseNegative": 62,
|
|
"falsePositive": 46,
|
|
"precision": 0.9505,
|
|
"predictedPositive": 929,
|
|
"recall": 0.9344,
|
|
"total": 1485,
|
|
"trueNegative": 494,
|
|
"truePositive": 883,
|
|
"wilsonPrecisionLower95": 0.9346
|
|
},
|
|
"threshold": 0.695
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.9454,
|
|
"falseNegative": 115,
|
|
"falsePositive": 90,
|
|
"precision": 0.9517,
|
|
"predictedPositive": 1865,
|
|
"recall": 0.9392,
|
|
"total": 2970,
|
|
"trueNegative": 990,
|
|
"truePositive": 1775,
|
|
"wilsonPrecisionLower95": 0.9411
|
|
},
|
|
"threshold": 0.7
|
|
},
|
|
"scheduleNegotiation": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 1.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 45,
|
|
"recall": 1.0,
|
|
"total": 1485,
|
|
"trueNegative": 1440,
|
|
"truePositive": 45,
|
|
"wilsonPrecisionLower95": 0.9213
|
|
},
|
|
"threshold": 0.5
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.967,
|
|
"falseNegative": 1,
|
|
"falsePositive": 2,
|
|
"precision": 0.9565,
|
|
"predictedPositive": 46,
|
|
"recall": 0.9778,
|
|
"total": 1485,
|
|
"trueNegative": 1438,
|
|
"truePositive": 44,
|
|
"wilsonPrecisionLower95": 0.8547
|
|
},
|
|
"threshold": 0.93
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.978,
|
|
"falseNegative": 1,
|
|
"falsePositive": 3,
|
|
"precision": 0.9674,
|
|
"predictedPositive": 92,
|
|
"recall": 0.9889,
|
|
"total": 2970,
|
|
"trueNegative": 2877,
|
|
"truePositive": 89,
|
|
"wilsonPrecisionLower95": 0.9085
|
|
},
|
|
"threshold": 0.91
|
|
},
|
|
"task": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8403,
|
|
"falseNegative": 89,
|
|
"falsePositive": 14,
|
|
"precision": 0.9509,
|
|
"predictedPositive": 285,
|
|
"recall": 0.7528,
|
|
"total": 1485,
|
|
"trueNegative": 1111,
|
|
"truePositive": 271,
|
|
"wilsonPrecisionLower95": 0.9192
|
|
},
|
|
"threshold": 0.835
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8278,
|
|
"falseNegative": 98,
|
|
"falsePositive": 11,
|
|
"precision": 0.9597,
|
|
"predictedPositive": 273,
|
|
"recall": 0.7278,
|
|
"total": 1485,
|
|
"trueNegative": 1114,
|
|
"truePositive": 262,
|
|
"wilsonPrecisionLower95": 0.9293
|
|
},
|
|
"threshold": 0.745
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.8637,
|
|
"falseNegative": 153,
|
|
"falsePositive": 26,
|
|
"precision": 0.9562,
|
|
"predictedPositive": 593,
|
|
"recall": 0.7875,
|
|
"total": 2970,
|
|
"trueNegative": 2224,
|
|
"truePositive": 567,
|
|
"wilsonPrecisionLower95": 0.9365
|
|
},
|
|
"threshold": 0.755
|
|
}
|
|
},
|
|
"trainingRecords": 15345,
|
|
"validation": {
|
|
"intentsMeeting95Precision": 9,
|
|
"macroF1": 0.822,
|
|
"macroPrecision": 0.9713,
|
|
"macroRecall": 0.7487,
|
|
"minimumPrecision": 0.9507,
|
|
"perIntent": {
|
|
"blessing": {
|
|
"f1": 0.3636,
|
|
"falseNegative": 70,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 20,
|
|
"recall": 0.2222,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 20,
|
|
"wilsonPrecisionLower95": 0.8389
|
|
},
|
|
"complaint": {
|
|
"f1": 0.9764,
|
|
"falseNegative": 9,
|
|
"falsePositive": 8,
|
|
"precision": 0.9777,
|
|
"predictedPositive": 359,
|
|
"recall": 0.975,
|
|
"total": 2970,
|
|
"trueNegative": 2602,
|
|
"truePositive": 351,
|
|
"wilsonPrecisionLower95": 0.9567
|
|
},
|
|
"confirmationDecision": {
|
|
"f1": 0.8418,
|
|
"falseNegative": 47,
|
|
"falsePositive": 3,
|
|
"precision": 0.9779,
|
|
"predictedPositive": 136,
|
|
"recall": 0.7389,
|
|
"total": 2970,
|
|
"trueNegative": 2787,
|
|
"truePositive": 133,
|
|
"wilsonPrecisionLower95": 0.9372
|
|
},
|
|
"followUpReminder": {
|
|
"f1": 0.8456,
|
|
"falseNegative": 89,
|
|
"falsePositive": 10,
|
|
"precision": 0.9644,
|
|
"predictedPositive": 281,
|
|
"recall": 0.7528,
|
|
"total": 2970,
|
|
"trueNegative": 2600,
|
|
"truePositive": 271,
|
|
"wilsonPrecisionLower95": 0.9357
|
|
},
|
|
"invitation": {
|
|
"f1": 0.7593,
|
|
"falseNegative": 68,
|
|
"falsePositive": 3,
|
|
"precision": 0.9739,
|
|
"predictedPositive": 115,
|
|
"recall": 0.6222,
|
|
"total": 2970,
|
|
"trueNegative": 2787,
|
|
"truePositive": 112,
|
|
"wilsonPrecisionLower95": 0.9261
|
|
},
|
|
"question": {
|
|
"f1": 0.849,
|
|
"falseNegative": 152,
|
|
"falsePositive": 18,
|
|
"precision": 0.9637,
|
|
"predictedPositive": 496,
|
|
"recall": 0.7587,
|
|
"total": 2970,
|
|
"trueNegative": 2322,
|
|
"truePositive": 478,
|
|
"wilsonPrecisionLower95": 0.9434
|
|
},
|
|
"replyableMessage": {
|
|
"f1": 0.9452,
|
|
"falseNegative": 114,
|
|
"falsePositive": 92,
|
|
"precision": 0.9507,
|
|
"predictedPositive": 1868,
|
|
"recall": 0.9397,
|
|
"total": 2970,
|
|
"trueNegative": 988,
|
|
"truePositive": 1776,
|
|
"wilsonPrecisionLower95": 0.94
|
|
},
|
|
"scheduleNegotiation": {
|
|
"f1": 0.9834,
|
|
"falseNegative": 1,
|
|
"falsePositive": 2,
|
|
"precision": 0.978,
|
|
"predictedPositive": 91,
|
|
"recall": 0.9889,
|
|
"total": 2970,
|
|
"trueNegative": 2878,
|
|
"truePositive": 89,
|
|
"wilsonPrecisionLower95": 0.9234
|
|
},
|
|
"task": {
|
|
"f1": 0.8341,
|
|
"falseNegative": 187,
|
|
"falsePositive": 25,
|
|
"precision": 0.9552,
|
|
"predictedPositive": 558,
|
|
"recall": 0.7403,
|
|
"total": 2970,
|
|
"trueNegative": 2225,
|
|
"truePositive": 533,
|
|
"wilsonPrecisionLower95": 0.9347
|
|
}
|
|
},
|
|
"recordCount": 2970
|
|
}
|
|
},
|
|
{
|
|
"configuration": {
|
|
"alpha": 1.5e-05,
|
|
"augmentation": "none",
|
|
"char_max": 5,
|
|
"char_min": 3,
|
|
"external_weight": 0.5900000000000001,
|
|
"hard_example_weight": 1.0,
|
|
"l1_ratio": 0.0,
|
|
"max_char_features": 50000,
|
|
"max_word_features": 25000,
|
|
"min_df": 2,
|
|
"pseudo_weight": 0.18,
|
|
"round": 2,
|
|
"word_max": 2
|
|
},
|
|
"featureCount": 61451,
|
|
"pseudoLabelCounts": {},
|
|
"round": 2,
|
|
"selectionObjective": 0.8228,
|
|
"syntheticRegression": {
|
|
"intentsMeeting95Precision": 7,
|
|
"macroF1": 0.751,
|
|
"macroPrecision": 0.9667,
|
|
"macroRecall": 0.6602,
|
|
"minimumPrecision": 0.8606,
|
|
"perIntent": {
|
|
"blessing": {
|
|
"f1": 0.2178,
|
|
"falseNegative": 79,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 11,
|
|
"recall": 0.1222,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 11,
|
|
"wilsonPrecisionLower95": 0.7412
|
|
},
|
|
"complaint": {
|
|
"f1": 0.9595,
|
|
"falseNegative": 28,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 332,
|
|
"recall": 0.9222,
|
|
"total": 2970,
|
|
"trueNegative": 2610,
|
|
"truePositive": 332,
|
|
"wilsonPrecisionLower95": 0.9886
|
|
},
|
|
"confirmationDecision": {
|
|
"f1": 0.8232,
|
|
"falseNegative": 38,
|
|
"falsePositive": 23,
|
|
"precision": 0.8606,
|
|
"predictedPositive": 165,
|
|
"recall": 0.7889,
|
|
"total": 2970,
|
|
"trueNegative": 2767,
|
|
"truePositive": 142,
|
|
"wilsonPrecisionLower95": 0.7995
|
|
},
|
|
"followUpReminder": {
|
|
"f1": 0.7565,
|
|
"falseNegative": 141,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 219,
|
|
"recall": 0.6083,
|
|
"total": 2970,
|
|
"trueNegative": 2610,
|
|
"truePositive": 219,
|
|
"wilsonPrecisionLower95": 0.9828
|
|
},
|
|
"invitation": {
|
|
"f1": 0.6394,
|
|
"falseNegative": 94,
|
|
"falsePositive": 3,
|
|
"precision": 0.9663,
|
|
"predictedPositive": 89,
|
|
"recall": 0.4778,
|
|
"total": 2970,
|
|
"trueNegative": 2787,
|
|
"truePositive": 86,
|
|
"wilsonPrecisionLower95": 0.9055
|
|
},
|
|
"question": {
|
|
"f1": 0.86,
|
|
"falseNegative": 151,
|
|
"falsePositive": 5,
|
|
"precision": 0.9897,
|
|
"predictedPositive": 484,
|
|
"recall": 0.7603,
|
|
"total": 2970,
|
|
"trueNegative": 2335,
|
|
"truePositive": 479,
|
|
"wilsonPrecisionLower95": 0.976
|
|
},
|
|
"replyableMessage": {
|
|
"f1": 0.9038,
|
|
"falseNegative": 293,
|
|
"falsePositive": 47,
|
|
"precision": 0.9714,
|
|
"predictedPositive": 1644,
|
|
"recall": 0.845,
|
|
"total": 2970,
|
|
"trueNegative": 1033,
|
|
"truePositive": 1597,
|
|
"wilsonPrecisionLower95": 0.9622
|
|
},
|
|
"scheduleNegotiation": {
|
|
"f1": 0.9425,
|
|
"falseNegative": 8,
|
|
"falsePositive": 2,
|
|
"precision": 0.9762,
|
|
"predictedPositive": 84,
|
|
"recall": 0.9111,
|
|
"total": 2970,
|
|
"trueNegative": 2878,
|
|
"truePositive": 82,
|
|
"wilsonPrecisionLower95": 0.9173
|
|
},
|
|
"task": {
|
|
"f1": 0.6564,
|
|
"falseNegative": 356,
|
|
"falsePositive": 25,
|
|
"precision": 0.9357,
|
|
"predictedPositive": 389,
|
|
"recall": 0.5056,
|
|
"total": 2970,
|
|
"trueNegative": 2225,
|
|
"truePositive": 364,
|
|
"wilsonPrecisionLower95": 0.9069
|
|
}
|
|
},
|
|
"recordCount": 2970
|
|
},
|
|
"thresholds": {
|
|
"blessing": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9048,
|
|
"falseNegative": 7,
|
|
"falsePositive": 1,
|
|
"precision": 0.9744,
|
|
"predictedPositive": 39,
|
|
"recall": 0.8444,
|
|
"total": 1485,
|
|
"trueNegative": 1439,
|
|
"truePositive": 38,
|
|
"wilsonPrecisionLower95": 0.8682
|
|
},
|
|
"threshold": 0.59
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"complaint": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 1.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 180,
|
|
"recall": 1.0,
|
|
"total": 1485,
|
|
"trueNegative": 1305,
|
|
"truePositive": 180,
|
|
"wilsonPrecisionLower95": 0.9791
|
|
},
|
|
"threshold": 0.5
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9226,
|
|
"falseNegative": 19,
|
|
"falsePositive": 8,
|
|
"precision": 0.9527,
|
|
"predictedPositive": 169,
|
|
"recall": 0.8944,
|
|
"total": 1485,
|
|
"trueNegative": 1297,
|
|
"truePositive": 161,
|
|
"wilsonPrecisionLower95": 0.9094
|
|
},
|
|
"threshold": 0.555
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.9649,
|
|
"falseNegative": 16,
|
|
"falsePositive": 9,
|
|
"precision": 0.9745,
|
|
"predictedPositive": 353,
|
|
"recall": 0.9556,
|
|
"total": 2970,
|
|
"trueNegative": 2601,
|
|
"truePositive": 344,
|
|
"wilsonPrecisionLower95": 0.9523
|
|
},
|
|
"threshold": 0.5
|
|
},
|
|
"confirmationDecision": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.954,
|
|
"falseNegative": 7,
|
|
"falsePositive": 1,
|
|
"precision": 0.9881,
|
|
"predictedPositive": 84,
|
|
"recall": 0.9222,
|
|
"total": 1485,
|
|
"trueNegative": 1394,
|
|
"truePositive": 83,
|
|
"wilsonPrecisionLower95": 0.9356
|
|
},
|
|
"threshold": 0.5
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.7183,
|
|
"falseNegative": 39,
|
|
"falsePositive": 1,
|
|
"precision": 0.9808,
|
|
"predictedPositive": 52,
|
|
"recall": 0.5667,
|
|
"total": 1485,
|
|
"trueNegative": 1394,
|
|
"truePositive": 51,
|
|
"wilsonPrecisionLower95": 0.8988
|
|
},
|
|
"threshold": 0.5
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.8481,
|
|
"falseNegative": 46,
|
|
"falsePositive": 2,
|
|
"precision": 0.9853,
|
|
"predictedPositive": 136,
|
|
"recall": 0.7444,
|
|
"total": 2970,
|
|
"trueNegative": 2788,
|
|
"truePositive": 134,
|
|
"wilsonPrecisionLower95": 0.948
|
|
},
|
|
"threshold": 0.5
|
|
},
|
|
"followUpReminder": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8328,
|
|
"falseNegative": 48,
|
|
"falsePositive": 5,
|
|
"precision": 0.9635,
|
|
"predictedPositive": 137,
|
|
"recall": 0.7333,
|
|
"total": 1485,
|
|
"trueNegative": 1300,
|
|
"truePositive": 132,
|
|
"wilsonPrecisionLower95": 0.9174
|
|
},
|
|
"threshold": 0.91
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.6879,
|
|
"falseNegative": 83,
|
|
"falsePositive": 5,
|
|
"precision": 0.951,
|
|
"predictedPositive": 102,
|
|
"recall": 0.5389,
|
|
"total": 1485,
|
|
"trueNegative": 1300,
|
|
"truePositive": 97,
|
|
"wilsonPrecisionLower95": 0.8903
|
|
},
|
|
"threshold": 0.765
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.6243,
|
|
"falseNegative": 193,
|
|
"falsePositive": 8,
|
|
"precision": 0.9543,
|
|
"predictedPositive": 175,
|
|
"recall": 0.4639,
|
|
"total": 2970,
|
|
"trueNegative": 2602,
|
|
"truePositive": 167,
|
|
"wilsonPrecisionLower95": 0.9124
|
|
},
|
|
"threshold": 0.905
|
|
},
|
|
"invitation": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9059,
|
|
"falseNegative": 13,
|
|
"falsePositive": 3,
|
|
"precision": 0.9625,
|
|
"predictedPositive": 80,
|
|
"recall": 0.8556,
|
|
"total": 1485,
|
|
"trueNegative": 1392,
|
|
"truePositive": 77,
|
|
"wilsonPrecisionLower95": 0.8955
|
|
},
|
|
"threshold": 0.86
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.6113,
|
|
"falseNegative": 99,
|
|
"falsePositive": 4,
|
|
"precision": 0.9529,
|
|
"predictedPositive": 85,
|
|
"recall": 0.45,
|
|
"total": 2970,
|
|
"trueNegative": 2786,
|
|
"truePositive": 81,
|
|
"wilsonPrecisionLower95": 0.8852
|
|
},
|
|
"threshold": 0.855
|
|
},
|
|
"question": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8505,
|
|
"falseNegative": 76,
|
|
"falsePositive": 8,
|
|
"precision": 0.9676,
|
|
"predictedPositive": 247,
|
|
"recall": 0.7587,
|
|
"total": 1485,
|
|
"trueNegative": 1162,
|
|
"truePositive": 239,
|
|
"wilsonPrecisionLower95": 0.9374
|
|
},
|
|
"threshold": 0.705
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.7308,
|
|
"falseNegative": 129,
|
|
"falsePositive": 8,
|
|
"precision": 0.9588,
|
|
"predictedPositive": 194,
|
|
"recall": 0.5905,
|
|
"total": 1485,
|
|
"trueNegative": 1162,
|
|
"truePositive": 186,
|
|
"wilsonPrecisionLower95": 0.9208
|
|
},
|
|
"threshold": 0.5
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.7813,
|
|
"falseNegative": 212,
|
|
"falsePositive": 22,
|
|
"precision": 0.95,
|
|
"predictedPositive": 440,
|
|
"recall": 0.6635,
|
|
"total": 2970,
|
|
"trueNegative": 2318,
|
|
"truePositive": 418,
|
|
"wilsonPrecisionLower95": 0.9255
|
|
},
|
|
"threshold": 0.57
|
|
},
|
|
"replyableMessage": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9629,
|
|
"falseNegative": 23,
|
|
"falsePositive": 48,
|
|
"precision": 0.9505,
|
|
"predictedPositive": 970,
|
|
"recall": 0.9757,
|
|
"total": 1485,
|
|
"trueNegative": 492,
|
|
"truePositive": 922,
|
|
"wilsonPrecisionLower95": 0.935
|
|
},
|
|
"threshold": 0.62
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8526,
|
|
"falseNegative": 216,
|
|
"falsePositive": 36,
|
|
"precision": 0.9529,
|
|
"predictedPositive": 765,
|
|
"recall": 0.7714,
|
|
"total": 1485,
|
|
"trueNegative": 504,
|
|
"truePositive": 729,
|
|
"wilsonPrecisionLower95": 0.9355
|
|
},
|
|
"threshold": 0.84
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.9224,
|
|
"falseNegative": 197,
|
|
"falsePositive": 88,
|
|
"precision": 0.9506,
|
|
"predictedPositive": 1781,
|
|
"recall": 0.8958,
|
|
"total": 2970,
|
|
"trueNegative": 992,
|
|
"truePositive": 1693,
|
|
"wilsonPrecisionLower95": 0.9395
|
|
},
|
|
"threshold": 0.775
|
|
},
|
|
"scheduleNegotiation": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9783,
|
|
"falseNegative": 0,
|
|
"falsePositive": 2,
|
|
"precision": 0.9574,
|
|
"predictedPositive": 47,
|
|
"recall": 1.0,
|
|
"total": 1485,
|
|
"trueNegative": 1438,
|
|
"truePositive": 45,
|
|
"wilsonPrecisionLower95": 0.8575
|
|
},
|
|
"threshold": 0.765
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9778,
|
|
"falseNegative": 1,
|
|
"falsePositive": 1,
|
|
"precision": 0.9778,
|
|
"predictedPositive": 45,
|
|
"recall": 0.9778,
|
|
"total": 1485,
|
|
"trueNegative": 1439,
|
|
"truePositive": 44,
|
|
"wilsonPrecisionLower95": 0.8843
|
|
},
|
|
"threshold": 0.855
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.978,
|
|
"falseNegative": 1,
|
|
"falsePositive": 3,
|
|
"precision": 0.9674,
|
|
"predictedPositive": 92,
|
|
"recall": 0.9889,
|
|
"total": 2970,
|
|
"trueNegative": 2877,
|
|
"truePositive": 89,
|
|
"wilsonPrecisionLower95": 0.9085
|
|
},
|
|
"threshold": 0.855
|
|
},
|
|
"task": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8541,
|
|
"falseNegative": 82,
|
|
"falsePositive": 13,
|
|
"precision": 0.9553,
|
|
"predictedPositive": 291,
|
|
"recall": 0.7722,
|
|
"total": 1485,
|
|
"trueNegative": 1112,
|
|
"truePositive": 278,
|
|
"wilsonPrecisionLower95": 0.9251
|
|
},
|
|
"threshold": 0.84
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.5418,
|
|
"falseNegative": 224,
|
|
"falsePositive": 6,
|
|
"precision": 0.9577,
|
|
"predictedPositive": 142,
|
|
"recall": 0.3778,
|
|
"total": 1485,
|
|
"trueNegative": 1119,
|
|
"truePositive": 136,
|
|
"wilsonPrecisionLower95": 0.9109
|
|
},
|
|
"threshold": 0.91
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.7246,
|
|
"falseNegative": 299,
|
|
"falsePositive": 21,
|
|
"precision": 0.9525,
|
|
"predictedPositive": 442,
|
|
"recall": 0.5847,
|
|
"total": 2970,
|
|
"trueNegative": 2229,
|
|
"truePositive": 421,
|
|
"wilsonPrecisionLower95": 0.9285
|
|
},
|
|
"threshold": 0.88
|
|
}
|
|
},
|
|
"trainingRecords": 15345,
|
|
"validation": {
|
|
"intentsMeeting95Precision": 9,
|
|
"macroF1": 0.7578,
|
|
"macroPrecision": 0.9691,
|
|
"macroRecall": 0.668,
|
|
"minimumPrecision": 0.9516,
|
|
"perIntent": {
|
|
"blessing": {
|
|
"f1": 0.2524,
|
|
"falseNegative": 77,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 13,
|
|
"recall": 0.1444,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 13,
|
|
"wilsonPrecisionLower95": 0.7719
|
|
},
|
|
"complaint": {
|
|
"f1": 0.9619,
|
|
"falseNegative": 19,
|
|
"falsePositive": 8,
|
|
"precision": 0.9771,
|
|
"predictedPositive": 349,
|
|
"recall": 0.9472,
|
|
"total": 2970,
|
|
"trueNegative": 2602,
|
|
"truePositive": 341,
|
|
"wilsonPrecisionLower95": 0.9554
|
|
},
|
|
"confirmationDecision": {
|
|
"f1": 0.8481,
|
|
"falseNegative": 46,
|
|
"falsePositive": 2,
|
|
"precision": 0.9853,
|
|
"predictedPositive": 136,
|
|
"recall": 0.7444,
|
|
"total": 2970,
|
|
"trueNegative": 2788,
|
|
"truePositive": 134,
|
|
"wilsonPrecisionLower95": 0.948
|
|
},
|
|
"followUpReminder": {
|
|
"f1": 0.7646,
|
|
"falseNegative": 131,
|
|
"falsePositive": 10,
|
|
"precision": 0.9582,
|
|
"predictedPositive": 239,
|
|
"recall": 0.6361,
|
|
"total": 2970,
|
|
"trueNegative": 2600,
|
|
"truePositive": 229,
|
|
"wilsonPrecisionLower95": 0.9247
|
|
},
|
|
"invitation": {
|
|
"f1": 0.5923,
|
|
"falseNegative": 103,
|
|
"falsePositive": 3,
|
|
"precision": 0.9625,
|
|
"predictedPositive": 80,
|
|
"recall": 0.4278,
|
|
"total": 2970,
|
|
"trueNegative": 2787,
|
|
"truePositive": 77,
|
|
"wilsonPrecisionLower95": 0.8955
|
|
},
|
|
"question": {
|
|
"f1": 0.7937,
|
|
"falseNegative": 205,
|
|
"falsePositive": 16,
|
|
"precision": 0.9637,
|
|
"predictedPositive": 441,
|
|
"recall": 0.6746,
|
|
"total": 2970,
|
|
"trueNegative": 2324,
|
|
"truePositive": 425,
|
|
"wilsonPrecisionLower95": 0.9419
|
|
},
|
|
"replyableMessage": {
|
|
"f1": 0.9109,
|
|
"falseNegative": 239,
|
|
"falsePositive": 84,
|
|
"precision": 0.9516,
|
|
"predictedPositive": 1735,
|
|
"recall": 0.8735,
|
|
"total": 2970,
|
|
"trueNegative": 996,
|
|
"truePositive": 1651,
|
|
"wilsonPrecisionLower95": 0.9404
|
|
},
|
|
"scheduleNegotiation": {
|
|
"f1": 0.978,
|
|
"falseNegative": 1,
|
|
"falsePositive": 3,
|
|
"precision": 0.9674,
|
|
"predictedPositive": 92,
|
|
"recall": 0.9889,
|
|
"total": 2970,
|
|
"trueNegative": 2877,
|
|
"truePositive": 89,
|
|
"wilsonPrecisionLower95": 0.9085
|
|
},
|
|
"task": {
|
|
"f1": 0.7181,
|
|
"falseNegative": 306,
|
|
"falsePositive": 19,
|
|
"precision": 0.9561,
|
|
"predictedPositive": 433,
|
|
"recall": 0.575,
|
|
"total": 2970,
|
|
"trueNegative": 2231,
|
|
"truePositive": 414,
|
|
"wilsonPrecisionLower95": 0.9325
|
|
}
|
|
},
|
|
"recordCount": 2970
|
|
}
|
|
},
|
|
{
|
|
"configuration": {
|
|
"alpha": 1e-05,
|
|
"augmentation": "none",
|
|
"char_max": 6,
|
|
"char_min": 2,
|
|
"external_weight": 0.6100000000000001,
|
|
"hard_example_weight": 1.2,
|
|
"l1_ratio": 0.05,
|
|
"max_char_features": 55000,
|
|
"max_word_features": 25000,
|
|
"min_df": 2,
|
|
"pseudo_weight": 0.195,
|
|
"round": 3,
|
|
"word_max": 2
|
|
},
|
|
"featureCount": 66451,
|
|
"pseudoLabelCounts": {},
|
|
"round": 3,
|
|
"selectionObjective": 0.8348,
|
|
"syntheticRegression": {
|
|
"intentsMeeting95Precision": 4,
|
|
"macroF1": 0.7343,
|
|
"macroPrecision": 0.9403,
|
|
"macroRecall": 0.6672,
|
|
"minimumPrecision": 0.809,
|
|
"perIntent": {
|
|
"blessing": {
|
|
"f1": 0.0645,
|
|
"falseNegative": 87,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 3,
|
|
"recall": 0.0333,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 3,
|
|
"wilsonPrecisionLower95": 0.4385
|
|
},
|
|
"complaint": {
|
|
"f1": 0.9816,
|
|
"falseNegative": 13,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 347,
|
|
"recall": 0.9639,
|
|
"total": 2970,
|
|
"trueNegative": 2610,
|
|
"truePositive": 347,
|
|
"wilsonPrecisionLower95": 0.9891
|
|
},
|
|
"confirmationDecision": {
|
|
"f1": 0.8045,
|
|
"falseNegative": 36,
|
|
"falsePositive": 34,
|
|
"precision": 0.809,
|
|
"predictedPositive": 178,
|
|
"recall": 0.8,
|
|
"total": 2970,
|
|
"trueNegative": 2756,
|
|
"truePositive": 144,
|
|
"wilsonPrecisionLower95": 0.745
|
|
},
|
|
"followUpReminder": {
|
|
"f1": 0.7967,
|
|
"falseNegative": 121,
|
|
"falsePositive": 1,
|
|
"precision": 0.9958,
|
|
"predictedPositive": 240,
|
|
"recall": 0.6639,
|
|
"total": 2970,
|
|
"trueNegative": 2609,
|
|
"truePositive": 239,
|
|
"wilsonPrecisionLower95": 0.9768
|
|
},
|
|
"invitation": {
|
|
"f1": 0.5985,
|
|
"falseNegative": 101,
|
|
"falsePositive": 5,
|
|
"precision": 0.9405,
|
|
"predictedPositive": 84,
|
|
"recall": 0.4389,
|
|
"total": 2970,
|
|
"trueNegative": 2785,
|
|
"truePositive": 79,
|
|
"wilsonPrecisionLower95": 0.8681
|
|
},
|
|
"question": {
|
|
"f1": 0.8643,
|
|
"falseNegative": 149,
|
|
"falsePositive": 2,
|
|
"precision": 0.9959,
|
|
"predictedPositive": 483,
|
|
"recall": 0.7635,
|
|
"total": 2970,
|
|
"trueNegative": 2338,
|
|
"truePositive": 481,
|
|
"wilsonPrecisionLower95": 0.985
|
|
},
|
|
"replyableMessage": {
|
|
"f1": 0.931,
|
|
"falseNegative": 189,
|
|
"falsePositive": 63,
|
|
"precision": 0.9643,
|
|
"predictedPositive": 1764,
|
|
"recall": 0.9,
|
|
"total": 2970,
|
|
"trueNegative": 1017,
|
|
"truePositive": 1701,
|
|
"wilsonPrecisionLower95": 0.9546
|
|
},
|
|
"scheduleNegotiation": {
|
|
"f1": 0.8222,
|
|
"falseNegative": 16,
|
|
"falsePositive": 16,
|
|
"precision": 0.8222,
|
|
"predictedPositive": 90,
|
|
"recall": 0.8222,
|
|
"total": 2970,
|
|
"trueNegative": 2864,
|
|
"truePositive": 74,
|
|
"wilsonPrecisionLower95": 0.7306
|
|
},
|
|
"task": {
|
|
"f1": 0.7452,
|
|
"falseNegative": 274,
|
|
"falsePositive": 31,
|
|
"precision": 0.935,
|
|
"predictedPositive": 477,
|
|
"recall": 0.6194,
|
|
"total": 2970,
|
|
"trueNegative": 2219,
|
|
"truePositive": 446,
|
|
"wilsonPrecisionLower95": 0.9092
|
|
}
|
|
},
|
|
"recordCount": 2970
|
|
},
|
|
"thresholds": {
|
|
"blessing": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"complaint": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 1.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 180,
|
|
"recall": 1.0,
|
|
"total": 1485,
|
|
"trueNegative": 1305,
|
|
"truePositive": 180,
|
|
"wilsonPrecisionLower95": 0.9791
|
|
},
|
|
"threshold": 0.5
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9371,
|
|
"falseNegative": 16,
|
|
"falsePositive": 6,
|
|
"precision": 0.9647,
|
|
"predictedPositive": 170,
|
|
"recall": 0.9111,
|
|
"total": 1485,
|
|
"trueNegative": 1299,
|
|
"truePositive": 164,
|
|
"wilsonPrecisionLower95": 0.9251
|
|
},
|
|
"threshold": 0.5
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.969,
|
|
"falseNegative": 16,
|
|
"falsePositive": 6,
|
|
"precision": 0.9829,
|
|
"predictedPositive": 350,
|
|
"recall": 0.9556,
|
|
"total": 2970,
|
|
"trueNegative": 2604,
|
|
"truePositive": 344,
|
|
"wilsonPrecisionLower95": 0.9631
|
|
},
|
|
"threshold": 0.5
|
|
},
|
|
"confirmationDecision": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9727,
|
|
"falseNegative": 1,
|
|
"falsePositive": 4,
|
|
"precision": 0.957,
|
|
"predictedPositive": 93,
|
|
"recall": 0.9889,
|
|
"total": 1485,
|
|
"trueNegative": 1391,
|
|
"truePositive": 89,
|
|
"wilsonPrecisionLower95": 0.8946
|
|
},
|
|
"threshold": 0.585
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.7815,
|
|
"falseNegative": 31,
|
|
"falsePositive": 2,
|
|
"precision": 0.9672,
|
|
"predictedPositive": 61,
|
|
"recall": 0.6556,
|
|
"total": 1485,
|
|
"trueNegative": 1393,
|
|
"truePositive": 59,
|
|
"wilsonPrecisionLower95": 0.8881
|
|
},
|
|
"threshold": 0.5
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.8701,
|
|
"falseNegative": 36,
|
|
"falsePositive": 7,
|
|
"precision": 0.9536,
|
|
"predictedPositive": 151,
|
|
"recall": 0.8,
|
|
"total": 2970,
|
|
"trueNegative": 2783,
|
|
"truePositive": 144,
|
|
"wilsonPrecisionLower95": 0.9074
|
|
},
|
|
"threshold": 0.555
|
|
},
|
|
"followUpReminder": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8454,
|
|
"falseNegative": 46,
|
|
"falsePositive": 3,
|
|
"precision": 0.9781,
|
|
"predictedPositive": 137,
|
|
"recall": 0.7444,
|
|
"total": 1485,
|
|
"trueNegative": 1302,
|
|
"truePositive": 134,
|
|
"wilsonPrecisionLower95": 0.9376
|
|
},
|
|
"threshold": 0.97
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.7525,
|
|
"falseNegative": 69,
|
|
"falsePositive": 4,
|
|
"precision": 0.9652,
|
|
"predictedPositive": 115,
|
|
"recall": 0.6167,
|
|
"total": 1485,
|
|
"trueNegative": 1301,
|
|
"truePositive": 111,
|
|
"wilsonPrecisionLower95": 0.914
|
|
},
|
|
"threshold": 0.785
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.703,
|
|
"falseNegative": 160,
|
|
"falsePositive": 9,
|
|
"precision": 0.9569,
|
|
"predictedPositive": 209,
|
|
"recall": 0.5556,
|
|
"total": 2970,
|
|
"trueNegative": 2601,
|
|
"truePositive": 200,
|
|
"wilsonPrecisionLower95": 0.9202
|
|
},
|
|
"threshold": 0.955
|
|
},
|
|
"invitation": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.7974,
|
|
"falseNegative": 29,
|
|
"falsePositive": 2,
|
|
"precision": 0.9683,
|
|
"predictedPositive": 63,
|
|
"recall": 0.6778,
|
|
"total": 1485,
|
|
"trueNegative": 1393,
|
|
"truePositive": 61,
|
|
"wilsonPrecisionLower95": 0.8914
|
|
},
|
|
"threshold": 0.5
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"question": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8259,
|
|
"falseNegative": 85,
|
|
"falsePositive": 12,
|
|
"precision": 0.9504,
|
|
"predictedPositive": 242,
|
|
"recall": 0.7302,
|
|
"total": 1485,
|
|
"trueNegative": 1158,
|
|
"truePositive": 230,
|
|
"wilsonPrecisionLower95": 0.9153
|
|
},
|
|
"threshold": 0.865
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8194,
|
|
"falseNegative": 95,
|
|
"falsePositive": 2,
|
|
"precision": 0.991,
|
|
"predictedPositive": 222,
|
|
"recall": 0.6984,
|
|
"total": 1485,
|
|
"trueNegative": 1168,
|
|
"truePositive": 220,
|
|
"wilsonPrecisionLower95": 0.9678
|
|
},
|
|
"threshold": 0.5
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.7959,
|
|
"falseNegative": 199,
|
|
"falsePositive": 22,
|
|
"precision": 0.9514,
|
|
"predictedPositive": 453,
|
|
"recall": 0.6841,
|
|
"total": 2970,
|
|
"trueNegative": 2318,
|
|
"truePositive": 431,
|
|
"wilsonPrecisionLower95": 0.9276
|
|
},
|
|
"threshold": 0.74
|
|
},
|
|
"replyableMessage": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9496,
|
|
"falseNegative": 50,
|
|
"falsePositive": 45,
|
|
"precision": 0.9521,
|
|
"predictedPositive": 940,
|
|
"recall": 0.9471,
|
|
"total": 1485,
|
|
"trueNegative": 495,
|
|
"truePositive": 895,
|
|
"wilsonPrecisionLower95": 0.9365
|
|
},
|
|
"threshold": 0.705
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9209,
|
|
"falseNegative": 101,
|
|
"falsePositive": 44,
|
|
"precision": 0.9505,
|
|
"predictedPositive": 888,
|
|
"recall": 0.8931,
|
|
"total": 1485,
|
|
"trueNegative": 496,
|
|
"truePositive": 844,
|
|
"wilsonPrecisionLower95": 0.9341
|
|
},
|
|
"threshold": 0.78
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.9318,
|
|
"falseNegative": 163,
|
|
"falsePositive": 90,
|
|
"precision": 0.9505,
|
|
"predictedPositive": 1817,
|
|
"recall": 0.9138,
|
|
"total": 2970,
|
|
"trueNegative": 990,
|
|
"truePositive": 1727,
|
|
"wilsonPrecisionLower95": 0.9395
|
|
},
|
|
"threshold": 0.77
|
|
},
|
|
"scheduleNegotiation": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 1.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 45,
|
|
"recall": 1.0,
|
|
"total": 1485,
|
|
"trueNegative": 1440,
|
|
"truePositive": 45,
|
|
"wilsonPrecisionLower95": 0.9213
|
|
},
|
|
"threshold": 0.745
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9783,
|
|
"falseNegative": 0,
|
|
"falsePositive": 2,
|
|
"precision": 0.9574,
|
|
"predictedPositive": 47,
|
|
"recall": 1.0,
|
|
"total": 1485,
|
|
"trueNegative": 1438,
|
|
"truePositive": 45,
|
|
"wilsonPrecisionLower95": 0.8575
|
|
},
|
|
"threshold": 0.93
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.9778,
|
|
"falseNegative": 2,
|
|
"falsePositive": 2,
|
|
"precision": 0.9778,
|
|
"predictedPositive": 90,
|
|
"recall": 0.9778,
|
|
"total": 2970,
|
|
"trueNegative": 2878,
|
|
"truePositive": 88,
|
|
"wilsonPrecisionLower95": 0.9226
|
|
},
|
|
"threshold": 0.93
|
|
},
|
|
"task": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8488,
|
|
"falseNegative": 85,
|
|
"falsePositive": 13,
|
|
"precision": 0.9549,
|
|
"predictedPositive": 288,
|
|
"recall": 0.7639,
|
|
"total": 1485,
|
|
"trueNegative": 1112,
|
|
"truePositive": 275,
|
|
"wilsonPrecisionLower95": 0.9243
|
|
},
|
|
"threshold": 0.865
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8006,
|
|
"falseNegative": 111,
|
|
"falsePositive": 13,
|
|
"precision": 0.9504,
|
|
"predictedPositive": 262,
|
|
"recall": 0.6917,
|
|
"total": 1485,
|
|
"trueNegative": 1112,
|
|
"truePositive": 249,
|
|
"wilsonPrecisionLower95": 0.917
|
|
},
|
|
"threshold": 0.8
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.8301,
|
|
"falseNegative": 190,
|
|
"falsePositive": 27,
|
|
"precision": 0.9515,
|
|
"predictedPositive": 557,
|
|
"recall": 0.7361,
|
|
"total": 2970,
|
|
"trueNegative": 2223,
|
|
"truePositive": 530,
|
|
"wilsonPrecisionLower95": 0.9304
|
|
},
|
|
"threshold": 0.82
|
|
}
|
|
},
|
|
"trainingRecords": 15345,
|
|
"validation": {
|
|
"intentsMeeting95Precision": 8,
|
|
"macroF1": 0.7723,
|
|
"macroPrecision": 0.9713,
|
|
"macroRecall": 0.7011,
|
|
"minimumPrecision": 0.9513,
|
|
"perIntent": {
|
|
"blessing": {
|
|
"f1": 0.1443,
|
|
"falseNegative": 83,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 7,
|
|
"recall": 0.0778,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 7,
|
|
"wilsonPrecisionLower95": 0.6457
|
|
},
|
|
"complaint": {
|
|
"f1": 0.969,
|
|
"falseNegative": 16,
|
|
"falsePositive": 6,
|
|
"precision": 0.9829,
|
|
"predictedPositive": 350,
|
|
"recall": 0.9556,
|
|
"total": 2970,
|
|
"trueNegative": 2604,
|
|
"truePositive": 344,
|
|
"wilsonPrecisionLower95": 0.9631
|
|
},
|
|
"confirmationDecision": {
|
|
"f1": 0.8862,
|
|
"falseNegative": 32,
|
|
"falsePositive": 6,
|
|
"precision": 0.961,
|
|
"predictedPositive": 154,
|
|
"recall": 0.8222,
|
|
"total": 2970,
|
|
"trueNegative": 2784,
|
|
"truePositive": 148,
|
|
"wilsonPrecisionLower95": 0.9176
|
|
},
|
|
"followUpReminder": {
|
|
"f1": 0.8007,
|
|
"falseNegative": 115,
|
|
"falsePositive": 7,
|
|
"precision": 0.9722,
|
|
"predictedPositive": 252,
|
|
"recall": 0.6806,
|
|
"total": 2970,
|
|
"trueNegative": 2603,
|
|
"truePositive": 245,
|
|
"wilsonPrecisionLower95": 0.9438
|
|
},
|
|
"invitation": {
|
|
"f1": 0.5781,
|
|
"falseNegative": 106,
|
|
"falsePositive": 2,
|
|
"precision": 0.9737,
|
|
"predictedPositive": 76,
|
|
"recall": 0.4111,
|
|
"total": 2970,
|
|
"trueNegative": 2788,
|
|
"truePositive": 74,
|
|
"wilsonPrecisionLower95": 0.909
|
|
},
|
|
"question": {
|
|
"f1": 0.8227,
|
|
"falseNegative": 180,
|
|
"falsePositive": 14,
|
|
"precision": 0.9698,
|
|
"predictedPositive": 464,
|
|
"recall": 0.7143,
|
|
"total": 2970,
|
|
"trueNegative": 2326,
|
|
"truePositive": 450,
|
|
"wilsonPrecisionLower95": 0.95
|
|
},
|
|
"replyableMessage": {
|
|
"f1": 0.9354,
|
|
"falseNegative": 151,
|
|
"falsePositive": 89,
|
|
"precision": 0.9513,
|
|
"predictedPositive": 1828,
|
|
"recall": 0.9201,
|
|
"total": 2970,
|
|
"trueNegative": 991,
|
|
"truePositive": 1739,
|
|
"wilsonPrecisionLower95": 0.9405
|
|
},
|
|
"scheduleNegotiation": {
|
|
"f1": 0.989,
|
|
"falseNegative": 0,
|
|
"falsePositive": 2,
|
|
"precision": 0.9783,
|
|
"predictedPositive": 92,
|
|
"recall": 1.0,
|
|
"total": 2970,
|
|
"trueNegative": 2878,
|
|
"truePositive": 90,
|
|
"wilsonPrecisionLower95": 0.9242
|
|
},
|
|
"task": {
|
|
"f1": 0.8252,
|
|
"falseNegative": 196,
|
|
"falsePositive": 26,
|
|
"precision": 0.9527,
|
|
"predictedPositive": 550,
|
|
"recall": 0.7278,
|
|
"total": 2970,
|
|
"trueNegative": 2224,
|
|
"truePositive": 524,
|
|
"wilsonPrecisionLower95": 0.9316
|
|
}
|
|
},
|
|
"recordCount": 2970
|
|
}
|
|
},
|
|
{
|
|
"configuration": {
|
|
"alpha": 8e-06,
|
|
"augmentation": "none",
|
|
"char_max": 6,
|
|
"char_min": 3,
|
|
"external_weight": 0.63,
|
|
"hard_example_weight": 1.2,
|
|
"l1_ratio": 0.05,
|
|
"max_char_features": 60000,
|
|
"max_word_features": 30000,
|
|
"min_df": 2,
|
|
"pseudo_weight": 0.21,
|
|
"round": 4,
|
|
"word_max": 3
|
|
},
|
|
"featureCount": 79458,
|
|
"pseudoLabelCounts": {
|
|
"blessing": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"complaint": {
|
|
"negative": 97,
|
|
"positive": 0
|
|
},
|
|
"confirmationDecision": {
|
|
"negative": 51,
|
|
"positive": 2
|
|
},
|
|
"followUpReminder": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"invitation": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"question": {
|
|
"negative": 36,
|
|
"positive": 1
|
|
},
|
|
"replyableMessage": {
|
|
"negative": 3,
|
|
"positive": 130
|
|
},
|
|
"scheduleNegotiation": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"task": {
|
|
"negative": 85,
|
|
"positive": 0
|
|
}
|
|
},
|
|
"round": 4,
|
|
"selectionObjective": 0.796,
|
|
"syntheticRegression": {
|
|
"intentsMeeting95Precision": 6,
|
|
"macroF1": 0.6978,
|
|
"macroPrecision": 0.9611,
|
|
"macroRecall": 0.5994,
|
|
"minimumPrecision": 0.8074,
|
|
"perIntent": {
|
|
"blessing": {
|
|
"f1": 0.0645,
|
|
"falseNegative": 87,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 3,
|
|
"recall": 0.0333,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 3,
|
|
"wilsonPrecisionLower95": 0.4385
|
|
},
|
|
"complaint": {
|
|
"f1": 0.967,
|
|
"falseNegative": 23,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 337,
|
|
"recall": 0.9361,
|
|
"total": 2970,
|
|
"trueNegative": 2610,
|
|
"truePositive": 337,
|
|
"wilsonPrecisionLower95": 0.9887
|
|
},
|
|
"confirmationDecision": {
|
|
"f1": 0.6921,
|
|
"falseNegative": 71,
|
|
"falsePositive": 26,
|
|
"precision": 0.8074,
|
|
"predictedPositive": 135,
|
|
"recall": 0.6056,
|
|
"total": 2970,
|
|
"trueNegative": 2764,
|
|
"truePositive": 109,
|
|
"wilsonPrecisionLower95": 0.7328
|
|
},
|
|
"followUpReminder": {
|
|
"f1": 0.7734,
|
|
"falseNegative": 133,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 227,
|
|
"recall": 0.6306,
|
|
"total": 2970,
|
|
"trueNegative": 2610,
|
|
"truePositive": 227,
|
|
"wilsonPrecisionLower95": 0.9834
|
|
},
|
|
"invitation": {
|
|
"f1": 0.5837,
|
|
"falseNegative": 105,
|
|
"falsePositive": 2,
|
|
"precision": 0.974,
|
|
"predictedPositive": 77,
|
|
"recall": 0.4167,
|
|
"total": 2970,
|
|
"trueNegative": 2788,
|
|
"truePositive": 75,
|
|
"wilsonPrecisionLower95": 0.9102
|
|
},
|
|
"question": {
|
|
"f1": 0.7613,
|
|
"falseNegative": 241,
|
|
"falsePositive": 3,
|
|
"precision": 0.9923,
|
|
"predictedPositive": 392,
|
|
"recall": 0.6175,
|
|
"total": 2970,
|
|
"trueNegative": 2337,
|
|
"truePositive": 389,
|
|
"wilsonPrecisionLower95": 0.9777
|
|
},
|
|
"replyableMessage": {
|
|
"f1": 0.8816,
|
|
"falseNegative": 353,
|
|
"falsePositive": 60,
|
|
"precision": 0.9624,
|
|
"predictedPositive": 1597,
|
|
"recall": 0.8132,
|
|
"total": 2970,
|
|
"trueNegative": 1020,
|
|
"truePositive": 1537,
|
|
"wilsonPrecisionLower95": 0.9519
|
|
},
|
|
"scheduleNegotiation": {
|
|
"f1": 0.9222,
|
|
"falseNegative": 13,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 77,
|
|
"recall": 0.8556,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 77,
|
|
"wilsonPrecisionLower95": 0.9525
|
|
},
|
|
"task": {
|
|
"f1": 0.6346,
|
|
"falseNegative": 370,
|
|
"falsePositive": 33,
|
|
"precision": 0.9138,
|
|
"predictedPositive": 383,
|
|
"recall": 0.4861,
|
|
"total": 2970,
|
|
"trueNegative": 2217,
|
|
"truePositive": 350,
|
|
"wilsonPrecisionLower95": 0.8815
|
|
}
|
|
},
|
|
"recordCount": 2970
|
|
},
|
|
"thresholds": {
|
|
"blessing": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"complaint": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9972,
|
|
"falseNegative": 1,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 179,
|
|
"recall": 0.9944,
|
|
"total": 1485,
|
|
"trueNegative": 1305,
|
|
"truePositive": 179,
|
|
"wilsonPrecisionLower95": 0.979
|
|
},
|
|
"threshold": 0.5
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9253,
|
|
"falseNegative": 19,
|
|
"falsePositive": 7,
|
|
"precision": 0.9583,
|
|
"predictedPositive": 168,
|
|
"recall": 0.8944,
|
|
"total": 1485,
|
|
"trueNegative": 1298,
|
|
"truePositive": 161,
|
|
"wilsonPrecisionLower95": 0.9165
|
|
},
|
|
"threshold": 0.5
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.9618,
|
|
"falseNegative": 20,
|
|
"falsePositive": 7,
|
|
"precision": 0.9798,
|
|
"predictedPositive": 347,
|
|
"recall": 0.9444,
|
|
"total": 2970,
|
|
"trueNegative": 2603,
|
|
"truePositive": 340,
|
|
"wilsonPrecisionLower95": 0.959
|
|
},
|
|
"threshold": 0.5
|
|
},
|
|
"confirmationDecision": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8765,
|
|
"falseNegative": 19,
|
|
"falsePositive": 1,
|
|
"precision": 0.9861,
|
|
"predictedPositive": 72,
|
|
"recall": 0.7889,
|
|
"total": 1485,
|
|
"trueNegative": 1394,
|
|
"truePositive": 71,
|
|
"wilsonPrecisionLower95": 0.9254
|
|
},
|
|
"threshold": 0.945
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.6569,
|
|
"falseNegative": 45,
|
|
"falsePositive": 2,
|
|
"precision": 0.9574,
|
|
"predictedPositive": 47,
|
|
"recall": 0.5,
|
|
"total": 1485,
|
|
"trueNegative": 1393,
|
|
"truePositive": 45,
|
|
"wilsonPrecisionLower95": 0.8575
|
|
},
|
|
"threshold": 0.64
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.7108,
|
|
"falseNegative": 78,
|
|
"falsePositive": 5,
|
|
"precision": 0.9533,
|
|
"predictedPositive": 107,
|
|
"recall": 0.5667,
|
|
"total": 2970,
|
|
"trueNegative": 2785,
|
|
"truePositive": 102,
|
|
"wilsonPrecisionLower95": 0.8952
|
|
},
|
|
"threshold": 0.91
|
|
},
|
|
"followUpReminder": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8,
|
|
"falseNegative": 56,
|
|
"falsePositive": 6,
|
|
"precision": 0.9538,
|
|
"predictedPositive": 130,
|
|
"recall": 0.6889,
|
|
"total": 1485,
|
|
"trueNegative": 1299,
|
|
"truePositive": 124,
|
|
"wilsonPrecisionLower95": 0.903
|
|
},
|
|
"threshold": 0.98
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.6165,
|
|
"falseNegative": 98,
|
|
"falsePositive": 4,
|
|
"precision": 0.9535,
|
|
"predictedPositive": 86,
|
|
"recall": 0.4556,
|
|
"total": 1485,
|
|
"trueNegative": 1301,
|
|
"truePositive": 82,
|
|
"wilsonPrecisionLower95": 0.8864
|
|
},
|
|
"threshold": 0.825
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.5912,
|
|
"falseNegative": 206,
|
|
"falsePositive": 7,
|
|
"precision": 0.9565,
|
|
"predictedPositive": 161,
|
|
"recall": 0.4278,
|
|
"total": 2970,
|
|
"trueNegative": 2603,
|
|
"truePositive": 154,
|
|
"wilsonPrecisionLower95": 0.913
|
|
},
|
|
"threshold": 0.98
|
|
},
|
|
"invitation": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8302,
|
|
"falseNegative": 24,
|
|
"falsePositive": 3,
|
|
"precision": 0.9565,
|
|
"predictedPositive": 69,
|
|
"recall": 0.7333,
|
|
"total": 1485,
|
|
"trueNegative": 1392,
|
|
"truePositive": 66,
|
|
"wilsonPrecisionLower95": 0.8798
|
|
},
|
|
"threshold": 0.98
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.536,
|
|
"falseNegative": 113,
|
|
"falsePositive": 3,
|
|
"precision": 0.9571,
|
|
"predictedPositive": 70,
|
|
"recall": 0.3722,
|
|
"total": 2970,
|
|
"trueNegative": 2787,
|
|
"truePositive": 67,
|
|
"wilsonPrecisionLower95": 0.8814
|
|
},
|
|
"threshold": 0.995
|
|
},
|
|
"question": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.7813,
|
|
"falseNegative": 106,
|
|
"falsePositive": 11,
|
|
"precision": 0.95,
|
|
"predictedPositive": 220,
|
|
"recall": 0.6635,
|
|
"total": 1485,
|
|
"trueNegative": 1159,
|
|
"truePositive": 209,
|
|
"wilsonPrecisionLower95": 0.9127
|
|
},
|
|
"threshold": 0.865
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.6844,
|
|
"falseNegative": 148,
|
|
"falsePositive": 6,
|
|
"precision": 0.9653,
|
|
"predictedPositive": 173,
|
|
"recall": 0.5302,
|
|
"total": 1485,
|
|
"trueNegative": 1164,
|
|
"truePositive": 167,
|
|
"wilsonPrecisionLower95": 0.9264
|
|
},
|
|
"threshold": 0.63
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.75,
|
|
"falseNegative": 240,
|
|
"falsePositive": 20,
|
|
"precision": 0.9512,
|
|
"predictedPositive": 410,
|
|
"recall": 0.619,
|
|
"total": 2970,
|
|
"trueNegative": 2320,
|
|
"truePositive": 390,
|
|
"wilsonPrecisionLower95": 0.9259
|
|
},
|
|
"threshold": 0.74
|
|
},
|
|
"replyableMessage": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9441,
|
|
"falseNegative": 59,
|
|
"falsePositive": 46,
|
|
"precision": 0.9506,
|
|
"predictedPositive": 932,
|
|
"recall": 0.9376,
|
|
"total": 1485,
|
|
"trueNegative": 494,
|
|
"truePositive": 886,
|
|
"wilsonPrecisionLower95": 0.9348
|
|
},
|
|
"threshold": 0.76
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8342,
|
|
"falseNegative": 243,
|
|
"falsePositive": 36,
|
|
"precision": 0.9512,
|
|
"predictedPositive": 738,
|
|
"recall": 0.7429,
|
|
"total": 1485,
|
|
"trueNegative": 504,
|
|
"truePositive": 702,
|
|
"wilsonPrecisionLower95": 0.9332
|
|
},
|
|
"threshold": 0.885
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.9056,
|
|
"falseNegative": 255,
|
|
"falsePositive": 86,
|
|
"precision": 0.95,
|
|
"predictedPositive": 1721,
|
|
"recall": 0.8651,
|
|
"total": 2970,
|
|
"trueNegative": 994,
|
|
"truePositive": 1635,
|
|
"wilsonPrecisionLower95": 0.9387
|
|
},
|
|
"threshold": 0.855
|
|
},
|
|
"scheduleNegotiation": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 1.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 45,
|
|
"recall": 1.0,
|
|
"total": 1485,
|
|
"trueNegative": 1440,
|
|
"truePositive": 45,
|
|
"wilsonPrecisionLower95": 0.9213
|
|
},
|
|
"threshold": 0.5
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9318,
|
|
"falseNegative": 4,
|
|
"falsePositive": 2,
|
|
"precision": 0.9535,
|
|
"predictedPositive": 43,
|
|
"recall": 0.9111,
|
|
"total": 1485,
|
|
"trueNegative": 1438,
|
|
"truePositive": 41,
|
|
"wilsonPrecisionLower95": 0.8454
|
|
},
|
|
"threshold": 0.945
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.9727,
|
|
"falseNegative": 1,
|
|
"falsePositive": 4,
|
|
"precision": 0.957,
|
|
"predictedPositive": 93,
|
|
"recall": 0.9889,
|
|
"total": 2970,
|
|
"trueNegative": 2876,
|
|
"truePositive": 89,
|
|
"wilsonPrecisionLower95": 0.8946
|
|
},
|
|
"threshold": 0.62
|
|
},
|
|
"task": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.851,
|
|
"falseNegative": 83,
|
|
"falsePositive": 14,
|
|
"precision": 0.9519,
|
|
"predictedPositive": 291,
|
|
"recall": 0.7694,
|
|
"total": 1485,
|
|
"trueNegative": 1111,
|
|
"truePositive": 277,
|
|
"wilsonPrecisionLower95": 0.9209
|
|
},
|
|
"threshold": 0.87
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.4886,
|
|
"falseNegative": 242,
|
|
"falsePositive": 5,
|
|
"precision": 0.9593,
|
|
"predictedPositive": 123,
|
|
"recall": 0.3278,
|
|
"total": 1485,
|
|
"trueNegative": 1120,
|
|
"truePositive": 118,
|
|
"wilsonPrecisionLower95": 0.9084
|
|
},
|
|
"threshold": 0.935
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.6978,
|
|
"falseNegative": 324,
|
|
"falsePositive": 19,
|
|
"precision": 0.9542,
|
|
"predictedPositive": 415,
|
|
"recall": 0.55,
|
|
"total": 2970,
|
|
"trueNegative": 2231,
|
|
"truePositive": 396,
|
|
"wilsonPrecisionLower95": 0.9296
|
|
},
|
|
"threshold": 0.915
|
|
}
|
|
},
|
|
"trainingRecords": 15345,
|
|
"validation": {
|
|
"intentsMeeting95Precision": 9,
|
|
"macroF1": 0.7192,
|
|
"macroPrecision": 0.9671,
|
|
"macroRecall": 0.62,
|
|
"minimumPrecision": 0.9509,
|
|
"perIntent": {
|
|
"blessing": {
|
|
"f1": 0.2,
|
|
"falseNegative": 80,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 10,
|
|
"recall": 0.1111,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 10,
|
|
"wilsonPrecisionLower95": 0.7225
|
|
},
|
|
"complaint": {
|
|
"f1": 0.9618,
|
|
"falseNegative": 20,
|
|
"falsePositive": 7,
|
|
"precision": 0.9798,
|
|
"predictedPositive": 347,
|
|
"recall": 0.9444,
|
|
"total": 2970,
|
|
"trueNegative": 2603,
|
|
"truePositive": 340,
|
|
"wilsonPrecisionLower95": 0.959
|
|
},
|
|
"confirmationDecision": {
|
|
"f1": 0.7759,
|
|
"falseNegative": 64,
|
|
"falsePositive": 3,
|
|
"precision": 0.9748,
|
|
"predictedPositive": 119,
|
|
"recall": 0.6444,
|
|
"total": 2970,
|
|
"trueNegative": 2787,
|
|
"truePositive": 116,
|
|
"wilsonPrecisionLower95": 0.9285
|
|
},
|
|
"followUpReminder": {
|
|
"f1": 0.7153,
|
|
"falseNegative": 154,
|
|
"falsePositive": 10,
|
|
"precision": 0.9537,
|
|
"predictedPositive": 216,
|
|
"recall": 0.5722,
|
|
"total": 2970,
|
|
"trueNegative": 2600,
|
|
"truePositive": 206,
|
|
"wilsonPrecisionLower95": 0.9169
|
|
},
|
|
"invitation": {
|
|
"f1": 0.5301,
|
|
"falseNegative": 114,
|
|
"falsePositive": 3,
|
|
"precision": 0.9565,
|
|
"predictedPositive": 69,
|
|
"recall": 0.3667,
|
|
"total": 2970,
|
|
"trueNegative": 2787,
|
|
"truePositive": 66,
|
|
"wilsonPrecisionLower95": 0.8798
|
|
},
|
|
"question": {
|
|
"f1": 0.7351,
|
|
"falseNegative": 254,
|
|
"falsePositive": 17,
|
|
"precision": 0.9567,
|
|
"predictedPositive": 393,
|
|
"recall": 0.5968,
|
|
"total": 2970,
|
|
"trueNegative": 2323,
|
|
"truePositive": 376,
|
|
"wilsonPrecisionLower95": 0.9318
|
|
},
|
|
"replyableMessage": {
|
|
"f1": 0.8921,
|
|
"falseNegative": 302,
|
|
"falsePositive": 82,
|
|
"precision": 0.9509,
|
|
"predictedPositive": 1670,
|
|
"recall": 0.8402,
|
|
"total": 2970,
|
|
"trueNegative": 998,
|
|
"truePositive": 1588,
|
|
"wilsonPrecisionLower95": 0.9395
|
|
},
|
|
"scheduleNegotiation": {
|
|
"f1": 0.9663,
|
|
"falseNegative": 4,
|
|
"falsePositive": 2,
|
|
"precision": 0.9773,
|
|
"predictedPositive": 88,
|
|
"recall": 0.9556,
|
|
"total": 2970,
|
|
"trueNegative": 2878,
|
|
"truePositive": 86,
|
|
"wilsonPrecisionLower95": 0.9209
|
|
},
|
|
"task": {
|
|
"f1": 0.6966,
|
|
"falseNegative": 325,
|
|
"falsePositive": 19,
|
|
"precision": 0.9541,
|
|
"predictedPositive": 414,
|
|
"recall": 0.5486,
|
|
"total": 2970,
|
|
"trueNegative": 2231,
|
|
"truePositive": 395,
|
|
"wilsonPrecisionLower95": 0.9294
|
|
}
|
|
},
|
|
"recordCount": 2970
|
|
}
|
|
},
|
|
{
|
|
"configuration": {
|
|
"alpha": 1.2e-05,
|
|
"augmentation": "none",
|
|
"char_max": 5,
|
|
"char_min": 2,
|
|
"external_weight": 0.65,
|
|
"hard_example_weight": 1.4,
|
|
"l1_ratio": 0.1,
|
|
"max_char_features": 60000,
|
|
"max_word_features": 32000,
|
|
"min_df": 1,
|
|
"pseudo_weight": 0.22499999999999998,
|
|
"round": 5,
|
|
"word_max": 3
|
|
},
|
|
"featureCount": 92000,
|
|
"pseudoLabelCounts": {
|
|
"blessing": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"complaint": {
|
|
"negative": 88,
|
|
"positive": 0
|
|
},
|
|
"confirmationDecision": {
|
|
"negative": 96,
|
|
"positive": 3
|
|
},
|
|
"followUpReminder": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"invitation": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"question": {
|
|
"negative": 40,
|
|
"positive": 1
|
|
},
|
|
"replyableMessage": {
|
|
"negative": 3,
|
|
"positive": 150
|
|
},
|
|
"scheduleNegotiation": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"task": {
|
|
"negative": 97,
|
|
"positive": 0
|
|
}
|
|
},
|
|
"round": 5,
|
|
"selectionObjective": 0.7771,
|
|
"syntheticRegression": {
|
|
"intentsMeeting95Precision": 5,
|
|
"macroF1": 0.6709,
|
|
"macroPrecision": 0.9586,
|
|
"macroRecall": 0.5803,
|
|
"minimumPrecision": 0.8588,
|
|
"perIntent": {
|
|
"blessing": {
|
|
"f1": 0.022,
|
|
"falseNegative": 89,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 1,
|
|
"recall": 0.0111,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 1,
|
|
"wilsonPrecisionLower95": 0.2065
|
|
},
|
|
"complaint": {
|
|
"f1": 0.967,
|
|
"falseNegative": 23,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 337,
|
|
"recall": 0.9361,
|
|
"total": 2970,
|
|
"trueNegative": 2610,
|
|
"truePositive": 337,
|
|
"wilsonPrecisionLower95": 0.9887
|
|
},
|
|
"confirmationDecision": {
|
|
"f1": 0.8343,
|
|
"falseNegative": 34,
|
|
"falsePositive": 24,
|
|
"precision": 0.8588,
|
|
"predictedPositive": 170,
|
|
"recall": 0.8111,
|
|
"total": 2970,
|
|
"trueNegative": 2766,
|
|
"truePositive": 146,
|
|
"wilsonPrecisionLower95": 0.7985
|
|
},
|
|
"followUpReminder": {
|
|
"f1": 0.7173,
|
|
"falseNegative": 157,
|
|
"falsePositive": 3,
|
|
"precision": 0.9854,
|
|
"predictedPositive": 206,
|
|
"recall": 0.5639,
|
|
"total": 2970,
|
|
"trueNegative": 2607,
|
|
"truePositive": 203,
|
|
"wilsonPrecisionLower95": 0.9581
|
|
},
|
|
"invitation": {
|
|
"f1": 0.3612,
|
|
"falseNegative": 139,
|
|
"falsePositive": 6,
|
|
"precision": 0.8723,
|
|
"predictedPositive": 47,
|
|
"recall": 0.2278,
|
|
"total": 2970,
|
|
"trueNegative": 2784,
|
|
"truePositive": 41,
|
|
"wilsonPrecisionLower95": 0.7483
|
|
},
|
|
"question": {
|
|
"f1": 0.8336,
|
|
"falseNegative": 179,
|
|
"falsePositive": 1,
|
|
"precision": 0.9978,
|
|
"predictedPositive": 452,
|
|
"recall": 0.7159,
|
|
"total": 2970,
|
|
"trueNegative": 2339,
|
|
"truePositive": 451,
|
|
"wilsonPrecisionLower95": 0.9876
|
|
},
|
|
"replyableMessage": {
|
|
"f1": 0.912,
|
|
"falseNegative": 257,
|
|
"falsePositive": 58,
|
|
"precision": 0.9657,
|
|
"predictedPositive": 1691,
|
|
"recall": 0.864,
|
|
"total": 2970,
|
|
"trueNegative": 1022,
|
|
"truePositive": 1633,
|
|
"wilsonPrecisionLower95": 0.9559
|
|
},
|
|
"scheduleNegotiation": {
|
|
"f1": 0.6364,
|
|
"falseNegative": 48,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 42,
|
|
"recall": 0.4667,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 42,
|
|
"wilsonPrecisionLower95": 0.9162
|
|
},
|
|
"task": {
|
|
"f1": 0.7542,
|
|
"falseNegative": 269,
|
|
"falsePositive": 25,
|
|
"precision": 0.9475,
|
|
"predictedPositive": 476,
|
|
"recall": 0.6264,
|
|
"total": 2970,
|
|
"trueNegative": 2225,
|
|
"truePositive": 451,
|
|
"wilsonPrecisionLower95": 0.9236
|
|
}
|
|
},
|
|
"recordCount": 2970
|
|
},
|
|
"thresholds": {
|
|
"blessing": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"complaint": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 1.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 180,
|
|
"recall": 1.0,
|
|
"total": 1485,
|
|
"trueNegative": 1305,
|
|
"truePositive": 180,
|
|
"wilsonPrecisionLower95": 0.9791
|
|
},
|
|
"threshold": 0.69
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9288,
|
|
"falseNegative": 17,
|
|
"falsePositive": 8,
|
|
"precision": 0.9532,
|
|
"predictedPositive": 171,
|
|
"recall": 0.9056,
|
|
"total": 1485,
|
|
"trueNegative": 1297,
|
|
"truePositive": 163,
|
|
"wilsonPrecisionLower95": 0.9104
|
|
},
|
|
"threshold": 0.51
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.9569,
|
|
"falseNegative": 16,
|
|
"falsePositive": 15,
|
|
"precision": 0.9582,
|
|
"predictedPositive": 359,
|
|
"recall": 0.9556,
|
|
"total": 2970,
|
|
"trueNegative": 2595,
|
|
"truePositive": 344,
|
|
"wilsonPrecisionLower95": 0.9322
|
|
},
|
|
"threshold": 0.5
|
|
},
|
|
"confirmationDecision": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8589,
|
|
"falseNegative": 20,
|
|
"falsePositive": 3,
|
|
"precision": 0.9589,
|
|
"predictedPositive": 73,
|
|
"recall": 0.7778,
|
|
"total": 1485,
|
|
"trueNegative": 1392,
|
|
"truePositive": 70,
|
|
"wilsonPrecisionLower95": 0.886
|
|
},
|
|
"threshold": 0.57
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.6763,
|
|
"falseNegative": 43,
|
|
"falsePositive": 2,
|
|
"precision": 0.9592,
|
|
"predictedPositive": 49,
|
|
"recall": 0.5222,
|
|
"total": 1485,
|
|
"trueNegative": 1393,
|
|
"truePositive": 47,
|
|
"wilsonPrecisionLower95": 0.8629
|
|
},
|
|
"threshold": 0.5
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.76,
|
|
"falseNegative": 66,
|
|
"falsePositive": 6,
|
|
"precision": 0.95,
|
|
"predictedPositive": 120,
|
|
"recall": 0.6333,
|
|
"total": 2970,
|
|
"trueNegative": 2784,
|
|
"truePositive": 114,
|
|
"wilsonPrecisionLower95": 0.8952
|
|
},
|
|
"threshold": 0.56
|
|
},
|
|
"followUpReminder": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8,
|
|
"falseNegative": 56,
|
|
"falsePositive": 6,
|
|
"precision": 0.9538,
|
|
"predictedPositive": 130,
|
|
"recall": 0.6889,
|
|
"total": 1485,
|
|
"trueNegative": 1299,
|
|
"truePositive": 124,
|
|
"wilsonPrecisionLower95": 0.903
|
|
},
|
|
"threshold": 0.96
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.7708,
|
|
"falseNegative": 64,
|
|
"falsePositive": 5,
|
|
"precision": 0.9587,
|
|
"predictedPositive": 121,
|
|
"recall": 0.6444,
|
|
"total": 1485,
|
|
"trueNegative": 1300,
|
|
"truePositive": 116,
|
|
"wilsonPrecisionLower95": 0.9069
|
|
},
|
|
"threshold": 0.58
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.6892,
|
|
"falseNegative": 166,
|
|
"falsePositive": 9,
|
|
"precision": 0.9557,
|
|
"predictedPositive": 203,
|
|
"recall": 0.5389,
|
|
"total": 2970,
|
|
"trueNegative": 2601,
|
|
"truePositive": 194,
|
|
"wilsonPrecisionLower95": 0.9179
|
|
},
|
|
"threshold": 0.915
|
|
},
|
|
"invitation": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.5203,
|
|
"falseNegative": 58,
|
|
"falsePositive": 1,
|
|
"precision": 0.9697,
|
|
"predictedPositive": 33,
|
|
"recall": 0.3556,
|
|
"total": 1485,
|
|
"trueNegative": 1394,
|
|
"truePositive": 32,
|
|
"wilsonPrecisionLower95": 0.8468
|
|
},
|
|
"threshold": 0.88
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"question": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8225,
|
|
"falseNegative": 88,
|
|
"falsePositive": 10,
|
|
"precision": 0.9578,
|
|
"predictedPositive": 237,
|
|
"recall": 0.7206,
|
|
"total": 1485,
|
|
"trueNegative": 1160,
|
|
"truePositive": 227,
|
|
"wilsonPrecisionLower95": 0.9241
|
|
},
|
|
"threshold": 0.84
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8388,
|
|
"falseNegative": 86,
|
|
"falsePositive": 2,
|
|
"precision": 0.9913,
|
|
"predictedPositive": 231,
|
|
"recall": 0.727,
|
|
"total": 1485,
|
|
"trueNegative": 1168,
|
|
"truePositive": 229,
|
|
"wilsonPrecisionLower95": 0.969
|
|
},
|
|
"threshold": 0.5
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.8591,
|
|
"falseNegative": 136,
|
|
"falsePositive": 26,
|
|
"precision": 0.95,
|
|
"predictedPositive": 520,
|
|
"recall": 0.7841,
|
|
"total": 2970,
|
|
"trueNegative": 2314,
|
|
"truePositive": 494,
|
|
"wilsonPrecisionLower95": 0.9277
|
|
},
|
|
"threshold": 0.505
|
|
},
|
|
"replyableMessage": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9541,
|
|
"falseNegative": 40,
|
|
"falsePositive": 47,
|
|
"precision": 0.9506,
|
|
"predictedPositive": 952,
|
|
"recall": 0.9577,
|
|
"total": 1485,
|
|
"trueNegative": 493,
|
|
"truePositive": 905,
|
|
"wilsonPrecisionLower95": 0.935
|
|
},
|
|
"threshold": 0.69
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8807,
|
|
"falseNegative": 170,
|
|
"falsePositive": 40,
|
|
"precision": 0.9509,
|
|
"predictedPositive": 815,
|
|
"recall": 0.8201,
|
|
"total": 1485,
|
|
"trueNegative": 500,
|
|
"truePositive": 775,
|
|
"wilsonPrecisionLower95": 0.9339
|
|
},
|
|
"threshold": 0.8
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.914,
|
|
"falseNegative": 227,
|
|
"falsePositive": 86,
|
|
"precision": 0.9508,
|
|
"predictedPositive": 1749,
|
|
"recall": 0.8799,
|
|
"total": 2970,
|
|
"trueNegative": 994,
|
|
"truePositive": 1663,
|
|
"wilsonPrecisionLower95": 0.9397
|
|
},
|
|
"threshold": 0.78
|
|
},
|
|
"scheduleNegotiation": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9888,
|
|
"falseNegative": 1,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 44,
|
|
"recall": 0.9778,
|
|
"total": 1485,
|
|
"trueNegative": 1440,
|
|
"truePositive": 44,
|
|
"wilsonPrecisionLower95": 0.9197
|
|
},
|
|
"threshold": 0.635
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.8129,
|
|
"falseNegative": 27,
|
|
"falsePositive": 2,
|
|
"precision": 0.9692,
|
|
"predictedPositive": 65,
|
|
"recall": 0.7,
|
|
"total": 2970,
|
|
"trueNegative": 2878,
|
|
"truePositive": 63,
|
|
"wilsonPrecisionLower95": 0.8946
|
|
},
|
|
"threshold": 0.995
|
|
},
|
|
"task": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8185,
|
|
"falseNegative": 103,
|
|
"falsePositive": 11,
|
|
"precision": 0.959,
|
|
"predictedPositive": 268,
|
|
"recall": 0.7139,
|
|
"total": 1485,
|
|
"trueNegative": 1114,
|
|
"truePositive": 257,
|
|
"wilsonPrecisionLower95": 0.928
|
|
},
|
|
"threshold": 0.83
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.7702,
|
|
"falseNegative": 127,
|
|
"falsePositive": 12,
|
|
"precision": 0.951,
|
|
"predictedPositive": 245,
|
|
"recall": 0.6472,
|
|
"total": 1485,
|
|
"trueNegative": 1113,
|
|
"truePositive": 233,
|
|
"wilsonPrecisionLower95": 0.9164
|
|
},
|
|
"threshold": 0.765
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.7816,
|
|
"falseNegative": 244,
|
|
"falsePositive": 22,
|
|
"precision": 0.9558,
|
|
"predictedPositive": 498,
|
|
"recall": 0.6611,
|
|
"total": 2970,
|
|
"trueNegative": 2228,
|
|
"truePositive": 476,
|
|
"wilsonPrecisionLower95": 0.934
|
|
},
|
|
"threshold": 0.805
|
|
}
|
|
},
|
|
"trainingRecords": 15345,
|
|
"validation": {
|
|
"intentsMeeting95Precision": 8,
|
|
"macroF1": 0.6867,
|
|
"macroPrecision": 0.9721,
|
|
"macroRecall": 0.5915,
|
|
"minimumPrecision": 0.9508,
|
|
"perIntent": {
|
|
"blessing": {
|
|
"f1": 0.0851,
|
|
"falseNegative": 86,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 4,
|
|
"recall": 0.0444,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 4,
|
|
"wilsonPrecisionLower95": 0.5101
|
|
},
|
|
"complaint": {
|
|
"f1": 0.9648,
|
|
"falseNegative": 17,
|
|
"falsePositive": 8,
|
|
"precision": 0.9772,
|
|
"predictedPositive": 351,
|
|
"recall": 0.9528,
|
|
"total": 2970,
|
|
"trueNegative": 2602,
|
|
"truePositive": 343,
|
|
"wilsonPrecisionLower95": 0.9557
|
|
},
|
|
"confirmationDecision": {
|
|
"f1": 0.7748,
|
|
"falseNegative": 63,
|
|
"falsePositive": 5,
|
|
"precision": 0.959,
|
|
"predictedPositive": 122,
|
|
"recall": 0.65,
|
|
"total": 2970,
|
|
"trueNegative": 2785,
|
|
"truePositive": 117,
|
|
"wilsonPrecisionLower95": 0.9076
|
|
},
|
|
"followUpReminder": {
|
|
"f1": 0.7856,
|
|
"falseNegative": 120,
|
|
"falsePositive": 11,
|
|
"precision": 0.9562,
|
|
"predictedPositive": 251,
|
|
"recall": 0.6667,
|
|
"total": 2970,
|
|
"trueNegative": 2599,
|
|
"truePositive": 240,
|
|
"wilsonPrecisionLower95": 0.9232
|
|
},
|
|
"invitation": {
|
|
"f1": 0.3694,
|
|
"falseNegative": 139,
|
|
"falsePositive": 1,
|
|
"precision": 0.9762,
|
|
"predictedPositive": 42,
|
|
"recall": 0.2278,
|
|
"total": 2970,
|
|
"trueNegative": 2789,
|
|
"truePositive": 41,
|
|
"wilsonPrecisionLower95": 0.8768
|
|
},
|
|
"question": {
|
|
"f1": 0.8306,
|
|
"falseNegative": 174,
|
|
"falsePositive": 12,
|
|
"precision": 0.9744,
|
|
"predictedPositive": 468,
|
|
"recall": 0.7238,
|
|
"total": 2970,
|
|
"trueNegative": 2328,
|
|
"truePositive": 456,
|
|
"wilsonPrecisionLower95": 0.9557
|
|
},
|
|
"replyableMessage": {
|
|
"f1": 0.9188,
|
|
"falseNegative": 210,
|
|
"falsePositive": 87,
|
|
"precision": 0.9508,
|
|
"predictedPositive": 1767,
|
|
"recall": 0.8889,
|
|
"total": 2970,
|
|
"trueNegative": 993,
|
|
"truePositive": 1680,
|
|
"wilsonPrecisionLower95": 0.9397
|
|
},
|
|
"scheduleNegotiation": {
|
|
"f1": 0.6567,
|
|
"falseNegative": 46,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 44,
|
|
"recall": 0.4889,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 44,
|
|
"wilsonPrecisionLower95": 0.9197
|
|
},
|
|
"task": {
|
|
"f1": 0.7948,
|
|
"falseNegative": 230,
|
|
"falsePositive": 23,
|
|
"precision": 0.9552,
|
|
"predictedPositive": 513,
|
|
"recall": 0.6806,
|
|
"total": 2970,
|
|
"trueNegative": 2227,
|
|
"truePositive": 490,
|
|
"wilsonPrecisionLower95": 0.9336
|
|
}
|
|
},
|
|
"recordCount": 2970
|
|
}
|
|
},
|
|
{
|
|
"configuration": {
|
|
"alpha": 8e-06,
|
|
"augmentation": "punctuation",
|
|
"char_max": 6,
|
|
"char_min": 2,
|
|
"external_weight": 0.67,
|
|
"hard_example_weight": 1.5,
|
|
"l1_ratio": 0.1,
|
|
"max_char_features": 65000,
|
|
"max_word_features": 28000,
|
|
"min_df": 2,
|
|
"pseudo_weight": 0.24,
|
|
"round": 6,
|
|
"word_max": 2
|
|
},
|
|
"featureCount": 78354,
|
|
"pseudoLabelCounts": {
|
|
"blessing": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"complaint": {
|
|
"negative": 146,
|
|
"positive": 0
|
|
},
|
|
"confirmationDecision": {
|
|
"negative": 88,
|
|
"positive": 2
|
|
},
|
|
"followUpReminder": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"invitation": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"question": {
|
|
"negative": 36,
|
|
"positive": 5
|
|
},
|
|
"replyableMessage": {
|
|
"negative": 2,
|
|
"positive": 130
|
|
},
|
|
"scheduleNegotiation": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"task": {
|
|
"negative": 83,
|
|
"positive": 0
|
|
}
|
|
},
|
|
"round": 6,
|
|
"selectionObjective": 0.811,
|
|
"syntheticRegression": {
|
|
"intentsMeeting95Precision": 5,
|
|
"macroF1": 0.7064,
|
|
"macroPrecision": 0.9443,
|
|
"macroRecall": 0.623,
|
|
"minimumPrecision": 0.7975,
|
|
"perIntent": {
|
|
"blessing": {
|
|
"f1": 0.0435,
|
|
"falseNegative": 88,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 2,
|
|
"recall": 0.0222,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 2,
|
|
"wilsonPrecisionLower95": 0.3424
|
|
},
|
|
"complaint": {
|
|
"f1": 0.967,
|
|
"falseNegative": 23,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 337,
|
|
"recall": 0.9361,
|
|
"total": 2970,
|
|
"trueNegative": 2610,
|
|
"truePositive": 337,
|
|
"wilsonPrecisionLower95": 0.9887
|
|
},
|
|
"confirmationDecision": {
|
|
"f1": 0.7215,
|
|
"falseNegative": 66,
|
|
"falsePositive": 22,
|
|
"precision": 0.8382,
|
|
"predictedPositive": 136,
|
|
"recall": 0.6333,
|
|
"total": 2970,
|
|
"trueNegative": 2768,
|
|
"truePositive": 114,
|
|
"wilsonPrecisionLower95": 0.7672
|
|
},
|
|
"followUpReminder": {
|
|
"f1": 0.6847,
|
|
"falseNegative": 170,
|
|
"falsePositive": 5,
|
|
"precision": 0.9744,
|
|
"predictedPositive": 195,
|
|
"recall": 0.5278,
|
|
"total": 2970,
|
|
"trueNegative": 2605,
|
|
"truePositive": 190,
|
|
"wilsonPrecisionLower95": 0.9414
|
|
},
|
|
"invitation": {
|
|
"f1": 0.4865,
|
|
"falseNegative": 117,
|
|
"falsePositive": 16,
|
|
"precision": 0.7975,
|
|
"predictedPositive": 79,
|
|
"recall": 0.35,
|
|
"total": 2970,
|
|
"trueNegative": 2774,
|
|
"truePositive": 63,
|
|
"wilsonPrecisionLower95": 0.696
|
|
},
|
|
"question": {
|
|
"f1": 0.8556,
|
|
"falseNegative": 156,
|
|
"falsePositive": 4,
|
|
"precision": 0.9916,
|
|
"predictedPositive": 478,
|
|
"recall": 0.7524,
|
|
"total": 2970,
|
|
"trueNegative": 2336,
|
|
"truePositive": 474,
|
|
"wilsonPrecisionLower95": 0.9787
|
|
},
|
|
"replyableMessage": {
|
|
"f1": 0.8888,
|
|
"falseNegative": 335,
|
|
"falsePositive": 54,
|
|
"precision": 0.9664,
|
|
"predictedPositive": 1609,
|
|
"recall": 0.8228,
|
|
"total": 2970,
|
|
"trueNegative": 1026,
|
|
"truePositive": 1555,
|
|
"wilsonPrecisionLower95": 0.9565
|
|
},
|
|
"scheduleNegotiation": {
|
|
"f1": 0.9831,
|
|
"falseNegative": 3,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 87,
|
|
"recall": 0.9667,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 87,
|
|
"wilsonPrecisionLower95": 0.9577
|
|
},
|
|
"task": {
|
|
"f1": 0.7265,
|
|
"falseNegative": 291,
|
|
"falsePositive": 32,
|
|
"precision": 0.9306,
|
|
"predictedPositive": 461,
|
|
"recall": 0.5958,
|
|
"total": 2970,
|
|
"trueNegative": 2218,
|
|
"truePositive": 429,
|
|
"wilsonPrecisionLower95": 0.9037
|
|
}
|
|
},
|
|
"recordCount": 2970
|
|
},
|
|
"thresholds": {
|
|
"blessing": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"complaint": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9834,
|
|
"falseNegative": 2,
|
|
"falsePositive": 4,
|
|
"precision": 0.978,
|
|
"predictedPositive": 182,
|
|
"recall": 0.9889,
|
|
"total": 1485,
|
|
"trueNegative": 1301,
|
|
"truePositive": 178,
|
|
"wilsonPrecisionLower95": 0.9449
|
|
},
|
|
"threshold": 0.555
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9494,
|
|
"falseNegative": 11,
|
|
"falsePositive": 7,
|
|
"precision": 0.9602,
|
|
"predictedPositive": 176,
|
|
"recall": 0.9389,
|
|
"total": 1485,
|
|
"trueNegative": 1298,
|
|
"truePositive": 169,
|
|
"wilsonPrecisionLower95": 0.9202
|
|
},
|
|
"threshold": 0.535
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.9652,
|
|
"falseNegative": 13,
|
|
"falsePositive": 12,
|
|
"precision": 0.9666,
|
|
"predictedPositive": 359,
|
|
"recall": 0.9639,
|
|
"total": 2970,
|
|
"trueNegative": 2598,
|
|
"truePositive": 347,
|
|
"wilsonPrecisionLower95": 0.9425
|
|
},
|
|
"threshold": 0.535
|
|
},
|
|
"confirmationDecision": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8,
|
|
"falseNegative": 28,
|
|
"falsePositive": 3,
|
|
"precision": 0.9538,
|
|
"predictedPositive": 65,
|
|
"recall": 0.6889,
|
|
"total": 1485,
|
|
"trueNegative": 1392,
|
|
"truePositive": 62,
|
|
"wilsonPrecisionLower95": 0.8729
|
|
},
|
|
"threshold": 0.695
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.731,
|
|
"falseNegative": 37,
|
|
"falsePositive": 2,
|
|
"precision": 0.9636,
|
|
"predictedPositive": 55,
|
|
"recall": 0.5889,
|
|
"total": 1485,
|
|
"trueNegative": 1393,
|
|
"truePositive": 53,
|
|
"wilsonPrecisionLower95": 0.8768
|
|
},
|
|
"threshold": 0.5
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.7354,
|
|
"falseNegative": 73,
|
|
"falsePositive": 4,
|
|
"precision": 0.964,
|
|
"predictedPositive": 111,
|
|
"recall": 0.5944,
|
|
"total": 2970,
|
|
"trueNegative": 2786,
|
|
"truePositive": 107,
|
|
"wilsonPrecisionLower95": 0.911
|
|
},
|
|
"threshold": 0.695
|
|
},
|
|
"followUpReminder": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8632,
|
|
"falseNegative": 38,
|
|
"falsePositive": 7,
|
|
"precision": 0.953,
|
|
"predictedPositive": 149,
|
|
"recall": 0.7889,
|
|
"total": 1485,
|
|
"trueNegative": 1298,
|
|
"truePositive": 142,
|
|
"wilsonPrecisionLower95": 0.9062
|
|
},
|
|
"threshold": 0.9
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.4938,
|
|
"falseNegative": 120,
|
|
"falsePositive": 3,
|
|
"precision": 0.9524,
|
|
"predictedPositive": 63,
|
|
"recall": 0.3333,
|
|
"total": 1485,
|
|
"trueNegative": 1302,
|
|
"truePositive": 60,
|
|
"wilsonPrecisionLower95": 0.8691
|
|
},
|
|
"threshold": 0.845
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.7018,
|
|
"falseNegative": 160,
|
|
"falsePositive": 10,
|
|
"precision": 0.9524,
|
|
"predictedPositive": 210,
|
|
"recall": 0.5556,
|
|
"total": 2970,
|
|
"trueNegative": 2600,
|
|
"truePositive": 200,
|
|
"wilsonPrecisionLower95": 0.9146
|
|
},
|
|
"threshold": 0.885
|
|
},
|
|
"invitation": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.7222,
|
|
"falseNegative": 38,
|
|
"falsePositive": 2,
|
|
"precision": 0.963,
|
|
"predictedPositive": 54,
|
|
"recall": 0.5778,
|
|
"total": 1485,
|
|
"trueNegative": 1393,
|
|
"truePositive": 52,
|
|
"wilsonPrecisionLower95": 0.8746
|
|
},
|
|
"threshold": 0.665
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"question": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8167,
|
|
"falseNegative": 90,
|
|
"falsePositive": 11,
|
|
"precision": 0.9534,
|
|
"predictedPositive": 236,
|
|
"recall": 0.7143,
|
|
"total": 1485,
|
|
"trueNegative": 1159,
|
|
"truePositive": 225,
|
|
"wilsonPrecisionLower95": 0.9185
|
|
},
|
|
"threshold": 0.865
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8205,
|
|
"falseNegative": 91,
|
|
"falsePositive": 7,
|
|
"precision": 0.9697,
|
|
"predictedPositive": 231,
|
|
"recall": 0.7111,
|
|
"total": 1485,
|
|
"trueNegative": 1163,
|
|
"truePositive": 224,
|
|
"wilsonPrecisionLower95": 0.9388
|
|
},
|
|
"threshold": 0.5
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.804,
|
|
"falseNegative": 191,
|
|
"falsePositive": 23,
|
|
"precision": 0.9502,
|
|
"predictedPositive": 462,
|
|
"recall": 0.6968,
|
|
"total": 2970,
|
|
"trueNegative": 2317,
|
|
"truePositive": 439,
|
|
"wilsonPrecisionLower95": 0.9264
|
|
},
|
|
"threshold": 0.68
|
|
},
|
|
"replyableMessage": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9268,
|
|
"falseNegative": 90,
|
|
"falsePositive": 45,
|
|
"precision": 0.95,
|
|
"predictedPositive": 900,
|
|
"recall": 0.9048,
|
|
"total": 1485,
|
|
"trueNegative": 495,
|
|
"truePositive": 855,
|
|
"wilsonPrecisionLower95": 0.9338
|
|
},
|
|
"threshold": 0.86
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8789,
|
|
"falseNegative": 176,
|
|
"falsePositive": 36,
|
|
"precision": 0.9553,
|
|
"predictedPositive": 805,
|
|
"recall": 0.8138,
|
|
"total": 1485,
|
|
"trueNegative": 504,
|
|
"truePositive": 769,
|
|
"wilsonPrecisionLower95": 0.9387
|
|
},
|
|
"threshold": 0.85
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.9046,
|
|
"falseNegative": 259,
|
|
"falsePositive": 85,
|
|
"precision": 0.9505,
|
|
"predictedPositive": 1716,
|
|
"recall": 0.863,
|
|
"total": 2970,
|
|
"trueNegative": 995,
|
|
"truePositive": 1631,
|
|
"wilsonPrecisionLower95": 0.9392
|
|
},
|
|
"threshold": 0.85
|
|
},
|
|
"scheduleNegotiation": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9773,
|
|
"falseNegative": 2,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 43,
|
|
"recall": 0.9556,
|
|
"total": 1485,
|
|
"trueNegative": 1440,
|
|
"truePositive": 43,
|
|
"wilsonPrecisionLower95": 0.918
|
|
},
|
|
"threshold": 0.5
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9888,
|
|
"falseNegative": 1,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 44,
|
|
"recall": 0.9778,
|
|
"total": 1485,
|
|
"trueNegative": 1440,
|
|
"truePositive": 44,
|
|
"wilsonPrecisionLower95": 0.9197
|
|
},
|
|
"threshold": 0.97
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.9497,
|
|
"falseNegative": 5,
|
|
"falsePositive": 4,
|
|
"precision": 0.9551,
|
|
"predictedPositive": 89,
|
|
"recall": 0.9444,
|
|
"total": 2970,
|
|
"trueNegative": 2876,
|
|
"truePositive": 85,
|
|
"wilsonPrecisionLower95": 0.8901
|
|
},
|
|
"threshold": 0.91
|
|
},
|
|
"task": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.827,
|
|
"falseNegative": 97,
|
|
"falsePositive": 13,
|
|
"precision": 0.9529,
|
|
"predictedPositive": 276,
|
|
"recall": 0.7306,
|
|
"total": 1485,
|
|
"trueNegative": 1112,
|
|
"truePositive": 263,
|
|
"wilsonPrecisionLower95": 0.9211
|
|
},
|
|
"threshold": 0.89
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.7902,
|
|
"falseNegative": 117,
|
|
"falsePositive": 12,
|
|
"precision": 0.9529,
|
|
"predictedPositive": 255,
|
|
"recall": 0.675,
|
|
"total": 1485,
|
|
"trueNegative": 1113,
|
|
"truePositive": 243,
|
|
"wilsonPrecisionLower95": 0.9196
|
|
},
|
|
"threshold": 0.785
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.7783,
|
|
"falseNegative": 246,
|
|
"falsePositive": 24,
|
|
"precision": 0.9518,
|
|
"predictedPositive": 498,
|
|
"recall": 0.6583,
|
|
"total": 2970,
|
|
"trueNegative": 2226,
|
|
"truePositive": 474,
|
|
"wilsonPrecisionLower95": 0.9293
|
|
},
|
|
"threshold": 0.85
|
|
}
|
|
},
|
|
"trainingRecords": 17767,
|
|
"validation": {
|
|
"intentsMeeting95Precision": 8,
|
|
"macroF1": 0.7395,
|
|
"macroPrecision": 0.9687,
|
|
"macroRecall": 0.6518,
|
|
"minimumPrecision": 0.9525,
|
|
"perIntent": {
|
|
"blessing": {
|
|
"f1": 0.1633,
|
|
"falseNegative": 82,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 8,
|
|
"recall": 0.0889,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 8,
|
|
"wilsonPrecisionLower95": 0.6756
|
|
},
|
|
"complaint": {
|
|
"f1": 0.9666,
|
|
"falseNegative": 13,
|
|
"falsePositive": 11,
|
|
"precision": 0.9693,
|
|
"predictedPositive": 358,
|
|
"recall": 0.9639,
|
|
"total": 2970,
|
|
"trueNegative": 2599,
|
|
"truePositive": 347,
|
|
"wilsonPrecisionLower95": 0.9458
|
|
},
|
|
"confirmationDecision": {
|
|
"f1": 0.7667,
|
|
"falseNegative": 65,
|
|
"falsePositive": 5,
|
|
"precision": 0.9583,
|
|
"predictedPositive": 120,
|
|
"recall": 0.6389,
|
|
"total": 2970,
|
|
"trueNegative": 2785,
|
|
"truePositive": 115,
|
|
"wilsonPrecisionLower95": 0.9062
|
|
},
|
|
"followUpReminder": {
|
|
"f1": 0.7063,
|
|
"falseNegative": 158,
|
|
"falsePositive": 10,
|
|
"precision": 0.9528,
|
|
"predictedPositive": 212,
|
|
"recall": 0.5611,
|
|
"total": 2970,
|
|
"trueNegative": 2600,
|
|
"truePositive": 202,
|
|
"wilsonPrecisionLower95": 0.9154
|
|
},
|
|
"invitation": {
|
|
"f1": 0.5382,
|
|
"falseNegative": 113,
|
|
"falsePositive": 2,
|
|
"precision": 0.971,
|
|
"predictedPositive": 69,
|
|
"recall": 0.3722,
|
|
"total": 2970,
|
|
"trueNegative": 2788,
|
|
"truePositive": 67,
|
|
"wilsonPrecisionLower95": 0.9003
|
|
},
|
|
"question": {
|
|
"f1": 0.8186,
|
|
"falseNegative": 181,
|
|
"falsePositive": 18,
|
|
"precision": 0.9615,
|
|
"predictedPositive": 467,
|
|
"recall": 0.7127,
|
|
"total": 2970,
|
|
"trueNegative": 2322,
|
|
"truePositive": 449,
|
|
"wilsonPrecisionLower95": 0.9399
|
|
},
|
|
"replyableMessage": {
|
|
"f1": 0.9035,
|
|
"falseNegative": 266,
|
|
"falsePositive": 81,
|
|
"precision": 0.9525,
|
|
"predictedPositive": 1705,
|
|
"recall": 0.8593,
|
|
"total": 2970,
|
|
"trueNegative": 999,
|
|
"truePositive": 1624,
|
|
"wilsonPrecisionLower95": 0.9413
|
|
},
|
|
"scheduleNegotiation": {
|
|
"f1": 0.9831,
|
|
"falseNegative": 3,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 87,
|
|
"recall": 0.9667,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 87,
|
|
"wilsonPrecisionLower95": 0.9577
|
|
},
|
|
"task": {
|
|
"f1": 0.809,
|
|
"falseNegative": 214,
|
|
"falsePositive": 25,
|
|
"precision": 0.9529,
|
|
"predictedPositive": 531,
|
|
"recall": 0.7028,
|
|
"total": 2970,
|
|
"trueNegative": 2225,
|
|
"truePositive": 506,
|
|
"wilsonPrecisionLower95": 0.9314
|
|
}
|
|
},
|
|
"recordCount": 2970
|
|
}
|
|
},
|
|
{
|
|
"configuration": {
|
|
"alpha": 6e-06,
|
|
"augmentation": "punctuation",
|
|
"char_max": 6,
|
|
"char_min": 3,
|
|
"external_weight": 0.6900000000000001,
|
|
"hard_example_weight": 1.5,
|
|
"l1_ratio": 0.1,
|
|
"max_char_features": 65000,
|
|
"max_word_features": 32000,
|
|
"min_df": 2,
|
|
"pseudo_weight": 0.255,
|
|
"round": 7,
|
|
"word_max": 3
|
|
},
|
|
"featureCount": 87507,
|
|
"pseudoLabelCounts": {
|
|
"blessing": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"complaint": {
|
|
"negative": 141,
|
|
"positive": 0
|
|
},
|
|
"confirmationDecision": {
|
|
"negative": 95,
|
|
"positive": 2
|
|
},
|
|
"followUpReminder": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"invitation": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"question": {
|
|
"negative": 36,
|
|
"positive": 5
|
|
},
|
|
"replyableMessage": {
|
|
"negative": 2,
|
|
"positive": 129
|
|
},
|
|
"scheduleNegotiation": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"task": {
|
|
"negative": 87,
|
|
"positive": 0
|
|
}
|
|
},
|
|
"round": 7,
|
|
"selectionObjective": 0.7501,
|
|
"syntheticRegression": {
|
|
"intentsMeeting95Precision": 5,
|
|
"macroF1": 0.587,
|
|
"macroPrecision": 0.9374,
|
|
"macroRecall": 0.4917,
|
|
"minimumPrecision": 0.7548,
|
|
"perIntent": {
|
|
"blessing": {
|
|
"f1": 0.0851,
|
|
"falseNegative": 86,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 4,
|
|
"recall": 0.0444,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 4,
|
|
"wilsonPrecisionLower95": 0.5101
|
|
},
|
|
"complaint": {
|
|
"f1": 0.9304,
|
|
"falseNegative": 39,
|
|
"falsePositive": 9,
|
|
"precision": 0.9727,
|
|
"predictedPositive": 330,
|
|
"recall": 0.8917,
|
|
"total": 2970,
|
|
"trueNegative": 2601,
|
|
"truePositive": 321,
|
|
"wilsonPrecisionLower95": 0.949
|
|
},
|
|
"confirmationDecision": {
|
|
"f1": 0.6985,
|
|
"falseNegative": 63,
|
|
"falsePositive": 38,
|
|
"precision": 0.7548,
|
|
"predictedPositive": 155,
|
|
"recall": 0.65,
|
|
"total": 2970,
|
|
"trueNegative": 2752,
|
|
"truePositive": 117,
|
|
"wilsonPrecisionLower95": 0.6815
|
|
},
|
|
"followUpReminder": {
|
|
"f1": 0.3974,
|
|
"falseNegative": 269,
|
|
"falsePositive": 7,
|
|
"precision": 0.9286,
|
|
"predictedPositive": 98,
|
|
"recall": 0.2528,
|
|
"total": 2970,
|
|
"trueNegative": 2603,
|
|
"truePositive": 91,
|
|
"wilsonPrecisionLower95": 0.8598
|
|
},
|
|
"invitation": {
|
|
"f1": 0.2609,
|
|
"falseNegative": 153,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 27,
|
|
"recall": 0.15,
|
|
"total": 2970,
|
|
"trueNegative": 2790,
|
|
"truePositive": 27,
|
|
"wilsonPrecisionLower95": 0.8754
|
|
},
|
|
"question": {
|
|
"f1": 0.697,
|
|
"falseNegative": 285,
|
|
"falsePositive": 15,
|
|
"precision": 0.9583,
|
|
"predictedPositive": 360,
|
|
"recall": 0.5476,
|
|
"total": 2970,
|
|
"trueNegative": 2325,
|
|
"truePositive": 345,
|
|
"wilsonPrecisionLower95": 0.9324
|
|
},
|
|
"replyableMessage": {
|
|
"f1": 0.8083,
|
|
"falseNegative": 572,
|
|
"falsePositive": 53,
|
|
"precision": 0.9613,
|
|
"predictedPositive": 1371,
|
|
"recall": 0.6974,
|
|
"total": 2970,
|
|
"trueNegative": 1027,
|
|
"truePositive": 1318,
|
|
"wilsonPrecisionLower95": 0.9498
|
|
},
|
|
"scheduleNegotiation": {
|
|
"f1": 0.8929,
|
|
"falseNegative": 15,
|
|
"falsePositive": 3,
|
|
"precision": 0.9615,
|
|
"predictedPositive": 78,
|
|
"recall": 0.8333,
|
|
"total": 2970,
|
|
"trueNegative": 2877,
|
|
"truePositive": 75,
|
|
"wilsonPrecisionLower95": 0.8929
|
|
},
|
|
"task": {
|
|
"f1": 0.5124,
|
|
"falseNegative": 462,
|
|
"falsePositive": 29,
|
|
"precision": 0.899,
|
|
"predictedPositive": 287,
|
|
"recall": 0.3583,
|
|
"total": 2970,
|
|
"trueNegative": 2221,
|
|
"truePositive": 258,
|
|
"wilsonPrecisionLower95": 0.8586
|
|
}
|
|
},
|
|
"recordCount": 2970
|
|
},
|
|
"thresholds": {
|
|
"blessing": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"complaint": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9809,
|
|
"falseNegative": 0,
|
|
"falsePositive": 7,
|
|
"precision": 0.9626,
|
|
"predictedPositive": 187,
|
|
"recall": 1.0,
|
|
"total": 1485,
|
|
"trueNegative": 1298,
|
|
"truePositive": 180,
|
|
"wilsonPrecisionLower95": 0.9248
|
|
},
|
|
"threshold": 0.74
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8974,
|
|
"falseNegative": 27,
|
|
"falsePositive": 8,
|
|
"precision": 0.9503,
|
|
"predictedPositive": 161,
|
|
"recall": 0.85,
|
|
"total": 1485,
|
|
"trueNegative": 1297,
|
|
"truePositive": 153,
|
|
"wilsonPrecisionLower95": 0.905
|
|
},
|
|
"threshold": 0.665
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.9529,
|
|
"falseNegative": 16,
|
|
"falsePositive": 18,
|
|
"precision": 0.9503,
|
|
"predictedPositive": 362,
|
|
"recall": 0.9556,
|
|
"total": 2970,
|
|
"trueNegative": 2592,
|
|
"truePositive": 344,
|
|
"wilsonPrecisionLower95": 0.9228
|
|
},
|
|
"threshold": 0.51
|
|
},
|
|
"confirmationDecision": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.7763,
|
|
"falseNegative": 31,
|
|
"falsePositive": 3,
|
|
"precision": 0.9516,
|
|
"predictedPositive": 62,
|
|
"recall": 0.6556,
|
|
"total": 1485,
|
|
"trueNegative": 1392,
|
|
"truePositive": 59,
|
|
"wilsonPrecisionLower95": 0.8671
|
|
},
|
|
"threshold": 0.895
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.7568,
|
|
"falseNegative": 34,
|
|
"falsePositive": 2,
|
|
"precision": 0.9655,
|
|
"predictedPositive": 58,
|
|
"recall": 0.6222,
|
|
"total": 1485,
|
|
"trueNegative": 1393,
|
|
"truePositive": 56,
|
|
"wilsonPrecisionLower95": 0.8827
|
|
},
|
|
"threshold": 0.5
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.6569,
|
|
"falseNegative": 90,
|
|
"falsePositive": 4,
|
|
"precision": 0.9574,
|
|
"predictedPositive": 94,
|
|
"recall": 0.5,
|
|
"total": 2970,
|
|
"trueNegative": 2786,
|
|
"truePositive": 90,
|
|
"wilsonPrecisionLower95": 0.8956
|
|
},
|
|
"threshold": 0.895
|
|
},
|
|
"followUpReminder": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8421,
|
|
"falseNegative": 44,
|
|
"falsePositive": 7,
|
|
"precision": 0.951,
|
|
"predictedPositive": 143,
|
|
"recall": 0.7556,
|
|
"total": 1485,
|
|
"trueNegative": 1298,
|
|
"truePositive": 136,
|
|
"wilsonPrecisionLower95": 0.9024
|
|
},
|
|
"threshold": 0.985
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.5985,
|
|
"falseNegative": 205,
|
|
"falsePositive": 3,
|
|
"precision": 0.981,
|
|
"predictedPositive": 158,
|
|
"recall": 0.4306,
|
|
"total": 2970,
|
|
"trueNegative": 2607,
|
|
"truePositive": 155,
|
|
"wilsonPrecisionLower95": 0.9457
|
|
},
|
|
"threshold": 0.995
|
|
},
|
|
"invitation": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"question": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.7143,
|
|
"falseNegative": 135,
|
|
"falsePositive": 9,
|
|
"precision": 0.9524,
|
|
"predictedPositive": 189,
|
|
"recall": 0.5714,
|
|
"total": 1485,
|
|
"trueNegative": 1161,
|
|
"truePositive": 180,
|
|
"wilsonPrecisionLower95": 0.912
|
|
},
|
|
"threshold": 0.965
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.7168,
|
|
"falseNegative": 134,
|
|
"falsePositive": 9,
|
|
"precision": 0.9526,
|
|
"predictedPositive": 190,
|
|
"recall": 0.5746,
|
|
"total": 1485,
|
|
"trueNegative": 1161,
|
|
"truePositive": 181,
|
|
"wilsonPrecisionLower95": 0.9124
|
|
},
|
|
"threshold": 0.52
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.7149,
|
|
"falseNegative": 269,
|
|
"falsePositive": 19,
|
|
"precision": 0.95,
|
|
"predictedPositive": 380,
|
|
"recall": 0.573,
|
|
"total": 2970,
|
|
"trueNegative": 2321,
|
|
"truePositive": 361,
|
|
"wilsonPrecisionLower95": 0.9232
|
|
},
|
|
"threshold": 0.795
|
|
},
|
|
"replyableMessage": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8846,
|
|
"falseNegative": 163,
|
|
"falsePositive": 41,
|
|
"precision": 0.9502,
|
|
"predictedPositive": 823,
|
|
"recall": 0.8275,
|
|
"total": 1485,
|
|
"trueNegative": 499,
|
|
"truePositive": 782,
|
|
"wilsonPrecisionLower95": 0.9331
|
|
},
|
|
"threshold": 0.89
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.7476,
|
|
"falseNegative": 363,
|
|
"falsePositive": 30,
|
|
"precision": 0.951,
|
|
"predictedPositive": 612,
|
|
"recall": 0.6159,
|
|
"total": 1485,
|
|
"trueNegative": 510,
|
|
"truePositive": 582,
|
|
"wilsonPrecisionLower95": 0.9309
|
|
},
|
|
"threshold": 0.92
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.828,
|
|
"falseNegative": 506,
|
|
"falsePositive": 69,
|
|
"precision": 0.9525,
|
|
"predictedPositive": 1453,
|
|
"recall": 0.7323,
|
|
"total": 2970,
|
|
"trueNegative": 1011,
|
|
"truePositive": 1384,
|
|
"wilsonPrecisionLower95": 0.9403
|
|
},
|
|
"threshold": 0.91
|
|
},
|
|
"scheduleNegotiation": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 1.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 45,
|
|
"recall": 1.0,
|
|
"total": 1485,
|
|
"trueNegative": 1440,
|
|
"truePositive": 45,
|
|
"wilsonPrecisionLower95": 0.9213
|
|
},
|
|
"threshold": 0.66
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 1.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 45,
|
|
"recall": 1.0,
|
|
"total": 1485,
|
|
"trueNegative": 1440,
|
|
"truePositive": 45,
|
|
"wilsonPrecisionLower95": 0.9213
|
|
},
|
|
"threshold": 0.55
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 1.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 90,
|
|
"recall": 1.0,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 90,
|
|
"wilsonPrecisionLower95": 0.9591
|
|
},
|
|
"threshold": 0.66
|
|
},
|
|
"task": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8615,
|
|
"falseNegative": 77,
|
|
"falsePositive": 14,
|
|
"precision": 0.9529,
|
|
"predictedPositive": 297,
|
|
"recall": 0.7861,
|
|
"total": 1485,
|
|
"trueNegative": 1111,
|
|
"truePositive": 283,
|
|
"wilsonPrecisionLower95": 0.9224
|
|
},
|
|
"threshold": 0.805
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.6868,
|
|
"falseNegative": 333,
|
|
"falsePositive": 20,
|
|
"precision": 0.9509,
|
|
"predictedPositive": 407,
|
|
"recall": 0.5375,
|
|
"total": 2970,
|
|
"trueNegative": 2230,
|
|
"truePositive": 387,
|
|
"wilsonPrecisionLower95": 0.9253
|
|
},
|
|
"threshold": 0.925
|
|
}
|
|
},
|
|
"trainingRecords": 17708,
|
|
"validation": {
|
|
"intentsMeeting95Precision": 8,
|
|
"macroF1": 0.6491,
|
|
"macroPrecision": 0.967,
|
|
"macroRecall": 0.5465,
|
|
"minimumPrecision": 0.9315,
|
|
"perIntent": {
|
|
"blessing": {
|
|
"f1": 0.2353,
|
|
"falseNegative": 78,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 12,
|
|
"recall": 0.1333,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 12,
|
|
"wilsonPrecisionLower95": 0.7575
|
|
},
|
|
"complaint": {
|
|
"f1": 0.9407,
|
|
"falseNegative": 27,
|
|
"falsePositive": 15,
|
|
"precision": 0.9569,
|
|
"predictedPositive": 348,
|
|
"recall": 0.925,
|
|
"total": 2970,
|
|
"trueNegative": 2595,
|
|
"truePositive": 333,
|
|
"wilsonPrecisionLower95": 0.9301
|
|
},
|
|
"confirmationDecision": {
|
|
"f1": 0.7667,
|
|
"falseNegative": 65,
|
|
"falsePositive": 5,
|
|
"precision": 0.9583,
|
|
"predictedPositive": 120,
|
|
"recall": 0.6389,
|
|
"total": 2970,
|
|
"trueNegative": 2785,
|
|
"truePositive": 115,
|
|
"wilsonPrecisionLower95": 0.9062
|
|
},
|
|
"followUpReminder": {
|
|
"f1": 0.5375,
|
|
"falseNegative": 224,
|
|
"falsePositive": 10,
|
|
"precision": 0.9315,
|
|
"predictedPositive": 146,
|
|
"recall": 0.3778,
|
|
"total": 2970,
|
|
"trueNegative": 2600,
|
|
"truePositive": 136,
|
|
"wilsonPrecisionLower95": 0.8785
|
|
},
|
|
"invitation": {
|
|
"f1": 0.2692,
|
|
"falseNegative": 152,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 28,
|
|
"recall": 0.1556,
|
|
"total": 2970,
|
|
"trueNegative": 2790,
|
|
"truePositive": 28,
|
|
"wilsonPrecisionLower95": 0.8794
|
|
},
|
|
"question": {
|
|
"f1": 0.7156,
|
|
"falseNegative": 269,
|
|
"falsePositive": 18,
|
|
"precision": 0.9525,
|
|
"predictedPositive": 379,
|
|
"recall": 0.573,
|
|
"total": 2970,
|
|
"trueNegative": 2322,
|
|
"truePositive": 361,
|
|
"wilsonPrecisionLower95": 0.9262
|
|
},
|
|
"replyableMessage": {
|
|
"f1": 0.8205,
|
|
"falseNegative": 526,
|
|
"falsePositive": 71,
|
|
"precision": 0.9505,
|
|
"predictedPositive": 1435,
|
|
"recall": 0.7217,
|
|
"total": 2970,
|
|
"trueNegative": 1009,
|
|
"truePositive": 1364,
|
|
"wilsonPrecisionLower95": 0.9381
|
|
},
|
|
"scheduleNegotiation": {
|
|
"f1": 1.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 90,
|
|
"recall": 1.0,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 90,
|
|
"wilsonPrecisionLower95": 0.9591
|
|
},
|
|
"task": {
|
|
"f1": 0.5565,
|
|
"falseNegative": 437,
|
|
"falsePositive": 14,
|
|
"precision": 0.9529,
|
|
"predictedPositive": 297,
|
|
"recall": 0.3931,
|
|
"total": 2970,
|
|
"trueNegative": 2236,
|
|
"truePositive": 283,
|
|
"wilsonPrecisionLower95": 0.9224
|
|
}
|
|
},
|
|
"recordCount": 2970
|
|
}
|
|
},
|
|
{
|
|
"configuration": {
|
|
"alpha": 5e-06,
|
|
"augmentation": "punctuation",
|
|
"char_max": 7,
|
|
"char_min": 2,
|
|
"external_weight": 0.7100000000000001,
|
|
"hard_example_weight": 1.7,
|
|
"l1_ratio": 0.15,
|
|
"max_char_features": 70000,
|
|
"max_word_features": 28000,
|
|
"min_df": 2,
|
|
"pseudo_weight": 0.27,
|
|
"round": 8,
|
|
"word_max": 2
|
|
},
|
|
"featureCount": 83313,
|
|
"pseudoLabelCounts": {
|
|
"blessing": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"complaint": {
|
|
"negative": 187,
|
|
"positive": 0
|
|
},
|
|
"confirmationDecision": {
|
|
"negative": 107,
|
|
"positive": 2
|
|
},
|
|
"followUpReminder": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"invitation": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"question": {
|
|
"negative": 40,
|
|
"positive": 5
|
|
},
|
|
"replyableMessage": {
|
|
"negative": 2,
|
|
"positive": 135
|
|
},
|
|
"scheduleNegotiation": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"task": {
|
|
"negative": 94,
|
|
"positive": 0
|
|
}
|
|
},
|
|
"round": 8,
|
|
"selectionObjective": 0.7837,
|
|
"syntheticRegression": {
|
|
"intentsMeeting95Precision": 4,
|
|
"macroF1": 0.6947,
|
|
"macroPrecision": 0.9526,
|
|
"macroRecall": 0.6066,
|
|
"minimumPrecision": 0.8509,
|
|
"perIntent": {
|
|
"blessing": {
|
|
"f1": 0.125,
|
|
"falseNegative": 84,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 6,
|
|
"recall": 0.0667,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 6,
|
|
"wilsonPrecisionLower95": 0.6097
|
|
},
|
|
"complaint": {
|
|
"f1": 0.9744,
|
|
"falseNegative": 18,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 342,
|
|
"recall": 0.95,
|
|
"total": 2970,
|
|
"trueNegative": 2610,
|
|
"truePositive": 342,
|
|
"wilsonPrecisionLower95": 0.9889
|
|
},
|
|
"confirmationDecision": {
|
|
"f1": 0.8035,
|
|
"falseNegative": 43,
|
|
"falsePositive": 24,
|
|
"precision": 0.8509,
|
|
"predictedPositive": 161,
|
|
"recall": 0.7611,
|
|
"total": 2970,
|
|
"trueNegative": 2766,
|
|
"truePositive": 137,
|
|
"wilsonPrecisionLower95": 0.7878
|
|
},
|
|
"followUpReminder": {
|
|
"f1": 0.5422,
|
|
"falseNegative": 222,
|
|
"falsePositive": 11,
|
|
"precision": 0.9262,
|
|
"predictedPositive": 149,
|
|
"recall": 0.3833,
|
|
"total": 2970,
|
|
"trueNegative": 2599,
|
|
"truePositive": 138,
|
|
"wilsonPrecisionLower95": 0.8726
|
|
},
|
|
"invitation": {
|
|
"f1": 0.5062,
|
|
"falseNegative": 119,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 61,
|
|
"recall": 0.3389,
|
|
"total": 2970,
|
|
"trueNegative": 2790,
|
|
"truePositive": 61,
|
|
"wilsonPrecisionLower95": 0.9408
|
|
},
|
|
"question": {
|
|
"f1": 0.7719,
|
|
"falseNegative": 229,
|
|
"falsePositive": 8,
|
|
"precision": 0.9804,
|
|
"predictedPositive": 409,
|
|
"recall": 0.6365,
|
|
"total": 2970,
|
|
"trueNegative": 2332,
|
|
"truePositive": 401,
|
|
"wilsonPrecisionLower95": 0.9619
|
|
},
|
|
"replyableMessage": {
|
|
"f1": 0.8832,
|
|
"falseNegative": 366,
|
|
"falsePositive": 37,
|
|
"precision": 0.9763,
|
|
"predictedPositive": 1561,
|
|
"recall": 0.8063,
|
|
"total": 2970,
|
|
"trueNegative": 1043,
|
|
"truePositive": 1524,
|
|
"wilsonPrecisionLower95": 0.9675
|
|
},
|
|
"scheduleNegotiation": {
|
|
"f1": 0.9162,
|
|
"falseNegative": 8,
|
|
"falsePositive": 7,
|
|
"precision": 0.9213,
|
|
"predictedPositive": 89,
|
|
"recall": 0.9111,
|
|
"total": 2970,
|
|
"trueNegative": 2873,
|
|
"truePositive": 82,
|
|
"wilsonPrecisionLower95": 0.8464
|
|
},
|
|
"task": {
|
|
"f1": 0.7297,
|
|
"falseNegative": 284,
|
|
"falsePositive": 39,
|
|
"precision": 0.9179,
|
|
"predictedPositive": 475,
|
|
"recall": 0.6056,
|
|
"total": 2970,
|
|
"trueNegative": 2211,
|
|
"truePositive": 436,
|
|
"wilsonPrecisionLower95": 0.8897
|
|
}
|
|
},
|
|
"recordCount": 2970
|
|
},
|
|
"thresholds": {
|
|
"blessing": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"complaint": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.989,
|
|
"falseNegative": 1,
|
|
"falsePositive": 3,
|
|
"precision": 0.9835,
|
|
"predictedPositive": 182,
|
|
"recall": 0.9944,
|
|
"total": 1485,
|
|
"trueNegative": 1302,
|
|
"truePositive": 179,
|
|
"wilsonPrecisionLower95": 0.9527
|
|
},
|
|
"threshold": 0.97
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9405,
|
|
"falseNegative": 14,
|
|
"falsePositive": 7,
|
|
"precision": 0.9595,
|
|
"predictedPositive": 173,
|
|
"recall": 0.9222,
|
|
"total": 1485,
|
|
"trueNegative": 1298,
|
|
"truePositive": 166,
|
|
"wilsonPrecisionLower95": 0.9188
|
|
},
|
|
"threshold": 0.54
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.9237,
|
|
"falseNegative": 39,
|
|
"falsePositive": 14,
|
|
"precision": 0.9582,
|
|
"predictedPositive": 335,
|
|
"recall": 0.8917,
|
|
"total": 2970,
|
|
"trueNegative": 2596,
|
|
"truePositive": 321,
|
|
"wilsonPrecisionLower95": 0.9311
|
|
},
|
|
"threshold": 0.925
|
|
},
|
|
"confirmationDecision": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9613,
|
|
"falseNegative": 3,
|
|
"falsePositive": 4,
|
|
"precision": 0.956,
|
|
"predictedPositive": 91,
|
|
"recall": 0.9667,
|
|
"total": 1485,
|
|
"trueNegative": 1391,
|
|
"truePositive": 87,
|
|
"wilsonPrecisionLower95": 0.8924
|
|
},
|
|
"threshold": 0.72
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.5556,
|
|
"falseNegative": 55,
|
|
"falsePositive": 1,
|
|
"precision": 0.9722,
|
|
"predictedPositive": 36,
|
|
"recall": 0.3889,
|
|
"total": 1485,
|
|
"trueNegative": 1394,
|
|
"truePositive": 35,
|
|
"wilsonPrecisionLower95": 0.8583
|
|
},
|
|
"threshold": 0.905
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.7922,
|
|
"falseNegative": 58,
|
|
"falsePositive": 6,
|
|
"precision": 0.9531,
|
|
"predictedPositive": 128,
|
|
"recall": 0.6778,
|
|
"total": 2970,
|
|
"trueNegative": 2784,
|
|
"truePositive": 122,
|
|
"wilsonPrecisionLower95": 0.9015
|
|
},
|
|
"threshold": 0.88
|
|
},
|
|
"followUpReminder": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8437,
|
|
"falseNegative": 45,
|
|
"falsePositive": 5,
|
|
"precision": 0.9643,
|
|
"predictedPositive": 140,
|
|
"recall": 0.75,
|
|
"total": 1485,
|
|
"trueNegative": 1300,
|
|
"truePositive": 135,
|
|
"wilsonPrecisionLower95": 0.9191
|
|
},
|
|
"threshold": 0.96
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.7063,
|
|
"falseNegative": 158,
|
|
"falsePositive": 10,
|
|
"precision": 0.9528,
|
|
"predictedPositive": 212,
|
|
"recall": 0.5611,
|
|
"total": 2970,
|
|
"trueNegative": 2600,
|
|
"truePositive": 202,
|
|
"wilsonPrecisionLower95": 0.9154
|
|
},
|
|
"threshold": 0.96
|
|
},
|
|
"invitation": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"question": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.7157,
|
|
"falseNegative": 135,
|
|
"falsePositive": 8,
|
|
"precision": 0.9574,
|
|
"predictedPositive": 188,
|
|
"recall": 0.5714,
|
|
"total": 1485,
|
|
"trueNegative": 1162,
|
|
"truePositive": 180,
|
|
"wilsonPrecisionLower95": 0.9183
|
|
},
|
|
"threshold": 0.975
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.822,
|
|
"falseNegative": 91,
|
|
"falsePositive": 6,
|
|
"precision": 0.9739,
|
|
"predictedPositive": 230,
|
|
"recall": 0.7111,
|
|
"total": 1485,
|
|
"trueNegative": 1164,
|
|
"truePositive": 224,
|
|
"wilsonPrecisionLower95": 0.9443
|
|
},
|
|
"threshold": 0.505
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.8095,
|
|
"falseNegative": 186,
|
|
"falsePositive": 23,
|
|
"precision": 0.9507,
|
|
"predictedPositive": 467,
|
|
"recall": 0.7048,
|
|
"total": 2970,
|
|
"trueNegative": 2317,
|
|
"truePositive": 444,
|
|
"wilsonPrecisionLower95": 0.9272
|
|
},
|
|
"threshold": 0.71
|
|
},
|
|
"replyableMessage": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8741,
|
|
"falseNegative": 181,
|
|
"falsePositive": 39,
|
|
"precision": 0.9514,
|
|
"predictedPositive": 803,
|
|
"recall": 0.8085,
|
|
"total": 1485,
|
|
"trueNegative": 501,
|
|
"truePositive": 764,
|
|
"wilsonPrecisionLower95": 0.9343
|
|
},
|
|
"threshold": 0.95
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8775,
|
|
"falseNegative": 175,
|
|
"falsePositive": 40,
|
|
"precision": 0.9506,
|
|
"predictedPositive": 810,
|
|
"recall": 0.8148,
|
|
"total": 1485,
|
|
"trueNegative": 500,
|
|
"truePositive": 770,
|
|
"wilsonPrecisionLower95": 0.9335
|
|
},
|
|
"threshold": 0.88
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.8762,
|
|
"falseNegative": 354,
|
|
"falsePositive": 80,
|
|
"precision": 0.9505,
|
|
"predictedPositive": 1616,
|
|
"recall": 0.8127,
|
|
"total": 2970,
|
|
"trueNegative": 1000,
|
|
"truePositive": 1536,
|
|
"wilsonPrecisionLower95": 0.9388
|
|
},
|
|
"threshold": 0.905
|
|
},
|
|
"scheduleNegotiation": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 1.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 45,
|
|
"recall": 1.0,
|
|
"total": 1485,
|
|
"trueNegative": 1440,
|
|
"truePositive": 45,
|
|
"wilsonPrecisionLower95": 0.9213
|
|
},
|
|
"threshold": 0.5
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9556,
|
|
"falseNegative": 2,
|
|
"falsePositive": 2,
|
|
"precision": 0.9556,
|
|
"predictedPositive": 45,
|
|
"recall": 0.9556,
|
|
"total": 1485,
|
|
"trueNegative": 1438,
|
|
"truePositive": 43,
|
|
"wilsonPrecisionLower95": 0.8517
|
|
},
|
|
"threshold": 0.75
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.9724,
|
|
"falseNegative": 2,
|
|
"falsePositive": 3,
|
|
"precision": 0.967,
|
|
"predictedPositive": 91,
|
|
"recall": 0.9778,
|
|
"total": 2970,
|
|
"trueNegative": 2877,
|
|
"truePositive": 88,
|
|
"wilsonPrecisionLower95": 0.9075
|
|
},
|
|
"threshold": 0.505
|
|
},
|
|
"task": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8331,
|
|
"falseNegative": 93,
|
|
"falsePositive": 14,
|
|
"precision": 0.9502,
|
|
"predictedPositive": 281,
|
|
"recall": 0.7417,
|
|
"total": 1485,
|
|
"trueNegative": 1111,
|
|
"truePositive": 267,
|
|
"wilsonPrecisionLower95": 0.9181
|
|
},
|
|
"threshold": 0.95
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.7445,
|
|
"falseNegative": 140,
|
|
"falsePositive": 11,
|
|
"precision": 0.9524,
|
|
"predictedPositive": 231,
|
|
"recall": 0.6111,
|
|
"total": 1485,
|
|
"trueNegative": 1114,
|
|
"truePositive": 220,
|
|
"wilsonPrecisionLower95": 0.9168
|
|
},
|
|
"threshold": 0.86
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.7672,
|
|
"falseNegative": 257,
|
|
"falsePositive": 24,
|
|
"precision": 0.9507,
|
|
"predictedPositive": 487,
|
|
"recall": 0.6431,
|
|
"total": 2970,
|
|
"trueNegative": 2226,
|
|
"truePositive": 463,
|
|
"wilsonPrecisionLower95": 0.9277
|
|
},
|
|
"threshold": 0.92
|
|
}
|
|
},
|
|
"trainingRecords": 17762,
|
|
"validation": {
|
|
"intentsMeeting95Precision": 8,
|
|
"macroF1": 0.698,
|
|
"macroPrecision": 0.9692,
|
|
"macroRecall": 0.6056,
|
|
"minimumPrecision": 0.9444,
|
|
"perIntent": {
|
|
"blessing": {
|
|
"f1": 0.2524,
|
|
"falseNegative": 77,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 13,
|
|
"recall": 0.1444,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 13,
|
|
"wilsonPrecisionLower95": 0.7719
|
|
},
|
|
"complaint": {
|
|
"f1": 0.965,
|
|
"falseNegative": 15,
|
|
"falsePositive": 10,
|
|
"precision": 0.9718,
|
|
"predictedPositive": 355,
|
|
"recall": 0.9583,
|
|
"total": 2970,
|
|
"trueNegative": 2600,
|
|
"truePositive": 345,
|
|
"wilsonPrecisionLower95": 0.9489
|
|
},
|
|
"confirmationDecision": {
|
|
"f1": 0.7948,
|
|
"falseNegative": 58,
|
|
"falsePositive": 5,
|
|
"precision": 0.9606,
|
|
"predictedPositive": 127,
|
|
"recall": 0.6778,
|
|
"total": 2970,
|
|
"trueNegative": 2785,
|
|
"truePositive": 122,
|
|
"wilsonPrecisionLower95": 0.9111
|
|
},
|
|
"followUpReminder": {
|
|
"f1": 0.5397,
|
|
"falseNegative": 224,
|
|
"falsePositive": 8,
|
|
"precision": 0.9444,
|
|
"predictedPositive": 144,
|
|
"recall": 0.3778,
|
|
"total": 2970,
|
|
"trueNegative": 2602,
|
|
"truePositive": 136,
|
|
"wilsonPrecisionLower95": 0.8942
|
|
},
|
|
"invitation": {
|
|
"f1": 0.3178,
|
|
"falseNegative": 146,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 34,
|
|
"recall": 0.1889,
|
|
"total": 2970,
|
|
"trueNegative": 2790,
|
|
"truePositive": 34,
|
|
"wilsonPrecisionLower95": 0.8985
|
|
},
|
|
"question": {
|
|
"f1": 0.771,
|
|
"falseNegative": 226,
|
|
"falsePositive": 14,
|
|
"precision": 0.9665,
|
|
"predictedPositive": 418,
|
|
"recall": 0.6413,
|
|
"total": 2970,
|
|
"trueNegative": 2326,
|
|
"truePositive": 404,
|
|
"wilsonPrecisionLower95": 0.9446
|
|
},
|
|
"replyableMessage": {
|
|
"f1": 0.8758,
|
|
"falseNegative": 356,
|
|
"falsePositive": 79,
|
|
"precision": 0.951,
|
|
"predictedPositive": 1613,
|
|
"recall": 0.8116,
|
|
"total": 2970,
|
|
"trueNegative": 1001,
|
|
"truePositive": 1534,
|
|
"wilsonPrecisionLower95": 0.9394
|
|
},
|
|
"scheduleNegotiation": {
|
|
"f1": 0.9778,
|
|
"falseNegative": 2,
|
|
"falsePositive": 2,
|
|
"precision": 0.9778,
|
|
"predictedPositive": 90,
|
|
"recall": 0.9778,
|
|
"total": 2970,
|
|
"trueNegative": 2878,
|
|
"truePositive": 88,
|
|
"wilsonPrecisionLower95": 0.9226
|
|
},
|
|
"task": {
|
|
"f1": 0.7876,
|
|
"falseNegative": 236,
|
|
"falsePositive": 25,
|
|
"precision": 0.9509,
|
|
"predictedPositive": 509,
|
|
"recall": 0.6722,
|
|
"total": 2970,
|
|
"trueNegative": 2225,
|
|
"truePositive": 484,
|
|
"wilsonPrecisionLower95": 0.9285
|
|
}
|
|
},
|
|
"recordCount": 2970
|
|
}
|
|
},
|
|
{
|
|
"configuration": {
|
|
"alpha": 4e-06,
|
|
"augmentation": "punctuation",
|
|
"char_max": 7,
|
|
"char_min": 3,
|
|
"external_weight": 0.73,
|
|
"hard_example_weight": 1.7,
|
|
"l1_ratio": 0.15,
|
|
"max_char_features": 70000,
|
|
"max_word_features": 35000,
|
|
"min_df": 2,
|
|
"pseudo_weight": 0.28500000000000003,
|
|
"round": 9,
|
|
"word_max": 3
|
|
},
|
|
"featureCount": 92641,
|
|
"pseudoLabelCounts": {
|
|
"blessing": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"complaint": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"confirmationDecision": {
|
|
"negative": 248,
|
|
"positive": 6
|
|
},
|
|
"followUpReminder": {
|
|
"negative": 250,
|
|
"positive": 1
|
|
},
|
|
"invitation": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"question": {
|
|
"negative": 146,
|
|
"positive": 8
|
|
},
|
|
"replyableMessage": {
|
|
"negative": 8,
|
|
"positive": 250
|
|
},
|
|
"scheduleNegotiation": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"task": {
|
|
"negative": 208,
|
|
"positive": 0
|
|
}
|
|
},
|
|
"round": 9,
|
|
"selectionObjective": 0.7717,
|
|
"syntheticRegression": {
|
|
"intentsMeeting95Precision": 4,
|
|
"macroF1": 0.6445,
|
|
"macroPrecision": 0.9357,
|
|
"macroRecall": 0.5386,
|
|
"minimumPrecision": 0.8583,
|
|
"perIntent": {
|
|
"blessing": {
|
|
"f1": 0.125,
|
|
"falseNegative": 84,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 6,
|
|
"recall": 0.0667,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 6,
|
|
"wilsonPrecisionLower95": 0.6097
|
|
},
|
|
"complaint": {
|
|
"f1": 0.949,
|
|
"falseNegative": 25,
|
|
"falsePositive": 11,
|
|
"precision": 0.9682,
|
|
"predictedPositive": 346,
|
|
"recall": 0.9306,
|
|
"total": 2970,
|
|
"trueNegative": 2599,
|
|
"truePositive": 335,
|
|
"wilsonPrecisionLower95": 0.944
|
|
},
|
|
"confirmationDecision": {
|
|
"f1": 0.7101,
|
|
"falseNegative": 71,
|
|
"falsePositive": 18,
|
|
"precision": 0.8583,
|
|
"predictedPositive": 127,
|
|
"recall": 0.6056,
|
|
"total": 2970,
|
|
"trueNegative": 2772,
|
|
"truePositive": 109,
|
|
"wilsonPrecisionLower95": 0.7871
|
|
},
|
|
"followUpReminder": {
|
|
"f1": 0.5244,
|
|
"falseNegative": 231,
|
|
"falsePositive": 3,
|
|
"precision": 0.9773,
|
|
"predictedPositive": 132,
|
|
"recall": 0.3583,
|
|
"total": 2970,
|
|
"trueNegative": 2607,
|
|
"truePositive": 129,
|
|
"wilsonPrecisionLower95": 0.9353
|
|
},
|
|
"invitation": {
|
|
"f1": 0.6286,
|
|
"falseNegative": 92,
|
|
"falsePositive": 12,
|
|
"precision": 0.88,
|
|
"predictedPositive": 100,
|
|
"recall": 0.4889,
|
|
"total": 2970,
|
|
"trueNegative": 2778,
|
|
"truePositive": 88,
|
|
"wilsonPrecisionLower95": 0.8019
|
|
},
|
|
"question": {
|
|
"f1": 0.6667,
|
|
"falseNegative": 310,
|
|
"falsePositive": 10,
|
|
"precision": 0.9697,
|
|
"predictedPositive": 330,
|
|
"recall": 0.5079,
|
|
"total": 2970,
|
|
"trueNegative": 2330,
|
|
"truePositive": 320,
|
|
"wilsonPrecisionLower95": 0.9451
|
|
},
|
|
"replyableMessage": {
|
|
"f1": 0.7919,
|
|
"falseNegative": 594,
|
|
"falsePositive": 87,
|
|
"precision": 0.9371,
|
|
"predictedPositive": 1383,
|
|
"recall": 0.6857,
|
|
"total": 2970,
|
|
"trueNegative": 993,
|
|
"truePositive": 1296,
|
|
"wilsonPrecisionLower95": 0.923
|
|
},
|
|
"scheduleNegotiation": {
|
|
"f1": 0.8941,
|
|
"falseNegative": 14,
|
|
"falsePositive": 4,
|
|
"precision": 0.95,
|
|
"predictedPositive": 80,
|
|
"recall": 0.8444,
|
|
"total": 2970,
|
|
"trueNegative": 2876,
|
|
"truePositive": 76,
|
|
"wilsonPrecisionLower95": 0.8784
|
|
},
|
|
"task": {
|
|
"f1": 0.5108,
|
|
"falseNegative": 461,
|
|
"falsePositive": 35,
|
|
"precision": 0.881,
|
|
"predictedPositive": 294,
|
|
"recall": 0.3597,
|
|
"total": 2970,
|
|
"trueNegative": 2215,
|
|
"truePositive": 259,
|
|
"wilsonPrecisionLower95": 0.8389
|
|
}
|
|
},
|
|
"recordCount": 2970
|
|
},
|
|
"thresholds": {
|
|
"blessing": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"complaint": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9862,
|
|
"falseNegative": 1,
|
|
"falsePositive": 4,
|
|
"precision": 0.9781,
|
|
"predictedPositive": 183,
|
|
"recall": 0.9944,
|
|
"total": 1485,
|
|
"trueNegative": 1301,
|
|
"truePositive": 179,
|
|
"wilsonPrecisionLower95": 0.9452
|
|
},
|
|
"threshold": 0.74
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9222,
|
|
"falseNegative": 20,
|
|
"falsePositive": 7,
|
|
"precision": 0.9581,
|
|
"predictedPositive": 167,
|
|
"recall": 0.8889,
|
|
"total": 1485,
|
|
"trueNegative": 1298,
|
|
"truePositive": 160,
|
|
"wilsonPrecisionLower95": 0.916
|
|
},
|
|
"threshold": 0.62
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.9528,
|
|
"falseNegative": 17,
|
|
"falsePositive": 17,
|
|
"precision": 0.9528,
|
|
"predictedPositive": 360,
|
|
"recall": 0.9528,
|
|
"total": 2970,
|
|
"trueNegative": 2593,
|
|
"truePositive": 343,
|
|
"wilsonPrecisionLower95": 0.9257
|
|
},
|
|
"threshold": 0.5
|
|
},
|
|
"confirmationDecision": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9492,
|
|
"falseNegative": 6,
|
|
"falsePositive": 3,
|
|
"precision": 0.9655,
|
|
"predictedPositive": 87,
|
|
"recall": 0.9333,
|
|
"total": 1485,
|
|
"trueNegative": 1392,
|
|
"truePositive": 84,
|
|
"wilsonPrecisionLower95": 0.9035
|
|
},
|
|
"threshold": 0.965
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.7042,
|
|
"falseNegative": 40,
|
|
"falsePositive": 2,
|
|
"precision": 0.9615,
|
|
"predictedPositive": 52,
|
|
"recall": 0.5556,
|
|
"total": 1485,
|
|
"trueNegative": 1393,
|
|
"truePositive": 50,
|
|
"wilsonPrecisionLower95": 0.8702
|
|
},
|
|
"threshold": 0.5
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.76,
|
|
"falseNegative": 66,
|
|
"falsePositive": 6,
|
|
"precision": 0.95,
|
|
"predictedPositive": 120,
|
|
"recall": 0.6333,
|
|
"total": 2970,
|
|
"trueNegative": 2784,
|
|
"truePositive": 114,
|
|
"wilsonPrecisionLower95": 0.8952
|
|
},
|
|
"threshold": 0.945
|
|
},
|
|
"followUpReminder": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8243,
|
|
"falseNegative": 51,
|
|
"falsePositive": 4,
|
|
"precision": 0.9699,
|
|
"predictedPositive": 133,
|
|
"recall": 0.7167,
|
|
"total": 1485,
|
|
"trueNegative": 1301,
|
|
"truePositive": 129,
|
|
"wilsonPrecisionLower95": 0.9252
|
|
},
|
|
"threshold": 0.99
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.6316,
|
|
"falseNegative": 192,
|
|
"falsePositive": 4,
|
|
"precision": 0.9767,
|
|
"predictedPositive": 172,
|
|
"recall": 0.4667,
|
|
"total": 2970,
|
|
"trueNegative": 2606,
|
|
"truePositive": 168,
|
|
"wilsonPrecisionLower95": 0.9417
|
|
},
|
|
"threshold": 0.995
|
|
},
|
|
"invitation": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.544,
|
|
"falseNegative": 56,
|
|
"falsePositive": 1,
|
|
"precision": 0.9714,
|
|
"predictedPositive": 35,
|
|
"recall": 0.3778,
|
|
"total": 1485,
|
|
"trueNegative": 1394,
|
|
"truePositive": 34,
|
|
"wilsonPrecisionLower95": 0.8547
|
|
},
|
|
"threshold": 0.82
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"question": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.6694,
|
|
"falseNegative": 153,
|
|
"falsePositive": 7,
|
|
"precision": 0.9586,
|
|
"predictedPositive": 169,
|
|
"recall": 0.5143,
|
|
"total": 1485,
|
|
"trueNegative": 1163,
|
|
"truePositive": 162,
|
|
"wilsonPrecisionLower95": 0.917
|
|
},
|
|
"threshold": 0.985
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.6762,
|
|
"falseNegative": 150,
|
|
"falsePositive": 8,
|
|
"precision": 0.9538,
|
|
"predictedPositive": 173,
|
|
"recall": 0.5238,
|
|
"total": 1485,
|
|
"trueNegative": 1162,
|
|
"truePositive": 165,
|
|
"wilsonPrecisionLower95": 0.9114
|
|
},
|
|
"threshold": 0.64
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.5392,
|
|
"falseNegative": 393,
|
|
"falsePositive": 12,
|
|
"precision": 0.9518,
|
|
"predictedPositive": 249,
|
|
"recall": 0.3762,
|
|
"total": 2970,
|
|
"trueNegative": 2328,
|
|
"truePositive": 237,
|
|
"wilsonPrecisionLower95": 0.9177
|
|
},
|
|
"threshold": 0.965
|
|
},
|
|
"replyableMessage": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9407,
|
|
"falseNegative": 65,
|
|
"falsePositive": 46,
|
|
"precision": 0.9503,
|
|
"predictedPositive": 926,
|
|
"recall": 0.9312,
|
|
"total": 1485,
|
|
"trueNegative": 494,
|
|
"truePositive": 880,
|
|
"wilsonPrecisionLower95": 0.9344
|
|
},
|
|
"threshold": 0.745
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.6118,
|
|
"falseNegative": 521,
|
|
"falsePositive": 17,
|
|
"precision": 0.9615,
|
|
"predictedPositive": 441,
|
|
"recall": 0.4487,
|
|
"total": 1485,
|
|
"trueNegative": 523,
|
|
"truePositive": 424,
|
|
"wilsonPrecisionLower95": 0.9391
|
|
},
|
|
"threshold": 0.965
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.8037,
|
|
"falseNegative": 576,
|
|
"falsePositive": 66,
|
|
"precision": 0.9522,
|
|
"predictedPositive": 1380,
|
|
"recall": 0.6952,
|
|
"total": 2970,
|
|
"trueNegative": 1014,
|
|
"truePositive": 1314,
|
|
"wilsonPrecisionLower95": 0.9396
|
|
},
|
|
"threshold": 0.95
|
|
},
|
|
"scheduleNegotiation": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9286,
|
|
"falseNegative": 6,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 39,
|
|
"recall": 0.8667,
|
|
"total": 1485,
|
|
"trueNegative": 1440,
|
|
"truePositive": 39,
|
|
"wilsonPrecisionLower95": 0.9103
|
|
},
|
|
"threshold": 0.5
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9302,
|
|
"falseNegative": 5,
|
|
"falsePositive": 1,
|
|
"precision": 0.9756,
|
|
"predictedPositive": 41,
|
|
"recall": 0.8889,
|
|
"total": 1485,
|
|
"trueNegative": 1439,
|
|
"truePositive": 40,
|
|
"wilsonPrecisionLower95": 0.874
|
|
},
|
|
"threshold": 0.61
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.9294,
|
|
"falseNegative": 11,
|
|
"falsePositive": 1,
|
|
"precision": 0.9875,
|
|
"predictedPositive": 80,
|
|
"recall": 0.8778,
|
|
"total": 2970,
|
|
"trueNegative": 2879,
|
|
"truePositive": 79,
|
|
"wilsonPrecisionLower95": 0.9325
|
|
},
|
|
"threshold": 0.61
|
|
},
|
|
"task": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8403,
|
|
"falseNegative": 89,
|
|
"falsePositive": 14,
|
|
"precision": 0.9509,
|
|
"predictedPositive": 285,
|
|
"recall": 0.7528,
|
|
"total": 1485,
|
|
"trueNegative": 1111,
|
|
"truePositive": 271,
|
|
"wilsonPrecisionLower95": 0.9192
|
|
},
|
|
"threshold": 0.93
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.2384,
|
|
"falseNegative": 311,
|
|
"falsePositive": 2,
|
|
"precision": 0.9608,
|
|
"predictedPositive": 51,
|
|
"recall": 0.1361,
|
|
"total": 1485,
|
|
"trueNegative": 1123,
|
|
"truePositive": 49,
|
|
"wilsonPrecisionLower95": 0.8678
|
|
},
|
|
"threshold": 0.995
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.6415,
|
|
"falseNegative": 372,
|
|
"falsePositive": 17,
|
|
"precision": 0.9534,
|
|
"predictedPositive": 365,
|
|
"recall": 0.4833,
|
|
"total": 2970,
|
|
"trueNegative": 2233,
|
|
"truePositive": 348,
|
|
"wilsonPrecisionLower95": 0.9267
|
|
},
|
|
"threshold": 0.975
|
|
}
|
|
},
|
|
"trainingRecords": 17823,
|
|
"validation": {
|
|
"intentsMeeting95Precision": 8,
|
|
"macroF1": 0.6858,
|
|
"macroPrecision": 0.9638,
|
|
"macroRecall": 0.5714,
|
|
"minimumPrecision": 0.9211,
|
|
"perIntent": {
|
|
"blessing": {
|
|
"f1": 0.2857,
|
|
"falseNegative": 75,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 15,
|
|
"recall": 0.1667,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 15,
|
|
"wilsonPrecisionLower95": 0.7961
|
|
},
|
|
"complaint": {
|
|
"f1": 0.9549,
|
|
"falseNegative": 21,
|
|
"falsePositive": 11,
|
|
"precision": 0.9686,
|
|
"predictedPositive": 350,
|
|
"recall": 0.9417,
|
|
"total": 2970,
|
|
"trueNegative": 2599,
|
|
"truePositive": 339,
|
|
"wilsonPrecisionLower95": 0.9446
|
|
},
|
|
"confirmationDecision": {
|
|
"f1": 0.8401,
|
|
"falseNegative": 46,
|
|
"falsePositive": 5,
|
|
"precision": 0.964,
|
|
"predictedPositive": 139,
|
|
"recall": 0.7444,
|
|
"total": 2970,
|
|
"trueNegative": 2785,
|
|
"truePositive": 134,
|
|
"wilsonPrecisionLower95": 0.9186
|
|
},
|
|
"followUpReminder": {
|
|
"f1": 0.5352,
|
|
"falseNegative": 227,
|
|
"falsePositive": 4,
|
|
"precision": 0.9708,
|
|
"predictedPositive": 137,
|
|
"recall": 0.3694,
|
|
"total": 2970,
|
|
"trueNegative": 2606,
|
|
"truePositive": 133,
|
|
"wilsonPrecisionLower95": 0.9273
|
|
},
|
|
"invitation": {
|
|
"f1": 0.5469,
|
|
"falseNegative": 110,
|
|
"falsePositive": 6,
|
|
"precision": 0.9211,
|
|
"predictedPositive": 76,
|
|
"recall": 0.3889,
|
|
"total": 2970,
|
|
"trueNegative": 2784,
|
|
"truePositive": 70,
|
|
"wilsonPrecisionLower95": 0.8383
|
|
},
|
|
"question": {
|
|
"f1": 0.6728,
|
|
"falseNegative": 303,
|
|
"falsePositive": 15,
|
|
"precision": 0.9561,
|
|
"predictedPositive": 342,
|
|
"recall": 0.519,
|
|
"total": 2970,
|
|
"trueNegative": 2325,
|
|
"truePositive": 327,
|
|
"wilsonPrecisionLower95": 0.9289
|
|
},
|
|
"replyableMessage": {
|
|
"f1": 0.8007,
|
|
"falseNegative": 586,
|
|
"falsePositive": 63,
|
|
"precision": 0.9539,
|
|
"predictedPositive": 1367,
|
|
"recall": 0.6899,
|
|
"total": 2970,
|
|
"trueNegative": 1017,
|
|
"truePositive": 1304,
|
|
"wilsonPrecisionLower95": 0.9415
|
|
},
|
|
"scheduleNegotiation": {
|
|
"f1": 0.9294,
|
|
"falseNegative": 11,
|
|
"falsePositive": 1,
|
|
"precision": 0.9875,
|
|
"predictedPositive": 80,
|
|
"recall": 0.8778,
|
|
"total": 2970,
|
|
"trueNegative": 2879,
|
|
"truePositive": 79,
|
|
"wilsonPrecisionLower95": 0.9325
|
|
},
|
|
"task": {
|
|
"f1": 0.6061,
|
|
"falseNegative": 400,
|
|
"falsePositive": 16,
|
|
"precision": 0.9524,
|
|
"predictedPositive": 336,
|
|
"recall": 0.4444,
|
|
"total": 2970,
|
|
"trueNegative": 2234,
|
|
"truePositive": 320,
|
|
"wilsonPrecisionLower95": 0.9241
|
|
}
|
|
},
|
|
"recordCount": 2970
|
|
}
|
|
},
|
|
{
|
|
"configuration": {
|
|
"alpha": 6e-06,
|
|
"augmentation": "punctuation",
|
|
"char_max": 6,
|
|
"char_min": 2,
|
|
"external_weight": 0.75,
|
|
"hard_example_weight": 1.8,
|
|
"l1_ratio": 0.2,
|
|
"max_char_features": 75000,
|
|
"max_word_features": 38000,
|
|
"min_df": 1,
|
|
"pseudo_weight": 0.3,
|
|
"round": 10,
|
|
"word_max": 3
|
|
},
|
|
"featureCount": 113000,
|
|
"pseudoLabelCounts": {
|
|
"blessing": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"complaint": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"confirmationDecision": {
|
|
"negative": 250,
|
|
"positive": 11
|
|
},
|
|
"followUpReminder": {
|
|
"negative": 250,
|
|
"positive": 4
|
|
},
|
|
"invitation": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"question": {
|
|
"negative": 182,
|
|
"positive": 10
|
|
},
|
|
"replyableMessage": {
|
|
"negative": 11,
|
|
"positive": 250
|
|
},
|
|
"scheduleNegotiation": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"task": {
|
|
"negative": 250,
|
|
"positive": 1
|
|
}
|
|
},
|
|
"round": 10,
|
|
"selectionObjective": 0.7542,
|
|
"syntheticRegression": {
|
|
"intentsMeeting95Precision": 4,
|
|
"macroF1": 0.6448,
|
|
"macroPrecision": 0.9609,
|
|
"macroRecall": 0.5363,
|
|
"minimumPrecision": 0.8734,
|
|
"perIntent": {
|
|
"blessing": {
|
|
"f1": 0.125,
|
|
"falseNegative": 84,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 6,
|
|
"recall": 0.0667,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 6,
|
|
"wilsonPrecisionLower95": 0.6097
|
|
},
|
|
"complaint": {
|
|
"f1": 0.9366,
|
|
"falseNegative": 20,
|
|
"falsePositive": 26,
|
|
"precision": 0.929,
|
|
"predictedPositive": 366,
|
|
"recall": 0.9444,
|
|
"total": 2970,
|
|
"trueNegative": 2584,
|
|
"truePositive": 340,
|
|
"wilsonPrecisionLower95": 0.8979
|
|
},
|
|
"confirmationDecision": {
|
|
"f1": 0.8166,
|
|
"falseNegative": 42,
|
|
"falsePositive": 20,
|
|
"precision": 0.8734,
|
|
"predictedPositive": 158,
|
|
"recall": 0.7667,
|
|
"total": 2970,
|
|
"trueNegative": 2770,
|
|
"truePositive": 138,
|
|
"wilsonPrecisionLower95": 0.8126
|
|
},
|
|
"followUpReminder": {
|
|
"f1": 0.5397,
|
|
"falseNegative": 224,
|
|
"falsePositive": 8,
|
|
"precision": 0.9444,
|
|
"predictedPositive": 144,
|
|
"recall": 0.3778,
|
|
"total": 2970,
|
|
"trueNegative": 2602,
|
|
"truePositive": 136,
|
|
"wilsonPrecisionLower95": 0.8942
|
|
},
|
|
"invitation": {
|
|
"f1": 0.5124,
|
|
"falseNegative": 118,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 62,
|
|
"recall": 0.3444,
|
|
"total": 2970,
|
|
"trueNegative": 2790,
|
|
"truePositive": 62,
|
|
"wilsonPrecisionLower95": 0.9417
|
|
},
|
|
"question": {
|
|
"f1": 0.5488,
|
|
"falseNegative": 391,
|
|
"falsePositive": 2,
|
|
"precision": 0.9917,
|
|
"predictedPositive": 241,
|
|
"recall": 0.3794,
|
|
"total": 2970,
|
|
"trueNegative": 2338,
|
|
"truePositive": 239,
|
|
"wilsonPrecisionLower95": 0.9703
|
|
},
|
|
"replyableMessage": {
|
|
"f1": 0.7216,
|
|
"falseNegative": 808,
|
|
"falsePositive": 27,
|
|
"precision": 0.9757,
|
|
"predictedPositive": 1109,
|
|
"recall": 0.5725,
|
|
"total": 2970,
|
|
"trueNegative": 1053,
|
|
"truePositive": 1082,
|
|
"wilsonPrecisionLower95": 0.9648
|
|
},
|
|
"scheduleNegotiation": {
|
|
"f1": 0.882,
|
|
"falseNegative": 19,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 71,
|
|
"recall": 0.7889,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 71,
|
|
"wilsonPrecisionLower95": 0.9487
|
|
},
|
|
"task": {
|
|
"f1": 0.7201,
|
|
"falseNegative": 298,
|
|
"falsePositive": 30,
|
|
"precision": 0.9336,
|
|
"predictedPositive": 452,
|
|
"recall": 0.5861,
|
|
"total": 2970,
|
|
"trueNegative": 2220,
|
|
"truePositive": 422,
|
|
"wilsonPrecisionLower95": 0.9068
|
|
}
|
|
},
|
|
"recordCount": 2970
|
|
},
|
|
"thresholds": {
|
|
"blessing": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"complaint": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9807,
|
|
"falseNegative": 2,
|
|
"falsePositive": 5,
|
|
"precision": 0.9727,
|
|
"predictedPositive": 183,
|
|
"recall": 0.9889,
|
|
"total": 1485,
|
|
"trueNegative": 1300,
|
|
"truePositive": 178,
|
|
"wilsonPrecisionLower95": 0.9376
|
|
},
|
|
"threshold": 0.765
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9288,
|
|
"falseNegative": 17,
|
|
"falsePositive": 8,
|
|
"precision": 0.9532,
|
|
"predictedPositive": 171,
|
|
"recall": 0.9056,
|
|
"total": 1485,
|
|
"trueNegative": 1297,
|
|
"truePositive": 163,
|
|
"wilsonPrecisionLower95": 0.9104
|
|
},
|
|
"threshold": 0.5
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.9455,
|
|
"falseNegative": 22,
|
|
"falsePositive": 17,
|
|
"precision": 0.9521,
|
|
"predictedPositive": 355,
|
|
"recall": 0.9389,
|
|
"total": 2970,
|
|
"trueNegative": 2593,
|
|
"truePositive": 338,
|
|
"wilsonPrecisionLower95": 0.9247
|
|
},
|
|
"threshold": 0.595
|
|
},
|
|
"confirmationDecision": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.7568,
|
|
"falseNegative": 34,
|
|
"falsePositive": 2,
|
|
"precision": 0.9655,
|
|
"predictedPositive": 58,
|
|
"recall": 0.6222,
|
|
"total": 1485,
|
|
"trueNegative": 1393,
|
|
"truePositive": 56,
|
|
"wilsonPrecisionLower95": 0.8827
|
|
},
|
|
"threshold": 0.955
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.7733,
|
|
"falseNegative": 32,
|
|
"falsePositive": 2,
|
|
"precision": 0.9667,
|
|
"predictedPositive": 60,
|
|
"recall": 0.6444,
|
|
"total": 1485,
|
|
"trueNegative": 1393,
|
|
"truePositive": 58,
|
|
"wilsonPrecisionLower95": 0.8864
|
|
},
|
|
"threshold": 0.5
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.7723,
|
|
"falseNegative": 63,
|
|
"falsePositive": 6,
|
|
"precision": 0.9512,
|
|
"predictedPositive": 123,
|
|
"recall": 0.65,
|
|
"total": 2970,
|
|
"trueNegative": 2784,
|
|
"truePositive": 117,
|
|
"wilsonPrecisionLower95": 0.8977
|
|
},
|
|
"threshold": 0.765
|
|
},
|
|
"followUpReminder": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8483,
|
|
"falseNegative": 43,
|
|
"falsePositive": 6,
|
|
"precision": 0.958,
|
|
"predictedPositive": 143,
|
|
"recall": 0.7611,
|
|
"total": 1485,
|
|
"trueNegative": 1299,
|
|
"truePositive": 137,
|
|
"wilsonPrecisionLower95": 0.9115
|
|
},
|
|
"threshold": 0.995
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.3529,
|
|
"falseNegative": 141,
|
|
"falsePositive": 2,
|
|
"precision": 0.9512,
|
|
"predictedPositive": 41,
|
|
"recall": 0.2167,
|
|
"total": 1485,
|
|
"trueNegative": 1303,
|
|
"truePositive": 39,
|
|
"wilsonPrecisionLower95": 0.8386
|
|
},
|
|
"threshold": 0.99
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.6292,
|
|
"falseNegative": 192,
|
|
"falsePositive": 6,
|
|
"precision": 0.9655,
|
|
"predictedPositive": 174,
|
|
"recall": 0.4667,
|
|
"total": 2970,
|
|
"trueNegative": 2604,
|
|
"truePositive": 168,
|
|
"wilsonPrecisionLower95": 0.9268
|
|
},
|
|
"threshold": 0.995
|
|
},
|
|
"invitation": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.3894,
|
|
"falseNegative": 68,
|
|
"falsePositive": 1,
|
|
"precision": 0.9565,
|
|
"predictedPositive": 23,
|
|
"recall": 0.2444,
|
|
"total": 1485,
|
|
"trueNegative": 1394,
|
|
"truePositive": 22,
|
|
"wilsonPrecisionLower95": 0.7901
|
|
},
|
|
"threshold": 0.98
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"question": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8175,
|
|
"falseNegative": 91,
|
|
"falsePositive": 9,
|
|
"precision": 0.9614,
|
|
"predictedPositive": 233,
|
|
"recall": 0.7111,
|
|
"total": 1485,
|
|
"trueNegative": 1161,
|
|
"truePositive": 224,
|
|
"wilsonPrecisionLower95": 0.9282
|
|
},
|
|
"threshold": 0.6
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"replyableMessage": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.7182,
|
|
"falseNegative": 401,
|
|
"falsePositive": 26,
|
|
"precision": 0.9544,
|
|
"predictedPositive": 570,
|
|
"recall": 0.5757,
|
|
"total": 1485,
|
|
"trueNegative": 514,
|
|
"truePositive": 544,
|
|
"wilsonPrecisionLower95": 0.934
|
|
},
|
|
"threshold": 0.985
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.6667,
|
|
"falseNegative": 460,
|
|
"falsePositive": 25,
|
|
"precision": 0.951,
|
|
"predictedPositive": 510,
|
|
"recall": 0.5132,
|
|
"total": 1485,
|
|
"trueNegative": 515,
|
|
"truePositive": 485,
|
|
"wilsonPrecisionLower95": 0.9286
|
|
},
|
|
"threshold": 0.945
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.7151,
|
|
"falseNegative": 807,
|
|
"falsePositive": 56,
|
|
"precision": 0.9508,
|
|
"predictedPositive": 1139,
|
|
"recall": 0.573,
|
|
"total": 2970,
|
|
"trueNegative": 1024,
|
|
"truePositive": 1083,
|
|
"wilsonPrecisionLower95": 0.9367
|
|
},
|
|
"threshold": 0.96
|
|
},
|
|
"scheduleNegotiation": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9412,
|
|
"falseNegative": 5,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 40,
|
|
"recall": 0.8889,
|
|
"total": 1485,
|
|
"trueNegative": 1440,
|
|
"truePositive": 40,
|
|
"wilsonPrecisionLower95": 0.9124
|
|
},
|
|
"threshold": 0.5
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8052,
|
|
"falseNegative": 14,
|
|
"falsePositive": 1,
|
|
"precision": 0.9688,
|
|
"predictedPositive": 32,
|
|
"recall": 0.6889,
|
|
"total": 1485,
|
|
"trueNegative": 1439,
|
|
"truePositive": 31,
|
|
"wilsonPrecisionLower95": 0.8426
|
|
},
|
|
"threshold": 0.98
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.8375,
|
|
"falseNegative": 23,
|
|
"falsePositive": 3,
|
|
"precision": 0.9571,
|
|
"predictedPositive": 70,
|
|
"recall": 0.7444,
|
|
"total": 2970,
|
|
"trueNegative": 2877,
|
|
"truePositive": 67,
|
|
"wilsonPrecisionLower95": 0.8814
|
|
},
|
|
"threshold": 0.92
|
|
},
|
|
"task": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.7783,
|
|
"falseNegative": 123,
|
|
"falsePositive": 12,
|
|
"precision": 0.9518,
|
|
"predictedPositive": 249,
|
|
"recall": 0.6583,
|
|
"total": 1485,
|
|
"trueNegative": 1113,
|
|
"truePositive": 237,
|
|
"wilsonPrecisionLower95": 0.9177
|
|
},
|
|
"threshold": 0.84
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.7994,
|
|
"falseNegative": 113,
|
|
"falsePositive": 11,
|
|
"precision": 0.9574,
|
|
"predictedPositive": 258,
|
|
"recall": 0.6861,
|
|
"total": 1485,
|
|
"trueNegative": 1114,
|
|
"truePositive": 247,
|
|
"wilsonPrecisionLower95": 0.9253
|
|
},
|
|
"threshold": 0.755
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.7977,
|
|
"falseNegative": 225,
|
|
"falsePositive": 26,
|
|
"precision": 0.9501,
|
|
"predictedPositive": 521,
|
|
"recall": 0.6875,
|
|
"total": 2970,
|
|
"trueNegative": 2224,
|
|
"truePositive": 495,
|
|
"wilsonPrecisionLower95": 0.9279
|
|
},
|
|
"threshold": 0.77
|
|
}
|
|
},
|
|
"trainingRecords": 17763,
|
|
"validation": {
|
|
"intentsMeeting95Precision": 8,
|
|
"macroF1": 0.6563,
|
|
"macroPrecision": 0.9695,
|
|
"macroRecall": 0.5386,
|
|
"minimumPrecision": 0.9528,
|
|
"perIntent": {
|
|
"blessing": {
|
|
"f1": 0.1818,
|
|
"falseNegative": 81,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 9,
|
|
"recall": 0.1,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 9,
|
|
"wilsonPrecisionLower95": 0.7009
|
|
},
|
|
"complaint": {
|
|
"f1": 0.9552,
|
|
"falseNegative": 19,
|
|
"falsePositive": 13,
|
|
"precision": 0.9633,
|
|
"predictedPositive": 354,
|
|
"recall": 0.9472,
|
|
"total": 2970,
|
|
"trueNegative": 2597,
|
|
"truePositive": 341,
|
|
"wilsonPrecisionLower95": 0.9382
|
|
},
|
|
"confirmationDecision": {
|
|
"f1": 0.7651,
|
|
"falseNegative": 66,
|
|
"falsePositive": 4,
|
|
"precision": 0.9661,
|
|
"predictedPositive": 118,
|
|
"recall": 0.6333,
|
|
"total": 2970,
|
|
"trueNegative": 2786,
|
|
"truePositive": 114,
|
|
"wilsonPrecisionLower95": 0.9161
|
|
},
|
|
"followUpReminder": {
|
|
"f1": 0.6471,
|
|
"falseNegative": 184,
|
|
"falsePositive": 8,
|
|
"precision": 0.9565,
|
|
"predictedPositive": 184,
|
|
"recall": 0.4889,
|
|
"total": 2970,
|
|
"trueNegative": 2602,
|
|
"truePositive": 176,
|
|
"wilsonPrecisionLower95": 0.9166
|
|
},
|
|
"invitation": {
|
|
"f1": 0.479,
|
|
"falseNegative": 123,
|
|
"falsePositive": 1,
|
|
"precision": 0.9828,
|
|
"predictedPositive": 58,
|
|
"recall": 0.3167,
|
|
"total": 2970,
|
|
"trueNegative": 2789,
|
|
"truePositive": 57,
|
|
"wilsonPrecisionLower95": 0.9086
|
|
},
|
|
"question": {
|
|
"f1": 0.5191,
|
|
"falseNegative": 406,
|
|
"falsePositive": 9,
|
|
"precision": 0.9614,
|
|
"predictedPositive": 233,
|
|
"recall": 0.3556,
|
|
"total": 2970,
|
|
"trueNegative": 2331,
|
|
"truePositive": 224,
|
|
"wilsonPrecisionLower95": 0.9282
|
|
},
|
|
"replyableMessage": {
|
|
"f1": 0.6929,
|
|
"falseNegative": 861,
|
|
"falsePositive": 51,
|
|
"precision": 0.9528,
|
|
"predictedPositive": 1080,
|
|
"recall": 0.5444,
|
|
"total": 2970,
|
|
"trueNegative": 1029,
|
|
"truePositive": 1029,
|
|
"wilsonPrecisionLower95": 0.9384
|
|
},
|
|
"scheduleNegotiation": {
|
|
"f1": 0.8765,
|
|
"falseNegative": 19,
|
|
"falsePositive": 1,
|
|
"precision": 0.9861,
|
|
"predictedPositive": 72,
|
|
"recall": 0.7889,
|
|
"total": 2970,
|
|
"trueNegative": 2879,
|
|
"truePositive": 71,
|
|
"wilsonPrecisionLower95": 0.9254
|
|
},
|
|
"task": {
|
|
"f1": 0.7896,
|
|
"falseNegative": 236,
|
|
"falsePositive": 22,
|
|
"precision": 0.9565,
|
|
"predictedPositive": 506,
|
|
"recall": 0.6722,
|
|
"total": 2970,
|
|
"trueNegative": 2228,
|
|
"truePositive": 484,
|
|
"wilsonPrecisionLower95": 0.935
|
|
}
|
|
},
|
|
"recordCount": 2970
|
|
}
|
|
},
|
|
{
|
|
"configuration": {
|
|
"alpha": 5e-06,
|
|
"augmentation": "prefix",
|
|
"char_max": 6,
|
|
"char_min": 2,
|
|
"external_weight": 0.77,
|
|
"hard_example_weight": 1.8,
|
|
"l1_ratio": 0.1,
|
|
"max_char_features": 70000,
|
|
"max_word_features": 30000,
|
|
"min_df": 2,
|
|
"pseudo_weight": 0.31499999999999995,
|
|
"round": 11,
|
|
"word_max": 2
|
|
},
|
|
"featureCount": 81451,
|
|
"pseudoLabelCounts": {
|
|
"blessing": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"complaint": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"confirmationDecision": {
|
|
"negative": 250,
|
|
"positive": 14
|
|
},
|
|
"followUpReminder": {
|
|
"negative": 250,
|
|
"positive": 2
|
|
},
|
|
"invitation": {
|
|
"negative": 250,
|
|
"positive": 1
|
|
},
|
|
"question": {
|
|
"negative": 233,
|
|
"positive": 0
|
|
},
|
|
"replyableMessage": {
|
|
"negative": 8,
|
|
"positive": 250
|
|
},
|
|
"scheduleNegotiation": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"task": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
}
|
|
},
|
|
"round": 11,
|
|
"selectionObjective": 0.8015,
|
|
"syntheticRegression": {
|
|
"intentsMeeting95Precision": 4,
|
|
"macroF1": 0.6807,
|
|
"macroPrecision": 0.963,
|
|
"macroRecall": 0.6053,
|
|
"minimumPrecision": 0.8782,
|
|
"perIntent": {
|
|
"blessing": {
|
|
"f1": 0.125,
|
|
"falseNegative": 84,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 6,
|
|
"recall": 0.0667,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 6,
|
|
"wilsonPrecisionLower95": 0.6097
|
|
},
|
|
"complaint": {
|
|
"f1": 0.9655,
|
|
"falseNegative": 24,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 336,
|
|
"recall": 0.9333,
|
|
"total": 2970,
|
|
"trueNegative": 2610,
|
|
"truePositive": 336,
|
|
"wilsonPrecisionLower95": 0.9887
|
|
},
|
|
"confirmationDecision": {
|
|
"f1": 0.8155,
|
|
"falseNegative": 43,
|
|
"falsePositive": 19,
|
|
"precision": 0.8782,
|
|
"predictedPositive": 156,
|
|
"recall": 0.7611,
|
|
"total": 2970,
|
|
"trueNegative": 2771,
|
|
"truePositive": 137,
|
|
"wilsonPrecisionLower95": 0.8176
|
|
},
|
|
"followUpReminder": {
|
|
"f1": 0.2439,
|
|
"falseNegative": 310,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 50,
|
|
"recall": 0.1389,
|
|
"total": 2970,
|
|
"trueNegative": 2610,
|
|
"truePositive": 50,
|
|
"wilsonPrecisionLower95": 0.9287
|
|
},
|
|
"invitation": {
|
|
"f1": 0.5425,
|
|
"falseNegative": 113,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 67,
|
|
"recall": 0.3722,
|
|
"total": 2970,
|
|
"trueNegative": 2790,
|
|
"truePositive": 67,
|
|
"wilsonPrecisionLower95": 0.9458
|
|
},
|
|
"question": {
|
|
"f1": 0.8494,
|
|
"falseNegative": 162,
|
|
"falsePositive": 4,
|
|
"precision": 0.9915,
|
|
"predictedPositive": 472,
|
|
"recall": 0.7429,
|
|
"total": 2970,
|
|
"trueNegative": 2336,
|
|
"truePositive": 468,
|
|
"wilsonPrecisionLower95": 0.9784
|
|
},
|
|
"replyableMessage": {
|
|
"f1": 0.8975,
|
|
"falseNegative": 253,
|
|
"falsePositive": 121,
|
|
"precision": 0.9312,
|
|
"predictedPositive": 1758,
|
|
"recall": 0.8661,
|
|
"total": 2970,
|
|
"trueNegative": 959,
|
|
"truePositive": 1637,
|
|
"wilsonPrecisionLower95": 0.9184
|
|
},
|
|
"scheduleNegotiation": {
|
|
"f1": 0.9444,
|
|
"falseNegative": 5,
|
|
"falsePositive": 5,
|
|
"precision": 0.9444,
|
|
"predictedPositive": 90,
|
|
"recall": 0.9444,
|
|
"total": 2970,
|
|
"trueNegative": 2875,
|
|
"truePositive": 85,
|
|
"wilsonPrecisionLower95": 0.8765
|
|
},
|
|
"task": {
|
|
"f1": 0.743,
|
|
"falseNegative": 272,
|
|
"falsePositive": 38,
|
|
"precision": 0.9218,
|
|
"predictedPositive": 486,
|
|
"recall": 0.6222,
|
|
"total": 2970,
|
|
"trueNegative": 2212,
|
|
"truePositive": 448,
|
|
"wilsonPrecisionLower95": 0.8945
|
|
}
|
|
},
|
|
"recordCount": 2970
|
|
},
|
|
"thresholds": {
|
|
"blessing": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"complaint": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9917,
|
|
"falseNegative": 0,
|
|
"falsePositive": 3,
|
|
"precision": 0.9836,
|
|
"predictedPositive": 183,
|
|
"recall": 1.0,
|
|
"total": 1485,
|
|
"trueNegative": 1302,
|
|
"truePositive": 180,
|
|
"wilsonPrecisionLower95": 0.9529
|
|
},
|
|
"threshold": 0.99
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9318,
|
|
"falseNegative": 16,
|
|
"falsePositive": 8,
|
|
"precision": 0.9535,
|
|
"predictedPositive": 172,
|
|
"recall": 0.9111,
|
|
"total": 1485,
|
|
"trueNegative": 1297,
|
|
"truePositive": 164,
|
|
"wilsonPrecisionLower95": 0.9109
|
|
},
|
|
"threshold": 0.84
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.9529,
|
|
"falseNegative": 16,
|
|
"falsePositive": 18,
|
|
"precision": 0.9503,
|
|
"predictedPositive": 362,
|
|
"recall": 0.9556,
|
|
"total": 2970,
|
|
"trueNegative": 2592,
|
|
"truePositive": 344,
|
|
"wilsonPrecisionLower95": 0.9228
|
|
},
|
|
"threshold": 0.84
|
|
},
|
|
"confirmationDecision": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.931,
|
|
"falseNegative": 9,
|
|
"falsePositive": 3,
|
|
"precision": 0.9643,
|
|
"predictedPositive": 84,
|
|
"recall": 0.9,
|
|
"total": 1485,
|
|
"trueNegative": 1392,
|
|
"truePositive": 81,
|
|
"wilsonPrecisionLower95": 0.9002
|
|
},
|
|
"threshold": 0.97
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.7222,
|
|
"falseNegative": 38,
|
|
"falsePositive": 2,
|
|
"precision": 0.963,
|
|
"predictedPositive": 54,
|
|
"recall": 0.5778,
|
|
"total": 1485,
|
|
"trueNegative": 1393,
|
|
"truePositive": 52,
|
|
"wilsonPrecisionLower95": 0.8746
|
|
},
|
|
"threshold": 0.86
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.8077,
|
|
"falseNegative": 54,
|
|
"falsePositive": 6,
|
|
"precision": 0.9545,
|
|
"predictedPositive": 132,
|
|
"recall": 0.7,
|
|
"total": 2970,
|
|
"trueNegative": 2784,
|
|
"truePositive": 126,
|
|
"wilsonPrecisionLower95": 0.9044
|
|
},
|
|
"threshold": 0.95
|
|
},
|
|
"followUpReminder": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"invitation": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.4833,
|
|
"falseNegative": 61,
|
|
"falsePositive": 1,
|
|
"precision": 0.9667,
|
|
"predictedPositive": 30,
|
|
"recall": 0.3222,
|
|
"total": 1485,
|
|
"trueNegative": 1394,
|
|
"truePositive": 29,
|
|
"wilsonPrecisionLower95": 0.8333
|
|
},
|
|
"threshold": 0.965
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"question": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.7993,
|
|
"falseNegative": 98,
|
|
"falsePositive": 11,
|
|
"precision": 0.9518,
|
|
"predictedPositive": 228,
|
|
"recall": 0.6889,
|
|
"total": 1485,
|
|
"trueNegative": 1159,
|
|
"truePositive": 217,
|
|
"wilsonPrecisionLower95": 0.9157
|
|
},
|
|
"threshold": 0.92
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.7572,
|
|
"falseNegative": 117,
|
|
"falsePositive": 10,
|
|
"precision": 0.9519,
|
|
"predictedPositive": 208,
|
|
"recall": 0.6286,
|
|
"total": 1485,
|
|
"trueNegative": 1160,
|
|
"truePositive": 198,
|
|
"wilsonPrecisionLower95": 0.9138
|
|
},
|
|
"threshold": 0.735
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.797,
|
|
"falseNegative": 198,
|
|
"falsePositive": 22,
|
|
"precision": 0.9515,
|
|
"predictedPositive": 454,
|
|
"recall": 0.6857,
|
|
"total": 2970,
|
|
"trueNegative": 2318,
|
|
"truePositive": 432,
|
|
"wilsonPrecisionLower95": 0.9277
|
|
},
|
|
"threshold": 0.8
|
|
},
|
|
"replyableMessage": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9373,
|
|
"falseNegative": 71,
|
|
"falsePositive": 46,
|
|
"precision": 0.95,
|
|
"predictedPositive": 920,
|
|
"recall": 0.9249,
|
|
"total": 1485,
|
|
"trueNegative": 494,
|
|
"truePositive": 874,
|
|
"wilsonPrecisionLower95": 0.934
|
|
},
|
|
"threshold": 0.79
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8754,
|
|
"falseNegative": 179,
|
|
"falsePositive": 39,
|
|
"precision": 0.9516,
|
|
"predictedPositive": 805,
|
|
"recall": 0.8106,
|
|
"total": 1485,
|
|
"trueNegative": 501,
|
|
"truePositive": 766,
|
|
"wilsonPrecisionLower95": 0.9345
|
|
},
|
|
"threshold": 0.91
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.9105,
|
|
"falseNegative": 242,
|
|
"falsePositive": 82,
|
|
"precision": 0.9526,
|
|
"predictedPositive": 1730,
|
|
"recall": 0.872,
|
|
"total": 2970,
|
|
"trueNegative": 998,
|
|
"truePositive": 1648,
|
|
"wilsonPrecisionLower95": 0.9415
|
|
},
|
|
"threshold": 0.895
|
|
},
|
|
"scheduleNegotiation": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9773,
|
|
"falseNegative": 2,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 43,
|
|
"recall": 0.9556,
|
|
"total": 1485,
|
|
"trueNegative": 1440,
|
|
"truePositive": 43,
|
|
"wilsonPrecisionLower95": 0.918
|
|
},
|
|
"threshold": 0.5
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9318,
|
|
"falseNegative": 4,
|
|
"falsePositive": 2,
|
|
"precision": 0.9535,
|
|
"predictedPositive": 43,
|
|
"recall": 0.9111,
|
|
"total": 1485,
|
|
"trueNegative": 1438,
|
|
"truePositive": 41,
|
|
"wilsonPrecisionLower95": 0.8454
|
|
},
|
|
"threshold": 0.9
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.9551,
|
|
"falseNegative": 5,
|
|
"falsePositive": 3,
|
|
"precision": 0.9659,
|
|
"predictedPositive": 88,
|
|
"recall": 0.9444,
|
|
"total": 2970,
|
|
"trueNegative": 2877,
|
|
"truePositive": 85,
|
|
"wilsonPrecisionLower95": 0.9045
|
|
},
|
|
"threshold": 0.885
|
|
},
|
|
"task": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8315,
|
|
"falseNegative": 96,
|
|
"falsePositive": 11,
|
|
"precision": 0.96,
|
|
"predictedPositive": 275,
|
|
"recall": 0.7333,
|
|
"total": 1485,
|
|
"trueNegative": 1114,
|
|
"truePositive": 264,
|
|
"wilsonPrecisionLower95": 0.9298
|
|
},
|
|
"threshold": 0.985
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8349,
|
|
"falseNegative": 92,
|
|
"falsePositive": 14,
|
|
"precision": 0.9504,
|
|
"predictedPositive": 282,
|
|
"recall": 0.7444,
|
|
"total": 1485,
|
|
"trueNegative": 1111,
|
|
"truePositive": 268,
|
|
"wilsonPrecisionLower95": 0.9184
|
|
},
|
|
"threshold": 0.845
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.7688,
|
|
"falseNegative": 256,
|
|
"falsePositive": 23,
|
|
"precision": 0.9528,
|
|
"predictedPositive": 487,
|
|
"recall": 0.6444,
|
|
"total": 2970,
|
|
"trueNegative": 2227,
|
|
"truePositive": 464,
|
|
"wilsonPrecisionLower95": 0.9301
|
|
},
|
|
"threshold": 0.975
|
|
}
|
|
},
|
|
"trainingRecords": 15345,
|
|
"validation": {
|
|
"intentsMeeting95Precision": 9,
|
|
"macroF1": 0.7269,
|
|
"macroPrecision": 0.9659,
|
|
"macroRecall": 0.6366,
|
|
"minimumPrecision": 0.9507,
|
|
"perIntent": {
|
|
"blessing": {
|
|
"f1": 0.2692,
|
|
"falseNegative": 76,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 14,
|
|
"recall": 0.1556,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 14,
|
|
"wilsonPrecisionLower95": 0.7847
|
|
},
|
|
"complaint": {
|
|
"f1": 0.9622,
|
|
"falseNegative": 16,
|
|
"falsePositive": 11,
|
|
"precision": 0.969,
|
|
"predictedPositive": 355,
|
|
"recall": 0.9556,
|
|
"total": 2970,
|
|
"trueNegative": 2599,
|
|
"truePositive": 344,
|
|
"wilsonPrecisionLower95": 0.9454
|
|
},
|
|
"confirmationDecision": {
|
|
"f1": 0.8365,
|
|
"falseNegative": 47,
|
|
"falsePositive": 5,
|
|
"precision": 0.9638,
|
|
"predictedPositive": 138,
|
|
"recall": 0.7389,
|
|
"total": 2970,
|
|
"trueNegative": 2785,
|
|
"truePositive": 133,
|
|
"wilsonPrecisionLower95": 0.918
|
|
},
|
|
"followUpReminder": {
|
|
"f1": 0.4335,
|
|
"falseNegative": 259,
|
|
"falsePositive": 5,
|
|
"precision": 0.9528,
|
|
"predictedPositive": 106,
|
|
"recall": 0.2806,
|
|
"total": 2970,
|
|
"trueNegative": 2605,
|
|
"truePositive": 101,
|
|
"wilsonPrecisionLower95": 0.8943
|
|
},
|
|
"invitation": {
|
|
"f1": 0.5669,
|
|
"falseNegative": 108,
|
|
"falsePositive": 2,
|
|
"precision": 0.973,
|
|
"predictedPositive": 74,
|
|
"recall": 0.4,
|
|
"total": 2970,
|
|
"trueNegative": 2788,
|
|
"truePositive": 72,
|
|
"wilsonPrecisionLower95": 0.9067
|
|
},
|
|
"question": {
|
|
"f1": 0.7786,
|
|
"falseNegative": 215,
|
|
"falsePositive": 21,
|
|
"precision": 0.9518,
|
|
"predictedPositive": 436,
|
|
"recall": 0.6587,
|
|
"total": 2970,
|
|
"trueNegative": 2319,
|
|
"truePositive": 415,
|
|
"wilsonPrecisionLower95": 0.9275
|
|
},
|
|
"replyableMessage": {
|
|
"f1": 0.9073,
|
|
"falseNegative": 250,
|
|
"falsePositive": 85,
|
|
"precision": 0.9507,
|
|
"predictedPositive": 1725,
|
|
"recall": 0.8677,
|
|
"total": 2970,
|
|
"trueNegative": 995,
|
|
"truePositive": 1640,
|
|
"wilsonPrecisionLower95": 0.9395
|
|
},
|
|
"scheduleNegotiation": {
|
|
"f1": 0.9545,
|
|
"falseNegative": 6,
|
|
"falsePositive": 2,
|
|
"precision": 0.9767,
|
|
"predictedPositive": 86,
|
|
"recall": 0.9333,
|
|
"total": 2970,
|
|
"trueNegative": 2878,
|
|
"truePositive": 84,
|
|
"wilsonPrecisionLower95": 0.9191
|
|
},
|
|
"task": {
|
|
"f1": 0.8332,
|
|
"falseNegative": 188,
|
|
"falsePositive": 25,
|
|
"precision": 0.9551,
|
|
"predictedPositive": 557,
|
|
"recall": 0.7389,
|
|
"total": 2970,
|
|
"trueNegative": 2225,
|
|
"truePositive": 532,
|
|
"wilsonPrecisionLower95": 0.9346
|
|
}
|
|
},
|
|
"recordCount": 2970
|
|
}
|
|
},
|
|
{
|
|
"configuration": {
|
|
"alpha": 4e-06,
|
|
"augmentation": "prefix",
|
|
"char_max": 6,
|
|
"char_min": 3,
|
|
"external_weight": 0.79,
|
|
"hard_example_weight": 2.0,
|
|
"l1_ratio": 0.15,
|
|
"max_char_features": 75000,
|
|
"max_word_features": 35000,
|
|
"min_df": 2,
|
|
"pseudo_weight": 0.32999999999999996,
|
|
"round": 12,
|
|
"word_max": 3
|
|
},
|
|
"featureCount": 94458,
|
|
"pseudoLabelCounts": {
|
|
"blessing": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"complaint": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"confirmationDecision": {
|
|
"negative": 250,
|
|
"positive": 15
|
|
},
|
|
"followUpReminder": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"invitation": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"question": {
|
|
"negative": 232,
|
|
"positive": 0
|
|
},
|
|
"replyableMessage": {
|
|
"negative": 10,
|
|
"positive": 250
|
|
},
|
|
"scheduleNegotiation": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"task": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
}
|
|
},
|
|
"round": 12,
|
|
"selectionObjective": 0.7563,
|
|
"syntheticRegression": {
|
|
"intentsMeeting95Precision": 5,
|
|
"macroF1": 0.6379,
|
|
"macroPrecision": 0.9476,
|
|
"macroRecall": 0.5322,
|
|
"minimumPrecision": 0.8452,
|
|
"perIntent": {
|
|
"blessing": {
|
|
"f1": 0.1053,
|
|
"falseNegative": 85,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 5,
|
|
"recall": 0.0556,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 5,
|
|
"wilsonPrecisionLower95": 0.5655
|
|
},
|
|
"complaint": {
|
|
"f1": 0.9625,
|
|
"falseNegative": 26,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 334,
|
|
"recall": 0.9278,
|
|
"total": 2970,
|
|
"trueNegative": 2610,
|
|
"truePositive": 334,
|
|
"wilsonPrecisionLower95": 0.9886
|
|
},
|
|
"confirmationDecision": {
|
|
"f1": 0.7821,
|
|
"falseNegative": 49,
|
|
"falsePositive": 24,
|
|
"precision": 0.8452,
|
|
"predictedPositive": 155,
|
|
"recall": 0.7278,
|
|
"total": 2970,
|
|
"trueNegative": 2766,
|
|
"truePositive": 131,
|
|
"wilsonPrecisionLower95": 0.7799
|
|
},
|
|
"followUpReminder": {
|
|
"f1": 0.6142,
|
|
"falseNegative": 200,
|
|
"falsePositive": 1,
|
|
"precision": 0.9938,
|
|
"predictedPositive": 161,
|
|
"recall": 0.4444,
|
|
"total": 2970,
|
|
"trueNegative": 2609,
|
|
"truePositive": 160,
|
|
"wilsonPrecisionLower95": 0.9657
|
|
},
|
|
"invitation": {
|
|
"f1": 0.5992,
|
|
"falseNegative": 103,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 77,
|
|
"recall": 0.4278,
|
|
"total": 2970,
|
|
"trueNegative": 2790,
|
|
"truePositive": 77,
|
|
"wilsonPrecisionLower95": 0.9525
|
|
},
|
|
"question": {
|
|
"f1": 0.4319,
|
|
"falseNegative": 454,
|
|
"falsePositive": 9,
|
|
"precision": 0.9514,
|
|
"predictedPositive": 185,
|
|
"recall": 0.2794,
|
|
"total": 2970,
|
|
"trueNegative": 2331,
|
|
"truePositive": 176,
|
|
"wilsonPrecisionLower95": 0.9101
|
|
},
|
|
"replyableMessage": {
|
|
"f1": 0.7888,
|
|
"falseNegative": 624,
|
|
"falsePositive": 54,
|
|
"precision": 0.9591,
|
|
"predictedPositive": 1320,
|
|
"recall": 0.6698,
|
|
"total": 2970,
|
|
"trueNegative": 1026,
|
|
"truePositive": 1266,
|
|
"wilsonPrecisionLower95": 0.947
|
|
},
|
|
"scheduleNegotiation": {
|
|
"f1": 0.8144,
|
|
"falseNegative": 22,
|
|
"falsePositive": 9,
|
|
"precision": 0.8831,
|
|
"predictedPositive": 77,
|
|
"recall": 0.7556,
|
|
"total": 2970,
|
|
"trueNegative": 2871,
|
|
"truePositive": 68,
|
|
"wilsonPrecisionLower95": 0.7925
|
|
},
|
|
"task": {
|
|
"f1": 0.6429,
|
|
"falseNegative": 359,
|
|
"falsePositive": 42,
|
|
"precision": 0.8958,
|
|
"predictedPositive": 403,
|
|
"recall": 0.5014,
|
|
"total": 2970,
|
|
"trueNegative": 2208,
|
|
"truePositive": 361,
|
|
"wilsonPrecisionLower95": 0.8621
|
|
}
|
|
},
|
|
"recordCount": 2970
|
|
},
|
|
"thresholds": {
|
|
"blessing": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"complaint": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9917,
|
|
"falseNegative": 0,
|
|
"falsePositive": 3,
|
|
"precision": 0.9836,
|
|
"predictedPositive": 183,
|
|
"recall": 1.0,
|
|
"total": 1485,
|
|
"trueNegative": 1302,
|
|
"truePositive": 180,
|
|
"wilsonPrecisionLower95": 0.9529
|
|
},
|
|
"threshold": 0.885
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8974,
|
|
"falseNegative": 27,
|
|
"falsePositive": 8,
|
|
"precision": 0.9503,
|
|
"predictedPositive": 161,
|
|
"recall": 0.85,
|
|
"total": 1485,
|
|
"trueNegative": 1297,
|
|
"truePositive": 153,
|
|
"wilsonPrecisionLower95": 0.905
|
|
},
|
|
"threshold": 0.9
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.9515,
|
|
"falseNegative": 17,
|
|
"falsePositive": 18,
|
|
"precision": 0.9501,
|
|
"predictedPositive": 361,
|
|
"recall": 0.9528,
|
|
"total": 2970,
|
|
"trueNegative": 2592,
|
|
"truePositive": 343,
|
|
"wilsonPrecisionLower95": 0.9226
|
|
},
|
|
"threshold": 0.77
|
|
},
|
|
"confirmationDecision": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9944,
|
|
"falseNegative": 1,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 89,
|
|
"recall": 0.9889,
|
|
"total": 1485,
|
|
"trueNegative": 1395,
|
|
"truePositive": 89,
|
|
"wilsonPrecisionLower95": 0.9586
|
|
},
|
|
"threshold": 0.915
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.4959,
|
|
"falseNegative": 60,
|
|
"falsePositive": 1,
|
|
"precision": 0.9677,
|
|
"predictedPositive": 31,
|
|
"recall": 0.3333,
|
|
"total": 1485,
|
|
"trueNegative": 1394,
|
|
"truePositive": 30,
|
|
"wilsonPrecisionLower95": 0.8381
|
|
},
|
|
"threshold": 0.985
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.8217,
|
|
"falseNegative": 51,
|
|
"falsePositive": 5,
|
|
"precision": 0.9627,
|
|
"predictedPositive": 134,
|
|
"recall": 0.7167,
|
|
"total": 2970,
|
|
"trueNegative": 2785,
|
|
"truePositive": 129,
|
|
"wilsonPrecisionLower95": 0.9156
|
|
},
|
|
"threshold": 0.915
|
|
},
|
|
"followUpReminder": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.5647,
|
|
"falseNegative": 108,
|
|
"falsePositive": 3,
|
|
"precision": 0.96,
|
|
"predictedPositive": 75,
|
|
"recall": 0.4,
|
|
"total": 1485,
|
|
"trueNegative": 1302,
|
|
"truePositive": 72,
|
|
"wilsonPrecisionLower95": 0.8889
|
|
},
|
|
"threshold": 0.995
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"invitation": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"question": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.6542,
|
|
"falseNegative": 158,
|
|
"falsePositive": 8,
|
|
"precision": 0.9515,
|
|
"predictedPositive": 165,
|
|
"recall": 0.4984,
|
|
"total": 1485,
|
|
"trueNegative": 1162,
|
|
"truePositive": 157,
|
|
"wilsonPrecisionLower95": 0.9073
|
|
},
|
|
"threshold": 0.805
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.5711,
|
|
"falseNegative": 373,
|
|
"falsePositive": 13,
|
|
"precision": 0.9519,
|
|
"predictedPositive": 270,
|
|
"recall": 0.4079,
|
|
"total": 2970,
|
|
"trueNegative": 2327,
|
|
"truePositive": 257,
|
|
"wilsonPrecisionLower95": 0.9194
|
|
},
|
|
"threshold": 0.985
|
|
},
|
|
"replyableMessage": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8956,
|
|
"falseNegative": 147,
|
|
"falsePositive": 39,
|
|
"precision": 0.9534,
|
|
"predictedPositive": 837,
|
|
"recall": 0.8444,
|
|
"total": 1485,
|
|
"trueNegative": 501,
|
|
"truePositive": 798,
|
|
"wilsonPrecisionLower95": 0.9369
|
|
},
|
|
"threshold": 0.975
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.6735,
|
|
"falseNegative": 453,
|
|
"falsePositive": 24,
|
|
"precision": 0.9535,
|
|
"predictedPositive": 516,
|
|
"recall": 0.5206,
|
|
"total": 1485,
|
|
"trueNegative": 516,
|
|
"truePositive": 492,
|
|
"wilsonPrecisionLower95": 0.9317
|
|
},
|
|
"threshold": 0.985
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.7866,
|
|
"falseNegative": 626,
|
|
"falsePositive": 60,
|
|
"precision": 0.9547,
|
|
"predictedPositive": 1324,
|
|
"recall": 0.6688,
|
|
"total": 2970,
|
|
"trueNegative": 1020,
|
|
"truePositive": 1264,
|
|
"wilsonPrecisionLower95": 0.9421
|
|
},
|
|
"threshold": 0.985
|
|
},
|
|
"scheduleNegotiation": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9888,
|
|
"falseNegative": 1,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 44,
|
|
"recall": 0.9778,
|
|
"total": 1485,
|
|
"trueNegative": 1440,
|
|
"truePositive": 44,
|
|
"wilsonPrecisionLower95": 0.9197
|
|
},
|
|
"threshold": 0.5
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8052,
|
|
"falseNegative": 14,
|
|
"falsePositive": 1,
|
|
"precision": 0.9688,
|
|
"predictedPositive": 32,
|
|
"recall": 0.6889,
|
|
"total": 1485,
|
|
"trueNegative": 1439,
|
|
"truePositive": 31,
|
|
"wilsonPrecisionLower95": 0.8426
|
|
},
|
|
"threshold": 0.94
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.9364,
|
|
"falseNegative": 9,
|
|
"falsePositive": 2,
|
|
"precision": 0.9759,
|
|
"predictedPositive": 83,
|
|
"recall": 0.9,
|
|
"total": 2970,
|
|
"trueNegative": 2878,
|
|
"truePositive": 81,
|
|
"wilsonPrecisionLower95": 0.9163
|
|
},
|
|
"threshold": 0.715
|
|
},
|
|
"task": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8475,
|
|
"falseNegative": 85,
|
|
"falsePositive": 14,
|
|
"precision": 0.9516,
|
|
"predictedPositive": 289,
|
|
"recall": 0.7639,
|
|
"total": 1485,
|
|
"trueNegative": 1111,
|
|
"truePositive": 275,
|
|
"wilsonPrecisionLower95": 0.9203
|
|
},
|
|
"threshold": 0.985
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.501,
|
|
"falseNegative": 238,
|
|
"falsePositive": 5,
|
|
"precision": 0.9606,
|
|
"predictedPositive": 127,
|
|
"recall": 0.3389,
|
|
"total": 1485,
|
|
"trueNegative": 1120,
|
|
"truePositive": 122,
|
|
"wilsonPrecisionLower95": 0.9111
|
|
},
|
|
"threshold": 0.985
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.6989,
|
|
"falseNegative": 323,
|
|
"falsePositive": 19,
|
|
"precision": 0.9543,
|
|
"predictedPositive": 416,
|
|
"recall": 0.5514,
|
|
"total": 2970,
|
|
"trueNegative": 2231,
|
|
"truePositive": 397,
|
|
"wilsonPrecisionLower95": 0.9298
|
|
},
|
|
"threshold": 0.985
|
|
}
|
|
},
|
|
"trainingRecords": 15345,
|
|
"validation": {
|
|
"intentsMeeting95Precision": 8,
|
|
"macroF1": 0.6603,
|
|
"macroPrecision": 0.9686,
|
|
"macroRecall": 0.5413,
|
|
"minimumPrecision": 0.9298,
|
|
"perIntent": {
|
|
"blessing": {
|
|
"f1": 0.2857,
|
|
"falseNegative": 75,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 15,
|
|
"recall": 0.1667,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 15,
|
|
"wilsonPrecisionLower95": 0.7961
|
|
},
|
|
"complaint": {
|
|
"f1": 0.946,
|
|
"falseNegative": 27,
|
|
"falsePositive": 11,
|
|
"precision": 0.968,
|
|
"predictedPositive": 344,
|
|
"recall": 0.925,
|
|
"total": 2970,
|
|
"trueNegative": 2599,
|
|
"truePositive": 333,
|
|
"wilsonPrecisionLower95": 0.9437
|
|
},
|
|
"confirmationDecision": {
|
|
"f1": 0.7933,
|
|
"falseNegative": 61,
|
|
"falsePositive": 1,
|
|
"precision": 0.9917,
|
|
"predictedPositive": 120,
|
|
"recall": 0.6611,
|
|
"total": 2970,
|
|
"trueNegative": 2789,
|
|
"truePositive": 119,
|
|
"wilsonPrecisionLower95": 0.9543
|
|
},
|
|
"followUpReminder": {
|
|
"f1": 0.6494,
|
|
"falseNegative": 185,
|
|
"falsePositive": 4,
|
|
"precision": 0.9777,
|
|
"predictedPositive": 179,
|
|
"recall": 0.4861,
|
|
"total": 2970,
|
|
"trueNegative": 2606,
|
|
"truePositive": 175,
|
|
"wilsonPrecisionLower95": 0.944
|
|
},
|
|
"invitation": {
|
|
"f1": 0.4473,
|
|
"falseNegative": 127,
|
|
"falsePositive": 4,
|
|
"precision": 0.9298,
|
|
"predictedPositive": 57,
|
|
"recall": 0.2944,
|
|
"total": 2970,
|
|
"trueNegative": 2786,
|
|
"truePositive": 53,
|
|
"wilsonPrecisionLower95": 0.833
|
|
},
|
|
"question": {
|
|
"f1": 0.4227,
|
|
"falseNegative": 459,
|
|
"falsePositive": 8,
|
|
"precision": 0.9553,
|
|
"predictedPositive": 179,
|
|
"recall": 0.2714,
|
|
"total": 2970,
|
|
"trueNegative": 2332,
|
|
"truePositive": 171,
|
|
"wilsonPrecisionLower95": 0.9143
|
|
},
|
|
"replyableMessage": {
|
|
"f1": 0.7956,
|
|
"falseNegative": 600,
|
|
"falsePositive": 63,
|
|
"precision": 0.9534,
|
|
"predictedPositive": 1353,
|
|
"recall": 0.6825,
|
|
"total": 2970,
|
|
"trueNegative": 1017,
|
|
"truePositive": 1290,
|
|
"wilsonPrecisionLower95": 0.9409
|
|
},
|
|
"scheduleNegotiation": {
|
|
"f1": 0.9036,
|
|
"falseNegative": 15,
|
|
"falsePositive": 1,
|
|
"precision": 0.9868,
|
|
"predictedPositive": 76,
|
|
"recall": 0.8333,
|
|
"total": 2970,
|
|
"trueNegative": 2879,
|
|
"truePositive": 75,
|
|
"wilsonPrecisionLower95": 0.9292
|
|
},
|
|
"task": {
|
|
"f1": 0.6989,
|
|
"falseNegative": 323,
|
|
"falsePositive": 19,
|
|
"precision": 0.9543,
|
|
"predictedPositive": 416,
|
|
"recall": 0.5514,
|
|
"total": 2970,
|
|
"trueNegative": 2231,
|
|
"truePositive": 397,
|
|
"wilsonPrecisionLower95": 0.9298
|
|
}
|
|
},
|
|
"recordCount": 2970
|
|
}
|
|
},
|
|
{
|
|
"configuration": {
|
|
"alpha": 3e-06,
|
|
"augmentation": "prefix",
|
|
"char_max": 7,
|
|
"char_min": 2,
|
|
"external_weight": 0.81,
|
|
"hard_example_weight": 2.0,
|
|
"l1_ratio": 0.15,
|
|
"max_char_features": 80000,
|
|
"max_word_features": 38000,
|
|
"min_df": 2,
|
|
"pseudo_weight": 0.345,
|
|
"round": 13,
|
|
"word_max": 3
|
|
},
|
|
"featureCount": 99458,
|
|
"pseudoLabelCounts": {
|
|
"blessing": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"complaint": {
|
|
"negative": 250,
|
|
"positive": 1
|
|
},
|
|
"confirmationDecision": {
|
|
"negative": 250,
|
|
"positive": 18
|
|
},
|
|
"followUpReminder": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"invitation": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"question": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"replyableMessage": {
|
|
"negative": 12,
|
|
"positive": 250
|
|
},
|
|
"scheduleNegotiation": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"task": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
}
|
|
},
|
|
"round": 13,
|
|
"selectionObjective": 0.768,
|
|
"syntheticRegression": {
|
|
"intentsMeeting95Precision": 4,
|
|
"macroF1": 0.6577,
|
|
"macroPrecision": 0.9568,
|
|
"macroRecall": 0.5722,
|
|
"minimumPrecision": 0.859,
|
|
"perIntent": {
|
|
"blessing": {
|
|
"f1": 0.1443,
|
|
"falseNegative": 83,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 7,
|
|
"recall": 0.0778,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 7,
|
|
"wilsonPrecisionLower95": 0.6457
|
|
},
|
|
"complaint": {
|
|
"f1": 0.9744,
|
|
"falseNegative": 17,
|
|
"falsePositive": 1,
|
|
"precision": 0.9971,
|
|
"predictedPositive": 344,
|
|
"recall": 0.9528,
|
|
"total": 2970,
|
|
"trueNegative": 2609,
|
|
"truePositive": 343,
|
|
"wilsonPrecisionLower95": 0.9837
|
|
},
|
|
"confirmationDecision": {
|
|
"f1": 0.7976,
|
|
"falseNegative": 46,
|
|
"falsePositive": 22,
|
|
"precision": 0.859,
|
|
"predictedPositive": 156,
|
|
"recall": 0.7444,
|
|
"total": 2970,
|
|
"trueNegative": 2768,
|
|
"truePositive": 134,
|
|
"wilsonPrecisionLower95": 0.7957
|
|
},
|
|
"followUpReminder": {
|
|
"f1": 0.2972,
|
|
"falseNegative": 297,
|
|
"falsePositive": 1,
|
|
"precision": 0.9844,
|
|
"predictedPositive": 64,
|
|
"recall": 0.175,
|
|
"total": 2970,
|
|
"trueNegative": 2609,
|
|
"truePositive": 63,
|
|
"wilsonPrecisionLower95": 0.9167
|
|
},
|
|
"invitation": {
|
|
"f1": 0.6077,
|
|
"falseNegative": 101,
|
|
"falsePositive": 1,
|
|
"precision": 0.9875,
|
|
"predictedPositive": 80,
|
|
"recall": 0.4389,
|
|
"total": 2970,
|
|
"trueNegative": 2789,
|
|
"truePositive": 79,
|
|
"wilsonPrecisionLower95": 0.9325
|
|
},
|
|
"question": {
|
|
"f1": 0.5229,
|
|
"falseNegative": 407,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 223,
|
|
"recall": 0.354,
|
|
"total": 2970,
|
|
"trueNegative": 2340,
|
|
"truePositive": 223,
|
|
"wilsonPrecisionLower95": 0.9831
|
|
},
|
|
"replyableMessage": {
|
|
"f1": 0.9179,
|
|
"falseNegative": 202,
|
|
"falsePositive": 100,
|
|
"precision": 0.9441,
|
|
"predictedPositive": 1788,
|
|
"recall": 0.8931,
|
|
"total": 2970,
|
|
"trueNegative": 980,
|
|
"truePositive": 1688,
|
|
"wilsonPrecisionLower95": 0.9324
|
|
},
|
|
"scheduleNegotiation": {
|
|
"f1": 0.8889,
|
|
"falseNegative": 14,
|
|
"falsePositive": 5,
|
|
"precision": 0.9383,
|
|
"predictedPositive": 81,
|
|
"recall": 0.8444,
|
|
"total": 2970,
|
|
"trueNegative": 2875,
|
|
"truePositive": 76,
|
|
"wilsonPrecisionLower95": 0.8635
|
|
},
|
|
"task": {
|
|
"f1": 0.7681,
|
|
"falseNegative": 238,
|
|
"falsePositive": 53,
|
|
"precision": 0.9009,
|
|
"predictedPositive": 535,
|
|
"recall": 0.6694,
|
|
"total": 2970,
|
|
"trueNegative": 2197,
|
|
"truePositive": 482,
|
|
"wilsonPrecisionLower95": 0.8727
|
|
}
|
|
},
|
|
"recordCount": 2970
|
|
},
|
|
"thresholds": {
|
|
"blessing": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"complaint": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9727,
|
|
"falseNegative": 2,
|
|
"falsePositive": 8,
|
|
"precision": 0.957,
|
|
"predictedPositive": 186,
|
|
"recall": 0.9889,
|
|
"total": 1485,
|
|
"trueNegative": 1297,
|
|
"truePositive": 178,
|
|
"wilsonPrecisionLower95": 0.9174
|
|
},
|
|
"threshold": 0.98
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.95,
|
|
"falseNegative": 9,
|
|
"falsePositive": 9,
|
|
"precision": 0.95,
|
|
"predictedPositive": 180,
|
|
"recall": 0.95,
|
|
"total": 1485,
|
|
"trueNegative": 1296,
|
|
"truePositive": 171,
|
|
"wilsonPrecisionLower95": 0.9077
|
|
},
|
|
"threshold": 0.905
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.9529,
|
|
"falseNegative": 16,
|
|
"falsePositive": 18,
|
|
"precision": 0.9503,
|
|
"predictedPositive": 362,
|
|
"recall": 0.9556,
|
|
"total": 2970,
|
|
"trueNegative": 2592,
|
|
"truePositive": 344,
|
|
"wilsonPrecisionLower95": 0.9228
|
|
},
|
|
"threshold": 0.965
|
|
},
|
|
"confirmationDecision": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.828,
|
|
"falseNegative": 25,
|
|
"falsePositive": 2,
|
|
"precision": 0.9701,
|
|
"predictedPositive": 67,
|
|
"recall": 0.7222,
|
|
"total": 1485,
|
|
"trueNegative": 1393,
|
|
"truePositive": 65,
|
|
"wilsonPrecisionLower95": 0.8975
|
|
},
|
|
"threshold": 0.5
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.6763,
|
|
"falseNegative": 43,
|
|
"falsePositive": 2,
|
|
"precision": 0.9592,
|
|
"predictedPositive": 49,
|
|
"recall": 0.5222,
|
|
"total": 1485,
|
|
"trueNegative": 1393,
|
|
"truePositive": 47,
|
|
"wilsonPrecisionLower95": 0.8629
|
|
},
|
|
"threshold": 0.93
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.7855,
|
|
"falseNegative": 61,
|
|
"falsePositive": 4,
|
|
"precision": 0.9675,
|
|
"predictedPositive": 123,
|
|
"recall": 0.6611,
|
|
"total": 2970,
|
|
"trueNegative": 2786,
|
|
"truePositive": 119,
|
|
"wilsonPrecisionLower95": 0.9194
|
|
},
|
|
"threshold": 0.82
|
|
},
|
|
"followUpReminder": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"invitation": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"question": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.7452,
|
|
"falseNegative": 122,
|
|
"falsePositive": 10,
|
|
"precision": 0.9507,
|
|
"predictedPositive": 203,
|
|
"recall": 0.6127,
|
|
"total": 1485,
|
|
"trueNegative": 1160,
|
|
"truePositive": 193,
|
|
"wilsonPrecisionLower95": 0.9117
|
|
},
|
|
"threshold": 0.915
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"replyableMessage": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9441,
|
|
"falseNegative": 59,
|
|
"falsePositive": 46,
|
|
"precision": 0.9506,
|
|
"predictedPositive": 932,
|
|
"recall": 0.9376,
|
|
"total": 1485,
|
|
"trueNegative": 494,
|
|
"truePositive": 886,
|
|
"wilsonPrecisionLower95": 0.9348
|
|
},
|
|
"threshold": 0.935
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9067,
|
|
"falseNegative": 129,
|
|
"falsePositive": 39,
|
|
"precision": 0.9544,
|
|
"predictedPositive": 855,
|
|
"recall": 0.8635,
|
|
"total": 1485,
|
|
"trueNegative": 501,
|
|
"truePositive": 816,
|
|
"wilsonPrecisionLower95": 0.9383
|
|
},
|
|
"threshold": 0.905
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.9267,
|
|
"falseNegative": 182,
|
|
"falsePositive": 88,
|
|
"precision": 0.951,
|
|
"predictedPositive": 1796,
|
|
"recall": 0.9037,
|
|
"total": 2970,
|
|
"trueNegative": 992,
|
|
"truePositive": 1708,
|
|
"wilsonPrecisionLower95": 0.94
|
|
},
|
|
"threshold": 0.905
|
|
},
|
|
"scheduleNegotiation": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9888,
|
|
"falseNegative": 1,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 44,
|
|
"recall": 0.9778,
|
|
"total": 1485,
|
|
"trueNegative": 1440,
|
|
"truePositive": 44,
|
|
"wilsonPrecisionLower95": 0.9197
|
|
},
|
|
"threshold": 0.5
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9655,
|
|
"falseNegative": 3,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 42,
|
|
"recall": 0.9333,
|
|
"total": 1485,
|
|
"trueNegative": 1440,
|
|
"truePositive": 42,
|
|
"wilsonPrecisionLower95": 0.9162
|
|
},
|
|
"threshold": 0.5
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.9773,
|
|
"falseNegative": 4,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 86,
|
|
"recall": 0.9556,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 86,
|
|
"wilsonPrecisionLower95": 0.9572
|
|
},
|
|
"threshold": 0.5
|
|
},
|
|
"task": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.7876,
|
|
"falseNegative": 119,
|
|
"falsePositive": 11,
|
|
"precision": 0.9563,
|
|
"predictedPositive": 252,
|
|
"recall": 0.6694,
|
|
"total": 1485,
|
|
"trueNegative": 1114,
|
|
"truePositive": 241,
|
|
"wilsonPrecisionLower95": 0.9235
|
|
},
|
|
"threshold": 0.995
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8416,
|
|
"falseNegative": 89,
|
|
"falsePositive": 13,
|
|
"precision": 0.9542,
|
|
"predictedPositive": 284,
|
|
"recall": 0.7528,
|
|
"total": 1485,
|
|
"trueNegative": 1112,
|
|
"truePositive": 271,
|
|
"wilsonPrecisionLower95": 0.9233
|
|
},
|
|
"threshold": 0.895
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.6962,
|
|
"falseNegative": 327,
|
|
"falsePositive": 16,
|
|
"precision": 0.9609,
|
|
"predictedPositive": 409,
|
|
"recall": 0.5458,
|
|
"total": 2970,
|
|
"trueNegative": 2234,
|
|
"truePositive": 393,
|
|
"wilsonPrecisionLower95": 0.9374
|
|
},
|
|
"threshold": 0.995
|
|
}
|
|
},
|
|
"trainingRecords": 15346,
|
|
"validation": {
|
|
"intentsMeeting95Precision": 9,
|
|
"macroF1": 0.6724,
|
|
"macroPrecision": 0.9724,
|
|
"macroRecall": 0.5779,
|
|
"minimumPrecision": 0.9524,
|
|
"perIntent": {
|
|
"blessing": {
|
|
"f1": 0.2178,
|
|
"falseNegative": 79,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 11,
|
|
"recall": 0.1222,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 11,
|
|
"wilsonPrecisionLower95": 0.7412
|
|
},
|
|
"complaint": {
|
|
"f1": 0.9614,
|
|
"falseNegative": 11,
|
|
"falsePositive": 17,
|
|
"precision": 0.9536,
|
|
"predictedPositive": 366,
|
|
"recall": 0.9694,
|
|
"total": 2970,
|
|
"trueNegative": 2593,
|
|
"truePositive": 349,
|
|
"wilsonPrecisionLower95": 0.9269
|
|
},
|
|
"confirmationDecision": {
|
|
"f1": 0.7568,
|
|
"falseNegative": 68,
|
|
"falsePositive": 4,
|
|
"precision": 0.9655,
|
|
"predictedPositive": 116,
|
|
"recall": 0.6222,
|
|
"total": 2970,
|
|
"trueNegative": 2786,
|
|
"truePositive": 112,
|
|
"wilsonPrecisionLower95": 0.9147
|
|
},
|
|
"followUpReminder": {
|
|
"f1": 0.4895,
|
|
"falseNegative": 243,
|
|
"falsePositive": 1,
|
|
"precision": 0.9915,
|
|
"predictedPositive": 118,
|
|
"recall": 0.325,
|
|
"total": 2970,
|
|
"trueNegative": 2609,
|
|
"truePositive": 117,
|
|
"wilsonPrecisionLower95": 0.9536
|
|
},
|
|
"invitation": {
|
|
"f1": 0.4192,
|
|
"falseNegative": 132,
|
|
"falsePositive": 1,
|
|
"precision": 0.9796,
|
|
"predictedPositive": 49,
|
|
"recall": 0.2667,
|
|
"total": 2970,
|
|
"trueNegative": 2789,
|
|
"truePositive": 48,
|
|
"wilsonPrecisionLower95": 0.8931
|
|
},
|
|
"question": {
|
|
"f1": 0.4888,
|
|
"falseNegative": 423,
|
|
"falsePositive": 10,
|
|
"precision": 0.9539,
|
|
"predictedPositive": 217,
|
|
"recall": 0.3286,
|
|
"total": 2970,
|
|
"trueNegative": 2330,
|
|
"truePositive": 207,
|
|
"wilsonPrecisionLower95": 0.9173
|
|
},
|
|
"replyableMessage": {
|
|
"f1": 0.9258,
|
|
"falseNegative": 188,
|
|
"falsePositive": 85,
|
|
"precision": 0.9524,
|
|
"predictedPositive": 1787,
|
|
"recall": 0.9005,
|
|
"total": 2970,
|
|
"trueNegative": 995,
|
|
"truePositive": 1702,
|
|
"wilsonPrecisionLower95": 0.9416
|
|
},
|
|
"scheduleNegotiation": {
|
|
"f1": 0.9773,
|
|
"falseNegative": 4,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 86,
|
|
"recall": 0.9556,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 86,
|
|
"wilsonPrecisionLower95": 0.9572
|
|
},
|
|
"task": {
|
|
"f1": 0.8153,
|
|
"falseNegative": 208,
|
|
"falsePositive": 24,
|
|
"precision": 0.9552,
|
|
"predictedPositive": 536,
|
|
"recall": 0.7111,
|
|
"total": 2970,
|
|
"trueNegative": 2226,
|
|
"truePositive": 512,
|
|
"wilsonPrecisionLower95": 0.9342
|
|
}
|
|
},
|
|
"recordCount": 2970
|
|
}
|
|
},
|
|
{
|
|
"configuration": {
|
|
"alpha": 2.5e-06,
|
|
"augmentation": "prefix",
|
|
"char_max": 7,
|
|
"char_min": 3,
|
|
"external_weight": 0.8300000000000001,
|
|
"hard_example_weight": 2.2,
|
|
"l1_ratio": 0.2,
|
|
"max_char_features": 80000,
|
|
"max_word_features": 40000,
|
|
"min_df": 1,
|
|
"pseudo_weight": 0.36,
|
|
"round": 14,
|
|
"word_max": 3
|
|
},
|
|
"featureCount": 120000,
|
|
"pseudoLabelCounts": {
|
|
"blessing": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"complaint": {
|
|
"negative": 250,
|
|
"positive": 8
|
|
},
|
|
"confirmationDecision": {
|
|
"negative": 250,
|
|
"positive": 26
|
|
},
|
|
"followUpReminder": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"invitation": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"question": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"replyableMessage": {
|
|
"negative": 34,
|
|
"positive": 250
|
|
},
|
|
"scheduleNegotiation": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"task": {
|
|
"negative": 250,
|
|
"positive": 3
|
|
}
|
|
},
|
|
"round": 14,
|
|
"selectionObjective": 0.7391,
|
|
"syntheticRegression": {
|
|
"intentsMeeting95Precision": 5,
|
|
"macroF1": 0.6159,
|
|
"macroPrecision": 0.9488,
|
|
"macroRecall": 0.5171,
|
|
"minimumPrecision": 0.8315,
|
|
"perIntent": {
|
|
"blessing": {
|
|
"f1": 0.1443,
|
|
"falseNegative": 83,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 7,
|
|
"recall": 0.0778,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 7,
|
|
"wilsonPrecisionLower95": 0.6457
|
|
},
|
|
"complaint": {
|
|
"f1": 0.7987,
|
|
"falseNegative": 114,
|
|
"falsePositive": 10,
|
|
"precision": 0.9609,
|
|
"predictedPositive": 256,
|
|
"recall": 0.6833,
|
|
"total": 2970,
|
|
"trueNegative": 2600,
|
|
"truePositive": 246,
|
|
"wilsonPrecisionLower95": 0.9296
|
|
},
|
|
"confirmationDecision": {
|
|
"f1": 0.8407,
|
|
"falseNegative": 27,
|
|
"falsePositive": 31,
|
|
"precision": 0.8315,
|
|
"predictedPositive": 184,
|
|
"recall": 0.85,
|
|
"total": 2970,
|
|
"trueNegative": 2759,
|
|
"truePositive": 153,
|
|
"wilsonPrecisionLower95": 0.7708
|
|
},
|
|
"followUpReminder": {
|
|
"f1": 0.7361,
|
|
"falseNegative": 148,
|
|
"falsePositive": 4,
|
|
"precision": 0.9815,
|
|
"predictedPositive": 216,
|
|
"recall": 0.5889,
|
|
"total": 2970,
|
|
"trueNegative": 2606,
|
|
"truePositive": 212,
|
|
"wilsonPrecisionLower95": 0.9534
|
|
},
|
|
"invitation": {
|
|
"f1": 0.5591,
|
|
"falseNegative": 109,
|
|
"falsePositive": 3,
|
|
"precision": 0.9595,
|
|
"predictedPositive": 74,
|
|
"recall": 0.3944,
|
|
"total": 2970,
|
|
"trueNegative": 2787,
|
|
"truePositive": 71,
|
|
"wilsonPrecisionLower95": 0.8875
|
|
},
|
|
"question": {
|
|
"f1": 0.7917,
|
|
"falseNegative": 212,
|
|
"falsePositive": 8,
|
|
"precision": 0.9812,
|
|
"predictedPositive": 426,
|
|
"recall": 0.6635,
|
|
"total": 2970,
|
|
"trueNegative": 2332,
|
|
"truePositive": 418,
|
|
"wilsonPrecisionLower95": 0.9634
|
|
},
|
|
"replyableMessage": {
|
|
"f1": 0.6113,
|
|
"falseNegative": 1037,
|
|
"falsePositive": 48,
|
|
"precision": 0.9467,
|
|
"predictedPositive": 901,
|
|
"recall": 0.4513,
|
|
"total": 2970,
|
|
"trueNegative": 1032,
|
|
"truePositive": 853,
|
|
"wilsonPrecisionLower95": 0.9301
|
|
},
|
|
"scheduleNegotiation": {
|
|
"f1": 0.9176,
|
|
"falseNegative": 12,
|
|
"falsePositive": 2,
|
|
"precision": 0.975,
|
|
"predictedPositive": 80,
|
|
"recall": 0.8667,
|
|
"total": 2970,
|
|
"trueNegative": 2878,
|
|
"truePositive": 78,
|
|
"wilsonPrecisionLower95": 0.9134
|
|
},
|
|
"task": {
|
|
"f1": 0.1432,
|
|
"falseNegative": 664,
|
|
"falsePositive": 6,
|
|
"precision": 0.9032,
|
|
"predictedPositive": 62,
|
|
"recall": 0.0778,
|
|
"total": 2970,
|
|
"trueNegative": 2244,
|
|
"truePositive": 56,
|
|
"wilsonPrecisionLower95": 0.8045
|
|
}
|
|
},
|
|
"recordCount": 2970
|
|
},
|
|
"thresholds": {
|
|
"blessing": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"complaint": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9809,
|
|
"falseNegative": 0,
|
|
"falsePositive": 7,
|
|
"precision": 0.9626,
|
|
"predictedPositive": 187,
|
|
"recall": 1.0,
|
|
"total": 1485,
|
|
"trueNegative": 1298,
|
|
"truePositive": 180,
|
|
"wilsonPrecisionLower95": 0.9248
|
|
},
|
|
"threshold": 0.99
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"confirmationDecision": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8625,
|
|
"falseNegative": 21,
|
|
"falsePositive": 1,
|
|
"precision": 0.9857,
|
|
"predictedPositive": 70,
|
|
"recall": 0.7667,
|
|
"total": 1485,
|
|
"trueNegative": 1394,
|
|
"truePositive": 69,
|
|
"wilsonPrecisionLower95": 0.9234
|
|
},
|
|
"threshold": 0.905
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.828,
|
|
"falseNegative": 25,
|
|
"falsePositive": 2,
|
|
"precision": 0.9701,
|
|
"predictedPositive": 67,
|
|
"recall": 0.7222,
|
|
"total": 1485,
|
|
"trueNegative": 1393,
|
|
"truePositive": 65,
|
|
"wilsonPrecisionLower95": 0.8975
|
|
},
|
|
"threshold": 0.5
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.8328,
|
|
"falseNegative": 48,
|
|
"falsePositive": 5,
|
|
"precision": 0.9635,
|
|
"predictedPositive": 137,
|
|
"recall": 0.7333,
|
|
"total": 2970,
|
|
"trueNegative": 2785,
|
|
"truePositive": 132,
|
|
"wilsonPrecisionLower95": 0.9174
|
|
},
|
|
"threshold": 0.685
|
|
},
|
|
"followUpReminder": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8788,
|
|
"falseNegative": 35,
|
|
"falsePositive": 5,
|
|
"precision": 0.9667,
|
|
"predictedPositive": 150,
|
|
"recall": 0.8056,
|
|
"total": 1485,
|
|
"trueNegative": 1300,
|
|
"truePositive": 145,
|
|
"wilsonPrecisionLower95": 0.9243
|
|
},
|
|
"threshold": 0.845
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.6113,
|
|
"falseNegative": 99,
|
|
"falsePositive": 4,
|
|
"precision": 0.9529,
|
|
"predictedPositive": 85,
|
|
"recall": 0.45,
|
|
"total": 1485,
|
|
"trueNegative": 1301,
|
|
"truePositive": 81,
|
|
"wilsonPrecisionLower95": 0.8852
|
|
},
|
|
"threshold": 0.97
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.7682,
|
|
"falseNegative": 128,
|
|
"falsePositive": 12,
|
|
"precision": 0.9508,
|
|
"predictedPositive": 244,
|
|
"recall": 0.6444,
|
|
"total": 2970,
|
|
"trueNegative": 2598,
|
|
"truePositive": 232,
|
|
"wilsonPrecisionLower95": 0.916
|
|
},
|
|
"threshold": 0.91
|
|
},
|
|
"invitation": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"question": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.847,
|
|
"falseNegative": 77,
|
|
"falsePositive": 9,
|
|
"precision": 0.9636,
|
|
"predictedPositive": 247,
|
|
"recall": 0.7556,
|
|
"total": 1485,
|
|
"trueNegative": 1161,
|
|
"truePositive": 238,
|
|
"wilsonPrecisionLower95": 0.9322
|
|
},
|
|
"threshold": 0.995
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.7168,
|
|
"falseNegative": 134,
|
|
"falsePositive": 9,
|
|
"precision": 0.9526,
|
|
"predictedPositive": 190,
|
|
"recall": 0.5746,
|
|
"total": 1485,
|
|
"trueNegative": 1161,
|
|
"truePositive": 181,
|
|
"wilsonPrecisionLower95": 0.9124
|
|
},
|
|
"threshold": 0.865
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.7524,
|
|
"falseNegative": 238,
|
|
"falsePositive": 20,
|
|
"precision": 0.9515,
|
|
"predictedPositive": 412,
|
|
"recall": 0.6222,
|
|
"total": 2970,
|
|
"trueNegative": 2320,
|
|
"truePositive": 392,
|
|
"wilsonPrecisionLower95": 0.9262
|
|
},
|
|
"threshold": 0.975
|
|
},
|
|
"replyableMessage": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9349,
|
|
"falseNegative": 76,
|
|
"falsePositive": 45,
|
|
"precision": 0.9508,
|
|
"predictedPositive": 914,
|
|
"recall": 0.9196,
|
|
"total": 1485,
|
|
"trueNegative": 495,
|
|
"truePositive": 869,
|
|
"wilsonPrecisionLower95": 0.9348
|
|
},
|
|
"threshold": 0.955
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"scheduleNegotiation": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 1.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 45,
|
|
"recall": 1.0,
|
|
"total": 1485,
|
|
"trueNegative": 1440,
|
|
"truePositive": 45,
|
|
"wilsonPrecisionLower95": 0.9213
|
|
},
|
|
"threshold": 0.5
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.967,
|
|
"falseNegative": 1,
|
|
"falsePositive": 2,
|
|
"precision": 0.9565,
|
|
"predictedPositive": 46,
|
|
"recall": 0.9778,
|
|
"total": 1485,
|
|
"trueNegative": 1438,
|
|
"truePositive": 44,
|
|
"wilsonPrecisionLower95": 0.8547
|
|
},
|
|
"threshold": 0.995
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.9727,
|
|
"falseNegative": 1,
|
|
"falsePositive": 4,
|
|
"precision": 0.957,
|
|
"predictedPositive": 93,
|
|
"recall": 0.9889,
|
|
"total": 2970,
|
|
"trueNegative": 2876,
|
|
"truePositive": 89,
|
|
"wilsonPrecisionLower95": 0.8946
|
|
},
|
|
"threshold": 0.64
|
|
},
|
|
"task": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"trainingRecords": 15345,
|
|
"validation": {
|
|
"intentsMeeting95Precision": 8,
|
|
"macroF1": 0.6332,
|
|
"macroPrecision": 0.9668,
|
|
"macroRecall": 0.5246,
|
|
"minimumPrecision": 0.9,
|
|
"perIntent": {
|
|
"blessing": {
|
|
"f1": 0.3019,
|
|
"falseNegative": 74,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 16,
|
|
"recall": 0.1778,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 16,
|
|
"wilsonPrecisionLower95": 0.8064
|
|
},
|
|
"complaint": {
|
|
"f1": 0.8199,
|
|
"falseNegative": 105,
|
|
"falsePositive": 7,
|
|
"precision": 0.9733,
|
|
"predictedPositive": 262,
|
|
"recall": 0.7083,
|
|
"total": 2970,
|
|
"trueNegative": 2603,
|
|
"truePositive": 255,
|
|
"wilsonPrecisionLower95": 0.9459
|
|
},
|
|
"confirmationDecision": {
|
|
"f1": 0.8454,
|
|
"falseNegative": 46,
|
|
"falsePositive": 3,
|
|
"precision": 0.9781,
|
|
"predictedPositive": 137,
|
|
"recall": 0.7444,
|
|
"total": 2970,
|
|
"trueNegative": 2787,
|
|
"truePositive": 134,
|
|
"wilsonPrecisionLower95": 0.9376
|
|
},
|
|
"followUpReminder": {
|
|
"f1": 0.7597,
|
|
"falseNegative": 134,
|
|
"falsePositive": 9,
|
|
"precision": 0.9617,
|
|
"predictedPositive": 235,
|
|
"recall": 0.6278,
|
|
"total": 2970,
|
|
"trueNegative": 2601,
|
|
"truePositive": 226,
|
|
"wilsonPrecisionLower95": 0.9288
|
|
},
|
|
"invitation": {
|
|
"f1": 0.3273,
|
|
"falseNegative": 144,
|
|
"falsePositive": 4,
|
|
"precision": 0.9,
|
|
"predictedPositive": 40,
|
|
"recall": 0.2,
|
|
"total": 2970,
|
|
"trueNegative": 2786,
|
|
"truePositive": 36,
|
|
"wilsonPrecisionLower95": 0.7695
|
|
},
|
|
"question": {
|
|
"f1": 0.7854,
|
|
"falseNegative": 211,
|
|
"falsePositive": 18,
|
|
"precision": 0.9588,
|
|
"predictedPositive": 437,
|
|
"recall": 0.6651,
|
|
"total": 2970,
|
|
"trueNegative": 2322,
|
|
"truePositive": 419,
|
|
"wilsonPrecisionLower95": 0.9358
|
|
},
|
|
"replyableMessage": {
|
|
"f1": 0.6272,
|
|
"falseNegative": 1006,
|
|
"falsePositive": 45,
|
|
"precision": 0.9516,
|
|
"predictedPositive": 929,
|
|
"recall": 0.4677,
|
|
"total": 2970,
|
|
"trueNegative": 1035,
|
|
"truePositive": 884,
|
|
"wilsonPrecisionLower95": 0.9358
|
|
},
|
|
"scheduleNegotiation": {
|
|
"f1": 0.9834,
|
|
"falseNegative": 1,
|
|
"falsePositive": 2,
|
|
"precision": 0.978,
|
|
"predictedPositive": 91,
|
|
"recall": 0.9889,
|
|
"total": 2970,
|
|
"trueNegative": 2878,
|
|
"truePositive": 89,
|
|
"wilsonPrecisionLower95": 0.9234
|
|
},
|
|
"task": {
|
|
"f1": 0.2482,
|
|
"falseNegative": 618,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 102,
|
|
"recall": 0.1417,
|
|
"total": 2970,
|
|
"trueNegative": 2250,
|
|
"truePositive": 102,
|
|
"wilsonPrecisionLower95": 0.9637
|
|
}
|
|
},
|
|
"recordCount": 2970
|
|
}
|
|
},
|
|
{
|
|
"configuration": {
|
|
"alpha": 3.5e-06,
|
|
"augmentation": "prefix",
|
|
"char_max": 6,
|
|
"char_min": 2,
|
|
"external_weight": 0.8500000000000001,
|
|
"hard_example_weight": 2.2,
|
|
"l1_ratio": 0.2,
|
|
"max_char_features": 85000,
|
|
"max_word_features": 42000,
|
|
"min_df": 1,
|
|
"pseudo_weight": 0.375,
|
|
"round": 15,
|
|
"word_max": 3
|
|
},
|
|
"featureCount": 127000,
|
|
"pseudoLabelCounts": {
|
|
"blessing": {
|
|
"negative": 250,
|
|
"positive": 1
|
|
},
|
|
"complaint": {
|
|
"negative": 250,
|
|
"positive": 8
|
|
},
|
|
"confirmationDecision": {
|
|
"negative": 250,
|
|
"positive": 30
|
|
},
|
|
"followUpReminder": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"invitation": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"question": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"replyableMessage": {
|
|
"negative": 39,
|
|
"positive": 250
|
|
},
|
|
"scheduleNegotiation": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"task": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
}
|
|
},
|
|
"round": 15,
|
|
"selectionObjective": 0.7896,
|
|
"syntheticRegression": {
|
|
"intentsMeeting95Precision": 7,
|
|
"macroF1": 0.7019,
|
|
"macroPrecision": 0.9601,
|
|
"macroRecall": 0.6059,
|
|
"minimumPrecision": 0.8634,
|
|
"perIntent": {
|
|
"blessing": {
|
|
"f1": 0.2,
|
|
"falseNegative": 80,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 10,
|
|
"recall": 0.1111,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 10,
|
|
"wilsonPrecisionLower95": 0.7225
|
|
},
|
|
"complaint": {
|
|
"f1": 0.9641,
|
|
"falseNegative": 24,
|
|
"falsePositive": 1,
|
|
"precision": 0.997,
|
|
"predictedPositive": 337,
|
|
"recall": 0.9333,
|
|
"total": 2970,
|
|
"trueNegative": 2609,
|
|
"truePositive": 336,
|
|
"wilsonPrecisionLower95": 0.9834
|
|
},
|
|
"confirmationDecision": {
|
|
"f1": 0.8152,
|
|
"falseNegative": 41,
|
|
"falsePositive": 22,
|
|
"precision": 0.8634,
|
|
"predictedPositive": 161,
|
|
"recall": 0.7722,
|
|
"total": 2970,
|
|
"trueNegative": 2768,
|
|
"truePositive": 139,
|
|
"wilsonPrecisionLower95": 0.8018
|
|
},
|
|
"followUpReminder": {
|
|
"f1": 0.6115,
|
|
"falseNegative": 201,
|
|
"falsePositive": 1,
|
|
"precision": 0.9938,
|
|
"predictedPositive": 160,
|
|
"recall": 0.4417,
|
|
"total": 2970,
|
|
"trueNegative": 2609,
|
|
"truePositive": 159,
|
|
"wilsonPrecisionLower95": 0.9655
|
|
},
|
|
"invitation": {
|
|
"f1": 0.6084,
|
|
"falseNegative": 100,
|
|
"falsePositive": 3,
|
|
"precision": 0.9639,
|
|
"predictedPositive": 83,
|
|
"recall": 0.4444,
|
|
"total": 2970,
|
|
"trueNegative": 2787,
|
|
"truePositive": 80,
|
|
"wilsonPrecisionLower95": 0.899
|
|
},
|
|
"question": {
|
|
"f1": 0.5421,
|
|
"falseNegative": 395,
|
|
"falsePositive": 2,
|
|
"precision": 0.9916,
|
|
"predictedPositive": 237,
|
|
"recall": 0.373,
|
|
"total": 2970,
|
|
"trueNegative": 2338,
|
|
"truePositive": 235,
|
|
"wilsonPrecisionLower95": 0.9698
|
|
},
|
|
"replyableMessage": {
|
|
"f1": 0.862,
|
|
"falseNegative": 410,
|
|
"falsePositive": 64,
|
|
"precision": 0.9585,
|
|
"predictedPositive": 1544,
|
|
"recall": 0.7831,
|
|
"total": 2970,
|
|
"trueNegative": 1016,
|
|
"truePositive": 1480,
|
|
"wilsonPrecisionLower95": 0.9474
|
|
},
|
|
"scheduleNegotiation": {
|
|
"f1": 0.9379,
|
|
"falseNegative": 7,
|
|
"falsePositive": 4,
|
|
"precision": 0.954,
|
|
"predictedPositive": 87,
|
|
"recall": 0.9222,
|
|
"total": 2970,
|
|
"trueNegative": 2876,
|
|
"truePositive": 83,
|
|
"wilsonPrecisionLower95": 0.8877
|
|
},
|
|
"task": {
|
|
"f1": 0.7763,
|
|
"falseNegative": 236,
|
|
"falsePositive": 43,
|
|
"precision": 0.9184,
|
|
"predictedPositive": 527,
|
|
"recall": 0.6722,
|
|
"total": 2970,
|
|
"trueNegative": 2207,
|
|
"truePositive": 484,
|
|
"wilsonPrecisionLower95": 0.8919
|
|
}
|
|
},
|
|
"recordCount": 2970
|
|
},
|
|
"thresholds": {
|
|
"blessing": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"complaint": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9836,
|
|
"falseNegative": 0,
|
|
"falsePositive": 6,
|
|
"precision": 0.9677,
|
|
"predictedPositive": 186,
|
|
"recall": 1.0,
|
|
"total": 1485,
|
|
"trueNegative": 1299,
|
|
"truePositive": 180,
|
|
"wilsonPrecisionLower95": 0.9314
|
|
},
|
|
"threshold": 0.945
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.928,
|
|
"falseNegative": 19,
|
|
"falsePositive": 6,
|
|
"precision": 0.9641,
|
|
"predictedPositive": 167,
|
|
"recall": 0.8944,
|
|
"total": 1485,
|
|
"trueNegative": 1299,
|
|
"truePositive": 161,
|
|
"wilsonPrecisionLower95": 0.9238
|
|
},
|
|
"threshold": 0.77
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.9365,
|
|
"falseNegative": 28,
|
|
"falsePositive": 17,
|
|
"precision": 0.9513,
|
|
"predictedPositive": 349,
|
|
"recall": 0.9222,
|
|
"total": 2970,
|
|
"trueNegative": 2593,
|
|
"truePositive": 332,
|
|
"wilsonPrecisionLower95": 0.9234
|
|
},
|
|
"threshold": 0.88
|
|
},
|
|
"confirmationDecision": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.954,
|
|
"falseNegative": 7,
|
|
"falsePositive": 1,
|
|
"precision": 0.9881,
|
|
"predictedPositive": 84,
|
|
"recall": 0.9222,
|
|
"total": 1485,
|
|
"trueNegative": 1394,
|
|
"truePositive": 83,
|
|
"wilsonPrecisionLower95": 0.9356
|
|
},
|
|
"threshold": 0.85
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.6906,
|
|
"falseNegative": 42,
|
|
"falsePositive": 1,
|
|
"precision": 0.9796,
|
|
"predictedPositive": 49,
|
|
"recall": 0.5333,
|
|
"total": 1485,
|
|
"trueNegative": 1394,
|
|
"truePositive": 48,
|
|
"wilsonPrecisionLower95": 0.8931
|
|
},
|
|
"threshold": 0.985
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.8598,
|
|
"falseNegative": 39,
|
|
"falsePositive": 7,
|
|
"precision": 0.9527,
|
|
"predictedPositive": 148,
|
|
"recall": 0.7833,
|
|
"total": 2970,
|
|
"trueNegative": 2783,
|
|
"truePositive": 141,
|
|
"wilsonPrecisionLower95": 0.9056
|
|
},
|
|
"threshold": 0.935
|
|
},
|
|
"followUpReminder": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8328,
|
|
"falseNegative": 48,
|
|
"falsePositive": 5,
|
|
"precision": 0.9635,
|
|
"predictedPositive": 137,
|
|
"recall": 0.7333,
|
|
"total": 1485,
|
|
"trueNegative": 1300,
|
|
"truePositive": 132,
|
|
"wilsonPrecisionLower95": 0.9174
|
|
},
|
|
"threshold": 0.99
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"invitation": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9133,
|
|
"falseNegative": 11,
|
|
"falsePositive": 4,
|
|
"precision": 0.9518,
|
|
"predictedPositive": 83,
|
|
"recall": 0.8778,
|
|
"total": 1485,
|
|
"trueNegative": 1391,
|
|
"truePositive": 79,
|
|
"wilsonPrecisionLower95": 0.8825
|
|
},
|
|
"threshold": 0.97
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"question": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.7548,
|
|
"falseNegative": 118,
|
|
"falsePositive": 10,
|
|
"precision": 0.9517,
|
|
"predictedPositive": 207,
|
|
"recall": 0.6254,
|
|
"total": 1485,
|
|
"trueNegative": 1160,
|
|
"truePositive": 197,
|
|
"wilsonPrecisionLower95": 0.9134
|
|
},
|
|
"threshold": 0.8
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.7678,
|
|
"falseNegative": 225,
|
|
"falsePositive": 20,
|
|
"precision": 0.9529,
|
|
"predictedPositive": 425,
|
|
"recall": 0.6429,
|
|
"total": 2970,
|
|
"trueNegative": 2320,
|
|
"truePositive": 405,
|
|
"wilsonPrecisionLower95": 0.9284
|
|
},
|
|
"threshold": 0.985
|
|
},
|
|
"replyableMessage": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.907,
|
|
"falseNegative": 126,
|
|
"falsePositive": 42,
|
|
"precision": 0.9512,
|
|
"predictedPositive": 861,
|
|
"recall": 0.8667,
|
|
"total": 1485,
|
|
"trueNegative": 498,
|
|
"truePositive": 819,
|
|
"wilsonPrecisionLower95": 0.9347
|
|
},
|
|
"threshold": 0.98
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8252,
|
|
"falseNegative": 258,
|
|
"falsePositive": 33,
|
|
"precision": 0.9542,
|
|
"predictedPositive": 720,
|
|
"recall": 0.727,
|
|
"total": 1485,
|
|
"trueNegative": 507,
|
|
"truePositive": 687,
|
|
"wilsonPrecisionLower95": 0.9363
|
|
},
|
|
"threshold": 0.96
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.8678,
|
|
"falseNegative": 384,
|
|
"falsePositive": 75,
|
|
"precision": 0.9526,
|
|
"predictedPositive": 1581,
|
|
"recall": 0.7968,
|
|
"total": 2970,
|
|
"trueNegative": 1005,
|
|
"truePositive": 1506,
|
|
"wilsonPrecisionLower95": 0.9409
|
|
},
|
|
"threshold": 0.965
|
|
},
|
|
"scheduleNegotiation": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9157,
|
|
"falseNegative": 7,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 38,
|
|
"recall": 0.8444,
|
|
"total": 1485,
|
|
"trueNegative": 1440,
|
|
"truePositive": 38,
|
|
"wilsonPrecisionLower95": 0.9082
|
|
},
|
|
"threshold": 0.5
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9556,
|
|
"falseNegative": 2,
|
|
"falsePositive": 2,
|
|
"precision": 0.9556,
|
|
"predictedPositive": 45,
|
|
"recall": 0.9556,
|
|
"total": 1485,
|
|
"trueNegative": 1438,
|
|
"truePositive": 43,
|
|
"wilsonPrecisionLower95": 0.8517
|
|
},
|
|
"threshold": 0.5
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.9364,
|
|
"falseNegative": 9,
|
|
"falsePositive": 2,
|
|
"precision": 0.9759,
|
|
"predictedPositive": 83,
|
|
"recall": 0.9,
|
|
"total": 2970,
|
|
"trueNegative": 2878,
|
|
"truePositive": 81,
|
|
"wilsonPrecisionLower95": 0.9163
|
|
},
|
|
"threshold": 0.5
|
|
},
|
|
"task": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8593,
|
|
"falseNegative": 79,
|
|
"falsePositive": 13,
|
|
"precision": 0.9558,
|
|
"predictedPositive": 294,
|
|
"recall": 0.7806,
|
|
"total": 1485,
|
|
"trueNegative": 1112,
|
|
"truePositive": 281,
|
|
"wilsonPrecisionLower95": 0.9258
|
|
},
|
|
"threshold": 0.98
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8166,
|
|
"falseNegative": 104,
|
|
"falsePositive": 11,
|
|
"precision": 0.9588,
|
|
"predictedPositive": 267,
|
|
"recall": 0.7111,
|
|
"total": 1485,
|
|
"trueNegative": 1114,
|
|
"truePositive": 256,
|
|
"wilsonPrecisionLower95": 0.9277
|
|
},
|
|
"threshold": 0.96
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.8434,
|
|
"falseNegative": 176,
|
|
"falsePositive": 26,
|
|
"precision": 0.9544,
|
|
"predictedPositive": 570,
|
|
"recall": 0.7556,
|
|
"total": 2970,
|
|
"trueNegative": 2224,
|
|
"truePositive": 544,
|
|
"wilsonPrecisionLower95": 0.934
|
|
},
|
|
"threshold": 0.965
|
|
}
|
|
},
|
|
"trainingRecords": 15345,
|
|
"validation": {
|
|
"intentsMeeting95Precision": 8,
|
|
"macroF1": 0.711,
|
|
"macroPrecision": 0.9652,
|
|
"macroRecall": 0.6071,
|
|
"minimumPrecision": 0.9441,
|
|
"perIntent": {
|
|
"blessing": {
|
|
"f1": 0.3333,
|
|
"falseNegative": 72,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 18,
|
|
"recall": 0.2,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 18,
|
|
"wilsonPrecisionLower95": 0.8241
|
|
},
|
|
"complaint": {
|
|
"f1": 0.9565,
|
|
"falseNegative": 19,
|
|
"falsePositive": 12,
|
|
"precision": 0.966,
|
|
"predictedPositive": 353,
|
|
"recall": 0.9472,
|
|
"total": 2970,
|
|
"trueNegative": 2598,
|
|
"truePositive": 341,
|
|
"wilsonPrecisionLower95": 0.9415
|
|
},
|
|
"confirmationDecision": {
|
|
"f1": 0.8371,
|
|
"falseNegative": 49,
|
|
"falsePositive": 2,
|
|
"precision": 0.985,
|
|
"predictedPositive": 133,
|
|
"recall": 0.7278,
|
|
"total": 2970,
|
|
"trueNegative": 2788,
|
|
"truePositive": 131,
|
|
"wilsonPrecisionLower95": 0.9468
|
|
},
|
|
"followUpReminder": {
|
|
"f1": 0.5368,
|
|
"falseNegative": 225,
|
|
"falsePositive": 8,
|
|
"precision": 0.9441,
|
|
"predictedPositive": 143,
|
|
"recall": 0.375,
|
|
"total": 2970,
|
|
"trueNegative": 2602,
|
|
"truePositive": 135,
|
|
"wilsonPrecisionLower95": 0.8935
|
|
},
|
|
"invitation": {
|
|
"f1": 0.6061,
|
|
"falseNegative": 100,
|
|
"falsePositive": 4,
|
|
"precision": 0.9524,
|
|
"predictedPositive": 84,
|
|
"recall": 0.4444,
|
|
"total": 2970,
|
|
"trueNegative": 2786,
|
|
"truePositive": 80,
|
|
"wilsonPrecisionLower95": 0.8839
|
|
},
|
|
"question": {
|
|
"f1": 0.487,
|
|
"falseNegative": 424,
|
|
"falsePositive": 10,
|
|
"precision": 0.9537,
|
|
"predictedPositive": 216,
|
|
"recall": 0.327,
|
|
"total": 2970,
|
|
"trueNegative": 2330,
|
|
"truePositive": 206,
|
|
"wilsonPrecisionLower95": 0.9169
|
|
},
|
|
"replyableMessage": {
|
|
"f1": 0.8678,
|
|
"falseNegative": 384,
|
|
"falsePositive": 75,
|
|
"precision": 0.9526,
|
|
"predictedPositive": 1581,
|
|
"recall": 0.7968,
|
|
"total": 2970,
|
|
"trueNegative": 1005,
|
|
"truePositive": 1506,
|
|
"wilsonPrecisionLower95": 0.9409
|
|
},
|
|
"scheduleNegotiation": {
|
|
"f1": 0.9364,
|
|
"falseNegative": 9,
|
|
"falsePositive": 2,
|
|
"precision": 0.9759,
|
|
"predictedPositive": 83,
|
|
"recall": 0.9,
|
|
"total": 2970,
|
|
"trueNegative": 2878,
|
|
"truePositive": 81,
|
|
"wilsonPrecisionLower95": 0.9163
|
|
},
|
|
"task": {
|
|
"f1": 0.8384,
|
|
"falseNegative": 183,
|
|
"falsePositive": 24,
|
|
"precision": 0.9572,
|
|
"predictedPositive": 561,
|
|
"recall": 0.7458,
|
|
"total": 2970,
|
|
"trueNegative": 2226,
|
|
"truePositive": 537,
|
|
"wilsonPrecisionLower95": 0.9371
|
|
}
|
|
},
|
|
"recordCount": 2970
|
|
}
|
|
},
|
|
{
|
|
"configuration": {
|
|
"alpha": 3e-06,
|
|
"augmentation": "numeric",
|
|
"char_max": 7,
|
|
"char_min": 2,
|
|
"external_weight": 0.8700000000000001,
|
|
"hard_example_weight": 2.3,
|
|
"l1_ratio": 0.15,
|
|
"max_char_features": 80000,
|
|
"max_word_features": 38000,
|
|
"min_df": 2,
|
|
"pseudo_weight": 0.39,
|
|
"round": 16,
|
|
"word_max": 3
|
|
},
|
|
"featureCount": 100252,
|
|
"pseudoLabelCounts": {
|
|
"blessing": {
|
|
"negative": 250,
|
|
"positive": 3
|
|
},
|
|
"complaint": {
|
|
"negative": 250,
|
|
"positive": 11
|
|
},
|
|
"confirmationDecision": {
|
|
"negative": 250,
|
|
"positive": 25
|
|
},
|
|
"followUpReminder": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"invitation": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"question": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"replyableMessage": {
|
|
"negative": 76,
|
|
"positive": 250
|
|
},
|
|
"scheduleNegotiation": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"task": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
}
|
|
},
|
|
"round": 16,
|
|
"selectionObjective": 0.8105,
|
|
"syntheticRegression": {
|
|
"intentsMeeting95Precision": 5,
|
|
"macroF1": 0.726,
|
|
"macroPrecision": 0.9485,
|
|
"macroRecall": 0.6388,
|
|
"minimumPrecision": 0.8195,
|
|
"perIntent": {
|
|
"blessing": {
|
|
"f1": 0.125,
|
|
"falseNegative": 84,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 6,
|
|
"recall": 0.0667,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 6,
|
|
"wilsonPrecisionLower95": 0.6097
|
|
},
|
|
"complaint": {
|
|
"f1": 0.9132,
|
|
"falseNegative": 55,
|
|
"falsePositive": 3,
|
|
"precision": 0.9903,
|
|
"predictedPositive": 308,
|
|
"recall": 0.8472,
|
|
"total": 2970,
|
|
"trueNegative": 2607,
|
|
"truePositive": 305,
|
|
"wilsonPrecisionLower95": 0.9718
|
|
},
|
|
"confirmationDecision": {
|
|
"f1": 0.6965,
|
|
"falseNegative": 71,
|
|
"falsePositive": 24,
|
|
"precision": 0.8195,
|
|
"predictedPositive": 133,
|
|
"recall": 0.6056,
|
|
"total": 2970,
|
|
"trueNegative": 2766,
|
|
"truePositive": 109,
|
|
"wilsonPrecisionLower95": 0.7455
|
|
},
|
|
"followUpReminder": {
|
|
"f1": 0.7409,
|
|
"falseNegative": 147,
|
|
"falsePositive": 2,
|
|
"precision": 0.9907,
|
|
"predictedPositive": 215,
|
|
"recall": 0.5917,
|
|
"total": 2970,
|
|
"trueNegative": 2608,
|
|
"truePositive": 213,
|
|
"wilsonPrecisionLower95": 0.9667
|
|
},
|
|
"invitation": {
|
|
"f1": 0.6264,
|
|
"falseNegative": 97,
|
|
"falsePositive": 2,
|
|
"precision": 0.9765,
|
|
"predictedPositive": 85,
|
|
"recall": 0.4611,
|
|
"total": 2970,
|
|
"trueNegative": 2788,
|
|
"truePositive": 83,
|
|
"wilsonPrecisionLower95": 0.9182
|
|
},
|
|
"question": {
|
|
"f1": 0.8457,
|
|
"falseNegative": 164,
|
|
"falsePositive": 6,
|
|
"precision": 0.9873,
|
|
"predictedPositive": 472,
|
|
"recall": 0.7397,
|
|
"total": 2970,
|
|
"trueNegative": 2334,
|
|
"truePositive": 466,
|
|
"wilsonPrecisionLower95": 0.9725
|
|
},
|
|
"replyableMessage": {
|
|
"f1": 0.8725,
|
|
"falseNegative": 371,
|
|
"falsePositive": 73,
|
|
"precision": 0.9541,
|
|
"predictedPositive": 1592,
|
|
"recall": 0.8037,
|
|
"total": 2970,
|
|
"trueNegative": 1007,
|
|
"truePositive": 1519,
|
|
"wilsonPrecisionLower95": 0.9427
|
|
},
|
|
"scheduleNegotiation": {
|
|
"f1": 0.9061,
|
|
"falseNegative": 8,
|
|
"falsePositive": 9,
|
|
"precision": 0.9011,
|
|
"predictedPositive": 91,
|
|
"recall": 0.9111,
|
|
"total": 2970,
|
|
"trueNegative": 2871,
|
|
"truePositive": 82,
|
|
"wilsonPrecisionLower95": 0.8226
|
|
},
|
|
"task": {
|
|
"f1": 0.8081,
|
|
"falseNegative": 200,
|
|
"falsePositive": 47,
|
|
"precision": 0.9171,
|
|
"predictedPositive": 567,
|
|
"recall": 0.7222,
|
|
"total": 2970,
|
|
"trueNegative": 2203,
|
|
"truePositive": 520,
|
|
"wilsonPrecisionLower95": 0.8915
|
|
}
|
|
},
|
|
"recordCount": 2970
|
|
},
|
|
"thresholds": {
|
|
"blessing": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"complaint": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9371,
|
|
"falseNegative": 16,
|
|
"falsePositive": 6,
|
|
"precision": 0.9647,
|
|
"predictedPositive": 170,
|
|
"recall": 0.9111,
|
|
"total": 1485,
|
|
"trueNegative": 1299,
|
|
"truePositive": 164,
|
|
"wilsonPrecisionLower95": 0.9251
|
|
},
|
|
"threshold": 0.995
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8836,
|
|
"falseNegative": 32,
|
|
"falsePositive": 7,
|
|
"precision": 0.9548,
|
|
"predictedPositive": 155,
|
|
"recall": 0.8222,
|
|
"total": 1485,
|
|
"trueNegative": 1298,
|
|
"truePositive": 148,
|
|
"wilsonPrecisionLower95": 0.9097
|
|
},
|
|
"threshold": 0.99
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.8994,
|
|
"falseNegative": 56,
|
|
"falsePositive": 12,
|
|
"precision": 0.962,
|
|
"predictedPositive": 316,
|
|
"recall": 0.8444,
|
|
"total": 2970,
|
|
"trueNegative": 2598,
|
|
"truePositive": 304,
|
|
"wilsonPrecisionLower95": 0.9348
|
|
},
|
|
"threshold": 0.995
|
|
},
|
|
"confirmationDecision": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8625,
|
|
"falseNegative": 21,
|
|
"falsePositive": 1,
|
|
"precision": 0.9857,
|
|
"predictedPositive": 70,
|
|
"recall": 0.7667,
|
|
"total": 1485,
|
|
"trueNegative": 1394,
|
|
"truePositive": 69,
|
|
"wilsonPrecisionLower95": 0.9234
|
|
},
|
|
"threshold": 0.915
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.7415,
|
|
"falseNegative": 71,
|
|
"falsePositive": 5,
|
|
"precision": 0.9561,
|
|
"predictedPositive": 114,
|
|
"recall": 0.6056,
|
|
"total": 2970,
|
|
"trueNegative": 2785,
|
|
"truePositive": 109,
|
|
"wilsonPrecisionLower95": 0.9014
|
|
},
|
|
"threshold": 0.96
|
|
},
|
|
"followUpReminder": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8667,
|
|
"falseNegative": 37,
|
|
"falsePositive": 7,
|
|
"precision": 0.9533,
|
|
"predictedPositive": 150,
|
|
"recall": 0.7944,
|
|
"total": 1485,
|
|
"trueNegative": 1298,
|
|
"truePositive": 143,
|
|
"wilsonPrecisionLower95": 0.9068
|
|
},
|
|
"threshold": 0.96
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.6031,
|
|
"falseNegative": 101,
|
|
"falsePositive": 3,
|
|
"precision": 0.9634,
|
|
"predictedPositive": 82,
|
|
"recall": 0.4389,
|
|
"total": 1485,
|
|
"trueNegative": 1302,
|
|
"truePositive": 79,
|
|
"wilsonPrecisionLower95": 0.8979
|
|
},
|
|
"threshold": 0.97
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.7492,
|
|
"falseNegative": 139,
|
|
"falsePositive": 9,
|
|
"precision": 0.9609,
|
|
"predictedPositive": 230,
|
|
"recall": 0.6139,
|
|
"total": 2970,
|
|
"trueNegative": 2601,
|
|
"truePositive": 221,
|
|
"wilsonPrecisionLower95": 0.9273
|
|
},
|
|
"threshold": 0.97
|
|
},
|
|
"invitation": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.3604,
|
|
"falseNegative": 70,
|
|
"falsePositive": 1,
|
|
"precision": 0.9524,
|
|
"predictedPositive": 21,
|
|
"recall": 0.2222,
|
|
"total": 1485,
|
|
"trueNegative": 1394,
|
|
"truePositive": 20,
|
|
"wilsonPrecisionLower95": 0.7733
|
|
},
|
|
"threshold": 0.965
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"question": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8273,
|
|
"falseNegative": 85,
|
|
"falsePositive": 11,
|
|
"precision": 0.9544,
|
|
"predictedPositive": 241,
|
|
"recall": 0.7302,
|
|
"total": 1485,
|
|
"trueNegative": 1159,
|
|
"truePositive": 230,
|
|
"wilsonPrecisionLower95": 0.9201
|
|
},
|
|
"threshold": 0.985
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.797,
|
|
"falseNegative": 99,
|
|
"falsePositive": 11,
|
|
"precision": 0.9515,
|
|
"predictedPositive": 227,
|
|
"recall": 0.6857,
|
|
"total": 1485,
|
|
"trueNegative": 1159,
|
|
"truePositive": 216,
|
|
"wilsonPrecisionLower95": 0.9153
|
|
},
|
|
"threshold": 0.835
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.8051,
|
|
"falseNegative": 190,
|
|
"falsePositive": 23,
|
|
"precision": 0.9503,
|
|
"predictedPositive": 463,
|
|
"recall": 0.6984,
|
|
"total": 2970,
|
|
"trueNegative": 2317,
|
|
"truePositive": 440,
|
|
"wilsonPrecisionLower95": 0.9266
|
|
},
|
|
"threshold": 0.94
|
|
},
|
|
"replyableMessage": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9343,
|
|
"falseNegative": 77,
|
|
"falsePositive": 45,
|
|
"precision": 0.9507,
|
|
"predictedPositive": 913,
|
|
"recall": 0.9185,
|
|
"total": 1485,
|
|
"trueNegative": 495,
|
|
"truePositive": 868,
|
|
"wilsonPrecisionLower95": 0.9347
|
|
},
|
|
"threshold": 0.91
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8207,
|
|
"falseNegative": 263,
|
|
"falsePositive": 35,
|
|
"precision": 0.9512,
|
|
"predictedPositive": 717,
|
|
"recall": 0.7217,
|
|
"total": 1485,
|
|
"trueNegative": 505,
|
|
"truePositive": 682,
|
|
"wilsonPrecisionLower95": 0.9329
|
|
},
|
|
"threshold": 0.98
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.8572,
|
|
"falseNegative": 419,
|
|
"falsePositive": 71,
|
|
"precision": 0.954,
|
|
"predictedPositive": 1542,
|
|
"recall": 0.7783,
|
|
"total": 2970,
|
|
"trueNegative": 1009,
|
|
"truePositive": 1471,
|
|
"wilsonPrecisionLower95": 0.9423
|
|
},
|
|
"threshold": 0.98
|
|
},
|
|
"scheduleNegotiation": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9888,
|
|
"falseNegative": 1,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 44,
|
|
"recall": 0.9778,
|
|
"total": 1485,
|
|
"trueNegative": 1440,
|
|
"truePositive": 44,
|
|
"wilsonPrecisionLower95": 0.9197
|
|
},
|
|
"threshold": 0.5
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.989,
|
|
"falseNegative": 0,
|
|
"falsePositive": 1,
|
|
"precision": 0.9783,
|
|
"predictedPositive": 46,
|
|
"recall": 1.0,
|
|
"total": 1485,
|
|
"trueNegative": 1439,
|
|
"truePositive": 45,
|
|
"wilsonPrecisionLower95": 0.8866
|
|
},
|
|
"threshold": 0.955
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.9778,
|
|
"falseNegative": 2,
|
|
"falsePositive": 2,
|
|
"precision": 0.9778,
|
|
"predictedPositive": 90,
|
|
"recall": 0.9778,
|
|
"total": 2970,
|
|
"trueNegative": 2878,
|
|
"truePositive": 88,
|
|
"wilsonPrecisionLower95": 0.9226
|
|
},
|
|
"threshold": 0.935
|
|
},
|
|
"task": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8836,
|
|
"falseNegative": 64,
|
|
"falsePositive": 14,
|
|
"precision": 0.9548,
|
|
"predictedPositive": 310,
|
|
"recall": 0.8222,
|
|
"total": 1485,
|
|
"trueNegative": 1111,
|
|
"truePositive": 296,
|
|
"wilsonPrecisionLower95": 0.9256
|
|
},
|
|
"threshold": 0.975
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.7896,
|
|
"falseNegative": 118,
|
|
"falsePositive": 11,
|
|
"precision": 0.9565,
|
|
"predictedPositive": 253,
|
|
"recall": 0.6722,
|
|
"total": 1485,
|
|
"trueNegative": 1114,
|
|
"truePositive": 242,
|
|
"wilsonPrecisionLower95": 0.9238
|
|
},
|
|
"threshold": 0.96
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.8316,
|
|
"falseNegative": 189,
|
|
"falsePositive": 26,
|
|
"precision": 0.9533,
|
|
"predictedPositive": 557,
|
|
"recall": 0.7375,
|
|
"total": 2970,
|
|
"trueNegative": 2224,
|
|
"truePositive": 531,
|
|
"wilsonPrecisionLower95": 0.9325
|
|
},
|
|
"threshold": 0.97
|
|
}
|
|
},
|
|
"trainingRecords": 16726,
|
|
"validation": {
|
|
"intentsMeeting95Precision": 9,
|
|
"macroF1": 0.7397,
|
|
"macroPrecision": 0.9707,
|
|
"macroRecall": 0.6388,
|
|
"minimumPrecision": 0.9509,
|
|
"perIntent": {
|
|
"blessing": {
|
|
"f1": 0.3636,
|
|
"falseNegative": 70,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 20,
|
|
"recall": 0.2222,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 20,
|
|
"wilsonPrecisionLower95": 0.8389
|
|
},
|
|
"complaint": {
|
|
"f1": 0.9109,
|
|
"falseNegative": 48,
|
|
"falsePositive": 13,
|
|
"precision": 0.96,
|
|
"predictedPositive": 325,
|
|
"recall": 0.8667,
|
|
"total": 2970,
|
|
"trueNegative": 2597,
|
|
"truePositive": 312,
|
|
"wilsonPrecisionLower95": 0.9328
|
|
},
|
|
"confirmationDecision": {
|
|
"f1": 0.6236,
|
|
"falseNegative": 98,
|
|
"falsePositive": 1,
|
|
"precision": 0.988,
|
|
"predictedPositive": 83,
|
|
"recall": 0.4556,
|
|
"total": 2970,
|
|
"trueNegative": 2789,
|
|
"truePositive": 82,
|
|
"wilsonPrecisionLower95": 0.9349
|
|
},
|
|
"followUpReminder": {
|
|
"f1": 0.75,
|
|
"falseNegative": 138,
|
|
"falsePositive": 10,
|
|
"precision": 0.9569,
|
|
"predictedPositive": 232,
|
|
"recall": 0.6167,
|
|
"total": 2970,
|
|
"trueNegative": 2600,
|
|
"truePositive": 222,
|
|
"wilsonPrecisionLower95": 0.9225
|
|
},
|
|
"invitation": {
|
|
"f1": 0.4917,
|
|
"falseNegative": 121,
|
|
"falsePositive": 1,
|
|
"precision": 0.9833,
|
|
"predictedPositive": 60,
|
|
"recall": 0.3278,
|
|
"total": 2970,
|
|
"trueNegative": 2789,
|
|
"truePositive": 59,
|
|
"wilsonPrecisionLower95": 0.9114
|
|
},
|
|
"question": {
|
|
"f1": 0.8124,
|
|
"falseNegative": 184,
|
|
"falsePositive": 22,
|
|
"precision": 0.953,
|
|
"predictedPositive": 468,
|
|
"recall": 0.7079,
|
|
"total": 2970,
|
|
"trueNegative": 2318,
|
|
"truePositive": 446,
|
|
"wilsonPrecisionLower95": 0.9299
|
|
},
|
|
"replyableMessage": {
|
|
"f1": 0.8807,
|
|
"falseNegative": 340,
|
|
"falsePositive": 80,
|
|
"precision": 0.9509,
|
|
"predictedPositive": 1630,
|
|
"recall": 0.8201,
|
|
"total": 2970,
|
|
"trueNegative": 1000,
|
|
"truePositive": 1550,
|
|
"wilsonPrecisionLower95": 0.9393
|
|
},
|
|
"scheduleNegotiation": {
|
|
"f1": 0.9889,
|
|
"falseNegative": 1,
|
|
"falsePositive": 1,
|
|
"precision": 0.9889,
|
|
"predictedPositive": 90,
|
|
"recall": 0.9889,
|
|
"total": 2970,
|
|
"trueNegative": 2879,
|
|
"truePositive": 89,
|
|
"wilsonPrecisionLower95": 0.9397
|
|
},
|
|
"task": {
|
|
"f1": 0.8359,
|
|
"falseNegative": 185,
|
|
"falsePositive": 25,
|
|
"precision": 0.9554,
|
|
"predictedPositive": 560,
|
|
"recall": 0.7431,
|
|
"total": 2970,
|
|
"trueNegative": 2225,
|
|
"truePositive": 535,
|
|
"wilsonPrecisionLower95": 0.9349
|
|
}
|
|
},
|
|
"recordCount": 2970
|
|
}
|
|
},
|
|
{
|
|
"configuration": {
|
|
"alpha": 2e-06,
|
|
"augmentation": "numeric",
|
|
"char_max": 7,
|
|
"char_min": 3,
|
|
"external_weight": 0.8900000000000001,
|
|
"hard_example_weight": 2.5,
|
|
"l1_ratio": 0.2,
|
|
"max_char_features": 85000,
|
|
"max_word_features": 42000,
|
|
"min_df": 1,
|
|
"pseudo_weight": 0.405,
|
|
"round": 17,
|
|
"word_max": 3
|
|
},
|
|
"featureCount": 127000,
|
|
"pseudoLabelCounts": {
|
|
"blessing": {
|
|
"negative": 250,
|
|
"positive": 1
|
|
},
|
|
"complaint": {
|
|
"negative": 250,
|
|
"positive": 4
|
|
},
|
|
"confirmationDecision": {
|
|
"negative": 250,
|
|
"positive": 23
|
|
},
|
|
"followUpReminder": {
|
|
"negative": 250,
|
|
"positive": 3
|
|
},
|
|
"invitation": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"question": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"replyableMessage": {
|
|
"negative": 70,
|
|
"positive": 250
|
|
},
|
|
"scheduleNegotiation": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"task": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
}
|
|
},
|
|
"round": 17,
|
|
"selectionObjective": 0.6932,
|
|
"syntheticRegression": {
|
|
"intentsMeeting95Precision": 4,
|
|
"macroF1": 0.5086,
|
|
"macroPrecision": 0.9202,
|
|
"macroRecall": 0.4091,
|
|
"minimumPrecision": 0.7329,
|
|
"perIntent": {
|
|
"blessing": {
|
|
"f1": 0.1633,
|
|
"falseNegative": 82,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 8,
|
|
"recall": 0.0889,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 8,
|
|
"wilsonPrecisionLower95": 0.6756
|
|
},
|
|
"complaint": {
|
|
"f1": 0.914,
|
|
"falseNegative": 36,
|
|
"falsePositive": 25,
|
|
"precision": 0.9284,
|
|
"predictedPositive": 349,
|
|
"recall": 0.9,
|
|
"total": 2970,
|
|
"trueNegative": 2585,
|
|
"truePositive": 324,
|
|
"wilsonPrecisionLower95": 0.8964
|
|
},
|
|
"confirmationDecision": {
|
|
"f1": 0.6564,
|
|
"falseNegative": 73,
|
|
"falsePositive": 39,
|
|
"precision": 0.7329,
|
|
"predictedPositive": 146,
|
|
"recall": 0.5944,
|
|
"total": 2970,
|
|
"trueNegative": 2751,
|
|
"truePositive": 107,
|
|
"wilsonPrecisionLower95": 0.6558
|
|
},
|
|
"followUpReminder": {
|
|
"f1": 0.4,
|
|
"falseNegative": 270,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 90,
|
|
"recall": 0.25,
|
|
"total": 2970,
|
|
"trueNegative": 2610,
|
|
"truePositive": 90,
|
|
"wilsonPrecisionLower95": 0.9591
|
|
},
|
|
"invitation": {
|
|
"f1": 0.5954,
|
|
"falseNegative": 102,
|
|
"falsePositive": 4,
|
|
"precision": 0.9512,
|
|
"predictedPositive": 82,
|
|
"recall": 0.4333,
|
|
"total": 2970,
|
|
"trueNegative": 2786,
|
|
"truePositive": 78,
|
|
"wilsonPrecisionLower95": 0.8812
|
|
},
|
|
"question": {
|
|
"f1": 0.4976,
|
|
"falseNegative": 419,
|
|
"falsePositive": 7,
|
|
"precision": 0.9679,
|
|
"predictedPositive": 218,
|
|
"recall": 0.3349,
|
|
"total": 2970,
|
|
"trueNegative": 2333,
|
|
"truePositive": 211,
|
|
"wilsonPrecisionLower95": 0.9352
|
|
},
|
|
"replyableMessage": {
|
|
"f1": 0.2549,
|
|
"falseNegative": 1612,
|
|
"falsePositive": 13,
|
|
"precision": 0.9553,
|
|
"predictedPositive": 291,
|
|
"recall": 0.1471,
|
|
"total": 2970,
|
|
"trueNegative": 1067,
|
|
"truePositive": 278,
|
|
"wilsonPrecisionLower95": 0.9251
|
|
},
|
|
"scheduleNegotiation": {
|
|
"f1": 0.8333,
|
|
"falseNegative": 20,
|
|
"falsePositive": 8,
|
|
"precision": 0.8974,
|
|
"predictedPositive": 78,
|
|
"recall": 0.7778,
|
|
"total": 2970,
|
|
"trueNegative": 2872,
|
|
"truePositive": 70,
|
|
"wilsonPrecisionLower95": 0.8105
|
|
},
|
|
"task": {
|
|
"f1": 0.2629,
|
|
"falseNegative": 608,
|
|
"falsePositive": 20,
|
|
"precision": 0.8485,
|
|
"predictedPositive": 132,
|
|
"recall": 0.1556,
|
|
"total": 2970,
|
|
"trueNegative": 2230,
|
|
"truePositive": 112,
|
|
"wilsonPrecisionLower95": 0.7775
|
|
}
|
|
},
|
|
"recordCount": 2970
|
|
},
|
|
"thresholds": {
|
|
"blessing": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"complaint": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.989,
|
|
"falseNegative": 0,
|
|
"falsePositive": 4,
|
|
"precision": 0.9783,
|
|
"predictedPositive": 184,
|
|
"recall": 1.0,
|
|
"total": 1485,
|
|
"trueNegative": 1301,
|
|
"truePositive": 180,
|
|
"wilsonPrecisionLower95": 0.9454
|
|
},
|
|
"threshold": 0.995
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8869,
|
|
"falseNegative": 31,
|
|
"falsePositive": 7,
|
|
"precision": 0.9551,
|
|
"predictedPositive": 156,
|
|
"recall": 0.8278,
|
|
"total": 1485,
|
|
"trueNegative": 1298,
|
|
"truePositive": 149,
|
|
"wilsonPrecisionLower95": 0.9103
|
|
},
|
|
"threshold": 0.995
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.944,
|
|
"falseNegative": 23,
|
|
"falsePositive": 17,
|
|
"precision": 0.952,
|
|
"predictedPositive": 354,
|
|
"recall": 0.9361,
|
|
"total": 2970,
|
|
"trueNegative": 2593,
|
|
"truePositive": 337,
|
|
"wilsonPrecisionLower95": 0.9244
|
|
},
|
|
"threshold": 0.975
|
|
},
|
|
"confirmationDecision": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8447,
|
|
"falseNegative": 22,
|
|
"falsePositive": 3,
|
|
"precision": 0.9577,
|
|
"predictedPositive": 71,
|
|
"recall": 0.7556,
|
|
"total": 1485,
|
|
"trueNegative": 1392,
|
|
"truePositive": 68,
|
|
"wilsonPrecisionLower95": 0.883
|
|
},
|
|
"threshold": 0.825
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.7723,
|
|
"falseNegative": 63,
|
|
"falsePositive": 6,
|
|
"precision": 0.9512,
|
|
"predictedPositive": 123,
|
|
"recall": 0.65,
|
|
"total": 2970,
|
|
"trueNegative": 2784,
|
|
"truePositive": 117,
|
|
"wilsonPrecisionLower95": 0.8977
|
|
},
|
|
"threshold": 0.965
|
|
},
|
|
"followUpReminder": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"invitation": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"question": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.6625,
|
|
"falseNegative": 155,
|
|
"falsePositive": 8,
|
|
"precision": 0.9524,
|
|
"predictedPositive": 168,
|
|
"recall": 0.5079,
|
|
"total": 1485,
|
|
"trueNegative": 1162,
|
|
"truePositive": 160,
|
|
"wilsonPrecisionLower95": 0.9089
|
|
},
|
|
"threshold": 0.97
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"replyableMessage": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"scheduleNegotiation": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9655,
|
|
"falseNegative": 3,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 42,
|
|
"recall": 0.9333,
|
|
"total": 1485,
|
|
"trueNegative": 1440,
|
|
"truePositive": 42,
|
|
"wilsonPrecisionLower95": 0.9162
|
|
},
|
|
"threshold": 0.5
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9425,
|
|
"falseNegative": 4,
|
|
"falsePositive": 1,
|
|
"precision": 0.9762,
|
|
"predictedPositive": 42,
|
|
"recall": 0.9111,
|
|
"total": 1485,
|
|
"trueNegative": 1439,
|
|
"truePositive": 41,
|
|
"wilsonPrecisionLower95": 0.8768
|
|
},
|
|
"threshold": 0.73
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.9486,
|
|
"falseNegative": 7,
|
|
"falsePositive": 2,
|
|
"precision": 0.9765,
|
|
"predictedPositive": 85,
|
|
"recall": 0.9222,
|
|
"total": 2970,
|
|
"trueNegative": 2878,
|
|
"truePositive": 83,
|
|
"wilsonPrecisionLower95": 0.9182
|
|
},
|
|
"threshold": 0.5
|
|
},
|
|
"task": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"trainingRecords": 16820,
|
|
"validation": {
|
|
"intentsMeeting95Precision": 8,
|
|
"macroF1": 0.5631,
|
|
"macroPrecision": 0.9693,
|
|
"macroRecall": 0.4421,
|
|
"minimumPrecision": 0.9474,
|
|
"perIntent": {
|
|
"blessing": {
|
|
"f1": 0.3019,
|
|
"falseNegative": 74,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 16,
|
|
"recall": 0.1778,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 16,
|
|
"wilsonPrecisionLower95": 0.8064
|
|
},
|
|
"complaint": {
|
|
"f1": 0.94,
|
|
"falseNegative": 31,
|
|
"falsePositive": 11,
|
|
"precision": 0.9676,
|
|
"predictedPositive": 340,
|
|
"recall": 0.9139,
|
|
"total": 2970,
|
|
"trueNegative": 2599,
|
|
"truePositive": 329,
|
|
"wilsonPrecisionLower95": 0.943
|
|
},
|
|
"confirmationDecision": {
|
|
"f1": 0.6545,
|
|
"falseNegative": 90,
|
|
"falsePositive": 5,
|
|
"precision": 0.9474,
|
|
"predictedPositive": 95,
|
|
"recall": 0.5,
|
|
"total": 2970,
|
|
"trueNegative": 2785,
|
|
"truePositive": 90,
|
|
"wilsonPrecisionLower95": 0.8827
|
|
},
|
|
"followUpReminder": {
|
|
"f1": 0.5282,
|
|
"falseNegative": 229,
|
|
"falsePositive": 5,
|
|
"precision": 0.9632,
|
|
"predictedPositive": 136,
|
|
"recall": 0.3639,
|
|
"total": 2970,
|
|
"trueNegative": 2605,
|
|
"truePositive": 131,
|
|
"wilsonPrecisionLower95": 0.9168
|
|
},
|
|
"invitation": {
|
|
"f1": 0.4378,
|
|
"falseNegative": 129,
|
|
"falsePositive": 2,
|
|
"precision": 0.9623,
|
|
"predictedPositive": 53,
|
|
"recall": 0.2833,
|
|
"total": 2970,
|
|
"trueNegative": 2788,
|
|
"truePositive": 51,
|
|
"wilsonPrecisionLower95": 0.8725
|
|
},
|
|
"question": {
|
|
"f1": 0.5428,
|
|
"falseNegative": 392,
|
|
"falsePositive": 9,
|
|
"precision": 0.9636,
|
|
"predictedPositive": 247,
|
|
"recall": 0.3778,
|
|
"total": 2970,
|
|
"trueNegative": 2331,
|
|
"truePositive": 238,
|
|
"wilsonPrecisionLower95": 0.9322
|
|
},
|
|
"replyableMessage": {
|
|
"f1": 0.2757,
|
|
"falseNegative": 1586,
|
|
"falsePositive": 11,
|
|
"precision": 0.9651,
|
|
"predictedPositive": 315,
|
|
"recall": 0.1608,
|
|
"total": 2970,
|
|
"trueNegative": 1069,
|
|
"truePositive": 304,
|
|
"wilsonPrecisionLower95": 0.9386
|
|
},
|
|
"scheduleNegotiation": {
|
|
"f1": 0.954,
|
|
"falseNegative": 7,
|
|
"falsePositive": 1,
|
|
"precision": 0.9881,
|
|
"predictedPositive": 84,
|
|
"recall": 0.9222,
|
|
"total": 2970,
|
|
"trueNegative": 2879,
|
|
"truePositive": 83,
|
|
"wilsonPrecisionLower95": 0.9356
|
|
},
|
|
"task": {
|
|
"f1": 0.4332,
|
|
"falseNegative": 519,
|
|
"falsePositive": 7,
|
|
"precision": 0.9663,
|
|
"predictedPositive": 208,
|
|
"recall": 0.2792,
|
|
"total": 2970,
|
|
"trueNegative": 2243,
|
|
"truePositive": 201,
|
|
"wilsonPrecisionLower95": 0.9322
|
|
}
|
|
},
|
|
"recordCount": 2970
|
|
}
|
|
},
|
|
{
|
|
"configuration": {
|
|
"alpha": 1.5e-06,
|
|
"augmentation": "numeric",
|
|
"char_max": 8,
|
|
"char_min": 2,
|
|
"external_weight": 0.91,
|
|
"hard_example_weight": 2.5,
|
|
"l1_ratio": 0.2,
|
|
"max_char_features": 90000,
|
|
"max_word_features": 45000,
|
|
"min_df": 1,
|
|
"pseudo_weight": 0.42000000000000004,
|
|
"round": 18,
|
|
"word_max": 3
|
|
},
|
|
"featureCount": 135000,
|
|
"pseudoLabelCounts": {
|
|
"blessing": {
|
|
"negative": 250,
|
|
"positive": 1
|
|
},
|
|
"complaint": {
|
|
"negative": 250,
|
|
"positive": 8
|
|
},
|
|
"confirmationDecision": {
|
|
"negative": 250,
|
|
"positive": 27
|
|
},
|
|
"followUpReminder": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"invitation": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"question": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"replyableMessage": {
|
|
"negative": 90,
|
|
"positive": 32
|
|
},
|
|
"scheduleNegotiation": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"task": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
}
|
|
},
|
|
"round": 18,
|
|
"selectionObjective": 0.7725,
|
|
"syntheticRegression": {
|
|
"intentsMeeting95Precision": 4,
|
|
"macroF1": 0.6489,
|
|
"macroPrecision": 0.9452,
|
|
"macroRecall": 0.5429,
|
|
"minimumPrecision": 0.8205,
|
|
"perIntent": {
|
|
"blessing": {
|
|
"f1": 0.2178,
|
|
"falseNegative": 79,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 11,
|
|
"recall": 0.1222,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 11,
|
|
"wilsonPrecisionLower95": 0.7412
|
|
},
|
|
"complaint": {
|
|
"f1": 0.8661,
|
|
"falseNegative": 85,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 275,
|
|
"recall": 0.7639,
|
|
"total": 2970,
|
|
"trueNegative": 2610,
|
|
"truePositive": 275,
|
|
"wilsonPrecisionLower95": 0.9862
|
|
},
|
|
"confirmationDecision": {
|
|
"f1": 0.6465,
|
|
"falseNegative": 84,
|
|
"falsePositive": 21,
|
|
"precision": 0.8205,
|
|
"predictedPositive": 117,
|
|
"recall": 0.5333,
|
|
"total": 2970,
|
|
"trueNegative": 2769,
|
|
"truePositive": 96,
|
|
"wilsonPrecisionLower95": 0.7411
|
|
},
|
|
"followUpReminder": {
|
|
"f1": 0.5851,
|
|
"falseNegative": 207,
|
|
"falsePositive": 10,
|
|
"precision": 0.9387,
|
|
"predictedPositive": 163,
|
|
"recall": 0.425,
|
|
"total": 2970,
|
|
"trueNegative": 2600,
|
|
"truePositive": 153,
|
|
"wilsonPrecisionLower95": 0.8908
|
|
},
|
|
"invitation": {
|
|
"f1": 0.626,
|
|
"falseNegative": 98,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 82,
|
|
"recall": 0.4556,
|
|
"total": 2970,
|
|
"trueNegative": 2790,
|
|
"truePositive": 82,
|
|
"wilsonPrecisionLower95": 0.9552
|
|
},
|
|
"question": {
|
|
"f1": 0.7311,
|
|
"falseNegative": 267,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 363,
|
|
"recall": 0.5762,
|
|
"total": 2970,
|
|
"trueNegative": 2340,
|
|
"truePositive": 363,
|
|
"wilsonPrecisionLower95": 0.9895
|
|
},
|
|
"replyableMessage": {
|
|
"f1": 0.8939,
|
|
"falseNegative": 273,
|
|
"falsePositive": 111,
|
|
"precision": 0.9358,
|
|
"predictedPositive": 1728,
|
|
"recall": 0.8556,
|
|
"total": 2970,
|
|
"trueNegative": 969,
|
|
"truePositive": 1617,
|
|
"wilsonPrecisionLower95": 0.9232
|
|
},
|
|
"scheduleNegotiation": {
|
|
"f1": 0.9341,
|
|
"falseNegative": 5,
|
|
"falsePositive": 7,
|
|
"precision": 0.9239,
|
|
"predictedPositive": 92,
|
|
"recall": 0.9444,
|
|
"total": 2970,
|
|
"trueNegative": 2873,
|
|
"truePositive": 85,
|
|
"wilsonPrecisionLower95": 0.8512
|
|
},
|
|
"task": {
|
|
"f1": 0.3393,
|
|
"falseNegative": 569,
|
|
"falsePositive": 19,
|
|
"precision": 0.8882,
|
|
"predictedPositive": 170,
|
|
"recall": 0.2097,
|
|
"total": 2970,
|
|
"trueNegative": 2231,
|
|
"truePositive": 151,
|
|
"wilsonPrecisionLower95": 0.832
|
|
}
|
|
},
|
|
"recordCount": 2970
|
|
},
|
|
"thresholds": {
|
|
"blessing": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"complaint": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"confirmationDecision": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9486,
|
|
"falseNegative": 7,
|
|
"falsePositive": 2,
|
|
"precision": 0.9765,
|
|
"predictedPositive": 85,
|
|
"recall": 0.9222,
|
|
"total": 1485,
|
|
"trueNegative": 1393,
|
|
"truePositive": 83,
|
|
"wilsonPrecisionLower95": 0.9182
|
|
},
|
|
"threshold": 0.9
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.8447,
|
|
"falseNegative": 44,
|
|
"falsePositive": 6,
|
|
"precision": 0.9577,
|
|
"predictedPositive": 142,
|
|
"recall": 0.7556,
|
|
"total": 2970,
|
|
"trueNegative": 2784,
|
|
"truePositive": 136,
|
|
"wilsonPrecisionLower95": 0.9109
|
|
},
|
|
"threshold": 0.955
|
|
},
|
|
"followUpReminder": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8822,
|
|
"falseNegative": 34,
|
|
"falsePositive": 5,
|
|
"precision": 0.9669,
|
|
"predictedPositive": 151,
|
|
"recall": 0.8111,
|
|
"total": 1485,
|
|
"trueNegative": 1300,
|
|
"truePositive": 146,
|
|
"wilsonPrecisionLower95": 0.9248
|
|
},
|
|
"threshold": 0.98
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.7641,
|
|
"falseNegative": 130,
|
|
"falsePositive": 12,
|
|
"precision": 0.9504,
|
|
"predictedPositive": 242,
|
|
"recall": 0.6389,
|
|
"total": 2970,
|
|
"trueNegative": 2598,
|
|
"truePositive": 230,
|
|
"wilsonPrecisionLower95": 0.9153
|
|
},
|
|
"threshold": 0.99
|
|
},
|
|
"invitation": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"question": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8103,
|
|
"falseNegative": 95,
|
|
"falsePositive": 8,
|
|
"precision": 0.9649,
|
|
"predictedPositive": 228,
|
|
"recall": 0.6984,
|
|
"total": 1485,
|
|
"trueNegative": 1162,
|
|
"truePositive": 220,
|
|
"wilsonPrecisionLower95": 0.9323
|
|
},
|
|
"threshold": 0.915
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"replyableMessage": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9575,
|
|
"falseNegative": 33,
|
|
"falsePositive": 48,
|
|
"precision": 0.95,
|
|
"predictedPositive": 960,
|
|
"recall": 0.9651,
|
|
"total": 1485,
|
|
"trueNegative": 492,
|
|
"truePositive": 912,
|
|
"wilsonPrecisionLower95": 0.9343
|
|
},
|
|
"threshold": 0.845
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8361,
|
|
"falseNegative": 241,
|
|
"falsePositive": 35,
|
|
"precision": 0.9526,
|
|
"predictedPositive": 739,
|
|
"recall": 0.745,
|
|
"total": 1485,
|
|
"trueNegative": 505,
|
|
"truePositive": 704,
|
|
"wilsonPrecisionLower95": 0.9348
|
|
},
|
|
"threshold": 0.98
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.901,
|
|
"falseNegative": 274,
|
|
"falsePositive": 81,
|
|
"precision": 0.9523,
|
|
"predictedPositive": 1697,
|
|
"recall": 0.855,
|
|
"total": 2970,
|
|
"trueNegative": 999,
|
|
"truePositive": 1616,
|
|
"wilsonPrecisionLower95": 0.9411
|
|
},
|
|
"threshold": 0.975
|
|
},
|
|
"scheduleNegotiation": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9655,
|
|
"falseNegative": 3,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 42,
|
|
"recall": 0.9333,
|
|
"total": 1485,
|
|
"trueNegative": 1440,
|
|
"truePositive": 42,
|
|
"wilsonPrecisionLower95": 0.9162
|
|
},
|
|
"threshold": 0.5
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 1.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 45,
|
|
"recall": 1.0,
|
|
"total": 1485,
|
|
"trueNegative": 1440,
|
|
"truePositive": 45,
|
|
"wilsonPrecisionLower95": 0.9213
|
|
},
|
|
"threshold": 0.5
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.9831,
|
|
"falseNegative": 3,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 87,
|
|
"recall": 0.9667,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 87,
|
|
"wilsonPrecisionLower95": 0.9577
|
|
},
|
|
"threshold": 0.5
|
|
},
|
|
"task": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"trainingRecords": 16841,
|
|
"validation": {
|
|
"intentsMeeting95Precision": 8,
|
|
"macroF1": 0.6817,
|
|
"macroPrecision": 0.9748,
|
|
"macroRecall": 0.5634,
|
|
"minimumPrecision": 0.931,
|
|
"perIntent": {
|
|
"blessing": {
|
|
"f1": 0.3486,
|
|
"falseNegative": 71,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 19,
|
|
"recall": 0.2111,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 19,
|
|
"wilsonPrecisionLower95": 0.8318
|
|
},
|
|
"complaint": {
|
|
"f1": 0.88,
|
|
"falseNegative": 74,
|
|
"falsePositive": 4,
|
|
"precision": 0.9862,
|
|
"predictedPositive": 290,
|
|
"recall": 0.7944,
|
|
"total": 2970,
|
|
"trueNegative": 2606,
|
|
"truePositive": 286,
|
|
"wilsonPrecisionLower95": 0.9651
|
|
},
|
|
"confirmationDecision": {
|
|
"f1": 0.7183,
|
|
"falseNegative": 78,
|
|
"falsePositive": 2,
|
|
"precision": 0.9808,
|
|
"predictedPositive": 104,
|
|
"recall": 0.5667,
|
|
"total": 2970,
|
|
"trueNegative": 2788,
|
|
"truePositive": 102,
|
|
"wilsonPrecisionLower95": 0.9326
|
|
},
|
|
"followUpReminder": {
|
|
"f1": 0.6345,
|
|
"falseNegative": 189,
|
|
"falsePositive": 8,
|
|
"precision": 0.9553,
|
|
"predictedPositive": 179,
|
|
"recall": 0.475,
|
|
"total": 2970,
|
|
"trueNegative": 2602,
|
|
"truePositive": 171,
|
|
"wilsonPrecisionLower95": 0.9143
|
|
},
|
|
"invitation": {
|
|
"f1": 0.4538,
|
|
"falseNegative": 126,
|
|
"falsePositive": 4,
|
|
"precision": 0.931,
|
|
"predictedPositive": 58,
|
|
"recall": 0.3,
|
|
"total": 2970,
|
|
"trueNegative": 2786,
|
|
"truePositive": 54,
|
|
"wilsonPrecisionLower95": 0.8357
|
|
},
|
|
"question": {
|
|
"f1": 0.7187,
|
|
"falseNegative": 271,
|
|
"falsePositive": 10,
|
|
"precision": 0.9729,
|
|
"predictedPositive": 369,
|
|
"recall": 0.5698,
|
|
"total": 2970,
|
|
"trueNegative": 2330,
|
|
"truePositive": 359,
|
|
"wilsonPrecisionLower95": 0.9508
|
|
},
|
|
"replyableMessage": {
|
|
"f1": 0.9005,
|
|
"falseNegative": 274,
|
|
"falsePositive": 83,
|
|
"precision": 0.9511,
|
|
"predictedPositive": 1699,
|
|
"recall": 0.855,
|
|
"total": 2970,
|
|
"trueNegative": 997,
|
|
"truePositive": 1616,
|
|
"wilsonPrecisionLower95": 0.9398
|
|
},
|
|
"scheduleNegotiation": {
|
|
"f1": 0.9831,
|
|
"falseNegative": 3,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 87,
|
|
"recall": 0.9667,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 87,
|
|
"wilsonPrecisionLower95": 0.9577
|
|
},
|
|
"task": {
|
|
"f1": 0.4979,
|
|
"falseNegative": 481,
|
|
"falsePositive": 1,
|
|
"precision": 0.9958,
|
|
"predictedPositive": 240,
|
|
"recall": 0.3319,
|
|
"total": 2970,
|
|
"trueNegative": 2249,
|
|
"truePositive": 239,
|
|
"wilsonPrecisionLower95": 0.9768
|
|
}
|
|
},
|
|
"recordCount": 2970
|
|
}
|
|
},
|
|
{
|
|
"configuration": {
|
|
"alpha": 1.2e-06,
|
|
"augmentation": "numeric",
|
|
"char_max": 8,
|
|
"char_min": 3,
|
|
"external_weight": 0.93,
|
|
"hard_example_weight": 2.7,
|
|
"l1_ratio": 0.25,
|
|
"max_char_features": 95000,
|
|
"max_word_features": 45000,
|
|
"min_df": 1,
|
|
"pseudo_weight": 0.43499999999999994,
|
|
"round": 19,
|
|
"word_max": 3
|
|
},
|
|
"featureCount": 140000,
|
|
"pseudoLabelCounts": {
|
|
"blessing": {
|
|
"negative": 250,
|
|
"positive": 1
|
|
},
|
|
"complaint": {
|
|
"negative": 250,
|
|
"positive": 2
|
|
},
|
|
"confirmationDecision": {
|
|
"negative": 250,
|
|
"positive": 36
|
|
},
|
|
"followUpReminder": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"invitation": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"question": {
|
|
"negative": 250,
|
|
"positive": 4
|
|
},
|
|
"replyableMessage": {
|
|
"negative": 99,
|
|
"positive": 32
|
|
},
|
|
"scheduleNegotiation": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"task": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
}
|
|
},
|
|
"round": 19,
|
|
"selectionObjective": 0.7336,
|
|
"syntheticRegression": {
|
|
"intentsMeeting95Precision": 6,
|
|
"macroF1": 0.6005,
|
|
"macroPrecision": 0.9664,
|
|
"macroRecall": 0.4678,
|
|
"minimumPrecision": 0.887,
|
|
"perIntent": {
|
|
"blessing": {
|
|
"f1": 0.2,
|
|
"falseNegative": 80,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 10,
|
|
"recall": 0.1111,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 10,
|
|
"wilsonPrecisionLower95": 0.7225
|
|
},
|
|
"complaint": {
|
|
"f1": 0.9205,
|
|
"falseNegative": 53,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 307,
|
|
"recall": 0.8528,
|
|
"total": 2970,
|
|
"trueNegative": 2610,
|
|
"truePositive": 307,
|
|
"wilsonPrecisionLower95": 0.9876
|
|
},
|
|
"confirmationDecision": {
|
|
"f1": 0.6915,
|
|
"falseNegative": 78,
|
|
"falsePositive": 13,
|
|
"precision": 0.887,
|
|
"predictedPositive": 115,
|
|
"recall": 0.5667,
|
|
"total": 2970,
|
|
"trueNegative": 2777,
|
|
"truePositive": 102,
|
|
"wilsonPrecisionLower95": 0.8162
|
|
},
|
|
"followUpReminder": {
|
|
"f1": 0.4778,
|
|
"falseNegative": 247,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 113,
|
|
"recall": 0.3139,
|
|
"total": 2970,
|
|
"trueNegative": 2610,
|
|
"truePositive": 113,
|
|
"wilsonPrecisionLower95": 0.9671
|
|
},
|
|
"invitation": {
|
|
"f1": 0.7092,
|
|
"falseNegative": 80,
|
|
"falsePositive": 2,
|
|
"precision": 0.9804,
|
|
"predictedPositive": 102,
|
|
"recall": 0.5556,
|
|
"total": 2970,
|
|
"trueNegative": 2788,
|
|
"truePositive": 100,
|
|
"wilsonPrecisionLower95": 0.9313
|
|
},
|
|
"question": {
|
|
"f1": 0.4703,
|
|
"falseNegative": 436,
|
|
"falsePositive": 1,
|
|
"precision": 0.9949,
|
|
"predictedPositive": 195,
|
|
"recall": 0.3079,
|
|
"total": 2970,
|
|
"trueNegative": 2339,
|
|
"truePositive": 194,
|
|
"wilsonPrecisionLower95": 0.9715
|
|
},
|
|
"replyableMessage": {
|
|
"f1": 0.6399,
|
|
"falseNegative": 957,
|
|
"falsePositive": 93,
|
|
"precision": 0.9094,
|
|
"predictedPositive": 1026,
|
|
"recall": 0.4937,
|
|
"total": 2970,
|
|
"trueNegative": 987,
|
|
"truePositive": 933,
|
|
"wilsonPrecisionLower95": 0.8902
|
|
},
|
|
"scheduleNegotiation": {
|
|
"f1": 0.8079,
|
|
"falseNegative": 29,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 61,
|
|
"recall": 0.6778,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 61,
|
|
"wilsonPrecisionLower95": 0.9408
|
|
},
|
|
"task": {
|
|
"f1": 0.4872,
|
|
"falseNegative": 482,
|
|
"falsePositive": 19,
|
|
"precision": 0.9261,
|
|
"predictedPositive": 257,
|
|
"recall": 0.3306,
|
|
"total": 2970,
|
|
"trueNegative": 2231,
|
|
"truePositive": 238,
|
|
"wilsonPrecisionLower95": 0.8874
|
|
}
|
|
},
|
|
"recordCount": 2970
|
|
},
|
|
"thresholds": {
|
|
"blessing": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"complaint": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"confirmationDecision": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"followUpReminder": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"invitation": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"question": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"replyableMessage": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9309,
|
|
"falseNegative": 83,
|
|
"falsePositive": 45,
|
|
"precision": 0.9504,
|
|
"predictedPositive": 907,
|
|
"recall": 0.9122,
|
|
"total": 1485,
|
|
"trueNegative": 495,
|
|
"truePositive": 862,
|
|
"wilsonPrecisionLower95": 0.9343
|
|
},
|
|
"threshold": 0.98
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"scheduleNegotiation": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 1.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 45,
|
|
"recall": 1.0,
|
|
"total": 1485,
|
|
"trueNegative": 1440,
|
|
"truePositive": 45,
|
|
"wilsonPrecisionLower95": 0.9213
|
|
},
|
|
"threshold": 0.5
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.9195,
|
|
"falseNegative": 5,
|
|
"falsePositive": 2,
|
|
"precision": 0.9524,
|
|
"predictedPositive": 42,
|
|
"recall": 0.8889,
|
|
"total": 1485,
|
|
"trueNegative": 1438,
|
|
"truePositive": 40,
|
|
"wilsonPrecisionLower95": 0.8421
|
|
},
|
|
"threshold": 0.945
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.9667,
|
|
"falseNegative": 3,
|
|
"falsePositive": 3,
|
|
"precision": 0.9667,
|
|
"predictedPositive": 90,
|
|
"recall": 0.9667,
|
|
"total": 2970,
|
|
"trueNegative": 2877,
|
|
"truePositive": 87,
|
|
"wilsonPrecisionLower95": 0.9065
|
|
},
|
|
"threshold": 0.83
|
|
},
|
|
"task": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"trainingRecords": 16801,
|
|
"validation": {
|
|
"intentsMeeting95Precision": 6,
|
|
"macroF1": 0.6316,
|
|
"macroPrecision": 0.9616,
|
|
"macroRecall": 0.4969,
|
|
"minimumPrecision": 0.9242,
|
|
"perIntent": {
|
|
"blessing": {
|
|
"f1": 0.4483,
|
|
"falseNegative": 64,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 26,
|
|
"recall": 0.2889,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 26,
|
|
"wilsonPrecisionLower95": 0.8713
|
|
},
|
|
"complaint": {
|
|
"f1": 0.8953,
|
|
"falseNegative": 65,
|
|
"falsePositive": 4,
|
|
"precision": 0.9866,
|
|
"predictedPositive": 299,
|
|
"recall": 0.8194,
|
|
"total": 2970,
|
|
"trueNegative": 2606,
|
|
"truePositive": 295,
|
|
"wilsonPrecisionLower95": 0.9661
|
|
},
|
|
"confirmationDecision": {
|
|
"f1": 0.5891,
|
|
"falseNegative": 104,
|
|
"falsePositive": 2,
|
|
"precision": 0.9744,
|
|
"predictedPositive": 78,
|
|
"recall": 0.4222,
|
|
"total": 2970,
|
|
"trueNegative": 2788,
|
|
"truePositive": 76,
|
|
"wilsonPrecisionLower95": 0.9112
|
|
},
|
|
"followUpReminder": {
|
|
"f1": 0.528,
|
|
"falseNegative": 228,
|
|
"falsePositive": 8,
|
|
"precision": 0.9429,
|
|
"predictedPositive": 140,
|
|
"recall": 0.3667,
|
|
"total": 2970,
|
|
"trueNegative": 2602,
|
|
"truePositive": 132,
|
|
"wilsonPrecisionLower95": 0.8913
|
|
},
|
|
"invitation": {
|
|
"f1": 0.6015,
|
|
"falseNegative": 100,
|
|
"falsePositive": 6,
|
|
"precision": 0.9302,
|
|
"predictedPositive": 86,
|
|
"recall": 0.4444,
|
|
"total": 2970,
|
|
"trueNegative": 2784,
|
|
"truePositive": 80,
|
|
"wilsonPrecisionLower95": 0.856
|
|
},
|
|
"question": {
|
|
"f1": 0.442,
|
|
"falseNegative": 447,
|
|
"falsePositive": 15,
|
|
"precision": 0.9242,
|
|
"predictedPositive": 198,
|
|
"recall": 0.2905,
|
|
"total": 2970,
|
|
"trueNegative": 2325,
|
|
"truePositive": 183,
|
|
"wilsonPrecisionLower95": 0.8788
|
|
},
|
|
"replyableMessage": {
|
|
"f1": 0.6347,
|
|
"falseNegative": 990,
|
|
"falsePositive": 46,
|
|
"precision": 0.9514,
|
|
"predictedPositive": 946,
|
|
"recall": 0.4762,
|
|
"total": 2970,
|
|
"trueNegative": 1034,
|
|
"truePositive": 900,
|
|
"wilsonPrecisionLower95": 0.9357
|
|
},
|
|
"scheduleNegotiation": {
|
|
"f1": 0.9605,
|
|
"falseNegative": 5,
|
|
"falsePositive": 2,
|
|
"precision": 0.977,
|
|
"predictedPositive": 87,
|
|
"recall": 0.9444,
|
|
"total": 2970,
|
|
"trueNegative": 2878,
|
|
"truePositive": 85,
|
|
"wilsonPrecisionLower95": 0.92
|
|
},
|
|
"task": {
|
|
"f1": 0.5853,
|
|
"falseNegative": 418,
|
|
"falsePositive": 10,
|
|
"precision": 0.9679,
|
|
"predictedPositive": 312,
|
|
"recall": 0.4194,
|
|
"total": 2970,
|
|
"trueNegative": 2240,
|
|
"truePositive": 302,
|
|
"wilsonPrecisionLower95": 0.942
|
|
}
|
|
},
|
|
"recordCount": 2970
|
|
}
|
|
},
|
|
{
|
|
"configuration": {
|
|
"alpha": 1e-06,
|
|
"augmentation": "numeric",
|
|
"char_max": 8,
|
|
"char_min": 2,
|
|
"external_weight": 0.9500000000000001,
|
|
"hard_example_weight": 3.0,
|
|
"l1_ratio": 0.25,
|
|
"max_char_features": 100000,
|
|
"max_word_features": 50000,
|
|
"min_df": 1,
|
|
"pseudo_weight": 0.44999999999999996,
|
|
"round": 20,
|
|
"word_max": 3
|
|
},
|
|
"featureCount": 150000,
|
|
"pseudoLabelCounts": {
|
|
"blessing": {
|
|
"negative": 250,
|
|
"positive": 1
|
|
},
|
|
"complaint": {
|
|
"negative": 250,
|
|
"positive": 2
|
|
},
|
|
"confirmationDecision": {
|
|
"negative": 250,
|
|
"positive": 25
|
|
},
|
|
"followUpReminder": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
},
|
|
"invitation": {
|
|
"negative": 250,
|
|
"positive": 1
|
|
},
|
|
"question": {
|
|
"negative": 250,
|
|
"positive": 2
|
|
},
|
|
"replyableMessage": {
|
|
"negative": 104,
|
|
"positive": 32
|
|
},
|
|
"scheduleNegotiation": {
|
|
"negative": 250,
|
|
"positive": 1
|
|
},
|
|
"task": {
|
|
"negative": 250,
|
|
"positive": 0
|
|
}
|
|
},
|
|
"round": 20,
|
|
"selectionObjective": 0.7724,
|
|
"syntheticRegression": {
|
|
"intentsMeeting95Precision": 7,
|
|
"macroF1": 0.6637,
|
|
"macroPrecision": 0.9685,
|
|
"macroRecall": 0.5421,
|
|
"minimumPrecision": 0.9094,
|
|
"perIntent": {
|
|
"blessing": {
|
|
"f1": 0.2353,
|
|
"falseNegative": 78,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 12,
|
|
"recall": 0.1333,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 12,
|
|
"wilsonPrecisionLower95": 0.7575
|
|
},
|
|
"complaint": {
|
|
"f1": 0.8274,
|
|
"falseNegative": 106,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 254,
|
|
"recall": 0.7056,
|
|
"total": 2970,
|
|
"trueNegative": 2610,
|
|
"truePositive": 254,
|
|
"wilsonPrecisionLower95": 0.9851
|
|
},
|
|
"confirmationDecision": {
|
|
"f1": 0.8261,
|
|
"falseNegative": 47,
|
|
"falsePositive": 9,
|
|
"precision": 0.9366,
|
|
"predictedPositive": 142,
|
|
"recall": 0.7389,
|
|
"total": 2970,
|
|
"trueNegative": 2781,
|
|
"truePositive": 133,
|
|
"wilsonPrecisionLower95": 0.8839
|
|
},
|
|
"followUpReminder": {
|
|
"f1": 0.5155,
|
|
"falseNegative": 235,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 125,
|
|
"recall": 0.3472,
|
|
"total": 2970,
|
|
"trueNegative": 2610,
|
|
"truePositive": 125,
|
|
"wilsonPrecisionLower95": 0.9702
|
|
},
|
|
"invitation": {
|
|
"f1": 0.7241,
|
|
"falseNegative": 75,
|
|
"falsePositive": 5,
|
|
"precision": 0.9545,
|
|
"predictedPositive": 110,
|
|
"recall": 0.5833,
|
|
"total": 2970,
|
|
"trueNegative": 2785,
|
|
"truePositive": 105,
|
|
"wilsonPrecisionLower95": 0.898
|
|
},
|
|
"question": {
|
|
"f1": 0.5534,
|
|
"falseNegative": 389,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 241,
|
|
"recall": 0.3825,
|
|
"total": 2970,
|
|
"trueNegative": 2340,
|
|
"truePositive": 241,
|
|
"wilsonPrecisionLower95": 0.9843
|
|
},
|
|
"replyableMessage": {
|
|
"f1": 0.7688,
|
|
"falseNegative": 671,
|
|
"falsePositive": 62,
|
|
"precision": 0.9516,
|
|
"predictedPositive": 1281,
|
|
"recall": 0.645,
|
|
"total": 2970,
|
|
"trueNegative": 1018,
|
|
"truePositive": 1219,
|
|
"wilsonPrecisionLower95": 0.9384
|
|
},
|
|
"scheduleNegotiation": {
|
|
"f1": 0.9371,
|
|
"falseNegative": 8,
|
|
"falsePositive": 3,
|
|
"precision": 0.9647,
|
|
"predictedPositive": 85,
|
|
"recall": 0.9111,
|
|
"total": 2970,
|
|
"trueNegative": 2877,
|
|
"truePositive": 82,
|
|
"wilsonPrecisionLower95": 0.9013
|
|
},
|
|
"task": {
|
|
"f1": 0.5857,
|
|
"falseNegative": 409,
|
|
"falsePositive": 31,
|
|
"precision": 0.9094,
|
|
"predictedPositive": 342,
|
|
"recall": 0.4319,
|
|
"total": 2970,
|
|
"trueNegative": 2219,
|
|
"truePositive": 311,
|
|
"wilsonPrecisionLower95": 0.8742
|
|
}
|
|
},
|
|
"recordCount": 2970
|
|
},
|
|
"thresholds": {
|
|
"blessing": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"complaint": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"confirmationDecision": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.5954,
|
|
"falseNegative": 51,
|
|
"falsePositive": 2,
|
|
"precision": 0.9512,
|
|
"predictedPositive": 41,
|
|
"recall": 0.4333,
|
|
"total": 1485,
|
|
"trueNegative": 1393,
|
|
"truePositive": 39,
|
|
"wilsonPrecisionLower95": 0.8386
|
|
},
|
|
"threshold": 0.995
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"followUpReminder": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"invitation": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"question": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"replyableMessage": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.8117,
|
|
"falseNegative": 279,
|
|
"falsePositive": 30,
|
|
"precision": 0.9569,
|
|
"predictedPositive": 696,
|
|
"recall": 0.7048,
|
|
"total": 1485,
|
|
"trueNegative": 510,
|
|
"truePositive": 666,
|
|
"wilsonPrecisionLower95": 0.9391
|
|
},
|
|
"threshold": 0.995
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.8736,
|
|
"falseNegative": 363,
|
|
"falsePositive": 79,
|
|
"precision": 0.9508,
|
|
"predictedPositive": 1606,
|
|
"recall": 0.8079,
|
|
"total": 2970,
|
|
"trueNegative": 1001,
|
|
"truePositive": 1527,
|
|
"wilsonPrecisionLower95": 0.9391
|
|
},
|
|
"threshold": 0.995
|
|
},
|
|
"scheduleNegotiation": {
|
|
"abstained": false,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 1.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 45,
|
|
"recall": 1.0,
|
|
"total": 1485,
|
|
"trueNegative": 1440,
|
|
"truePositive": 45,
|
|
"wilsonPrecisionLower95": 0.9213
|
|
},
|
|
"threshold": 0.5
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": false,
|
|
"metrics": {
|
|
"f1": 0.967,
|
|
"falseNegative": 1,
|
|
"falsePositive": 2,
|
|
"precision": 0.9565,
|
|
"predictedPositive": 46,
|
|
"recall": 0.9778,
|
|
"total": 1485,
|
|
"trueNegative": 1438,
|
|
"truePositive": 44,
|
|
"wilsonPrecisionLower95": 0.8547
|
|
},
|
|
"threshold": 0.83
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.9834,
|
|
"falseNegative": 1,
|
|
"falsePositive": 2,
|
|
"precision": 0.978,
|
|
"predictedPositive": 91,
|
|
"recall": 0.9889,
|
|
"total": 2970,
|
|
"trueNegative": 2878,
|
|
"truePositive": 89,
|
|
"wilsonPrecisionLower95": 0.9234
|
|
},
|
|
"threshold": 0.83
|
|
},
|
|
"task": {
|
|
"abstained": true,
|
|
"byLanguage": {
|
|
"en": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
},
|
|
"zh-Hans": {
|
|
"abstained": true,
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"metrics": {
|
|
"f1": 0.0,
|
|
"falseNegative": 0,
|
|
"falsePositive": 0,
|
|
"precision": 0.0,
|
|
"predictedPositive": 0,
|
|
"recall": 0.0,
|
|
"total": 0,
|
|
"trueNegative": 0,
|
|
"truePositive": 0,
|
|
"wilsonPrecisionLower95": 0
|
|
},
|
|
"threshold": 1.0
|
|
}
|
|
},
|
|
"trainingRecords": 16755,
|
|
"validation": {
|
|
"intentsMeeting95Precision": 8,
|
|
"macroF1": 0.6867,
|
|
"macroPrecision": 0.9687,
|
|
"macroRecall": 0.5569,
|
|
"minimumPrecision": 0.9038,
|
|
"perIntent": {
|
|
"blessing": {
|
|
"f1": 0.3929,
|
|
"falseNegative": 68,
|
|
"falsePositive": 0,
|
|
"precision": 1.0,
|
|
"predictedPositive": 22,
|
|
"recall": 0.2444,
|
|
"total": 2970,
|
|
"trueNegative": 2880,
|
|
"truePositive": 22,
|
|
"wilsonPrecisionLower95": 0.8513
|
|
},
|
|
"complaint": {
|
|
"f1": 0.8066,
|
|
"falseNegative": 114,
|
|
"falsePositive": 4,
|
|
"precision": 0.984,
|
|
"predictedPositive": 250,
|
|
"recall": 0.6833,
|
|
"total": 2970,
|
|
"trueNegative": 2606,
|
|
"truePositive": 246,
|
|
"wilsonPrecisionLower95": 0.9596
|
|
},
|
|
"confirmationDecision": {
|
|
"f1": 0.6618,
|
|
"falseNegative": 90,
|
|
"falsePositive": 2,
|
|
"precision": 0.9783,
|
|
"predictedPositive": 92,
|
|
"recall": 0.5,
|
|
"total": 2970,
|
|
"trueNegative": 2788,
|
|
"truePositive": 90,
|
|
"wilsonPrecisionLower95": 0.9242
|
|
},
|
|
"followUpReminder": {
|
|
"f1": 0.5977,
|
|
"falseNegative": 204,
|
|
"falsePositive": 6,
|
|
"precision": 0.963,
|
|
"predictedPositive": 162,
|
|
"recall": 0.4333,
|
|
"total": 2970,
|
|
"trueNegative": 2604,
|
|
"truePositive": 156,
|
|
"wilsonPrecisionLower95": 0.9216
|
|
},
|
|
"invitation": {
|
|
"f1": 0.662,
|
|
"falseNegative": 86,
|
|
"falsePositive": 10,
|
|
"precision": 0.9038,
|
|
"predictedPositive": 104,
|
|
"recall": 0.5222,
|
|
"total": 2970,
|
|
"trueNegative": 2780,
|
|
"truePositive": 94,
|
|
"wilsonPrecisionLower95": 0.832
|
|
},
|
|
"question": {
|
|
"f1": 0.5643,
|
|
"falseNegative": 380,
|
|
"falsePositive": 6,
|
|
"precision": 0.9766,
|
|
"predictedPositive": 256,
|
|
"recall": 0.3968,
|
|
"total": 2970,
|
|
"trueNegative": 2334,
|
|
"truePositive": 250,
|
|
"wilsonPrecisionLower95": 0.9498
|
|
},
|
|
"replyableMessage": {
|
|
"f1": 0.7934,
|
|
"falseNegative": 621,
|
|
"falsePositive": 40,
|
|
"precision": 0.9694,
|
|
"predictedPositive": 1309,
|
|
"recall": 0.6714,
|
|
"total": 2970,
|
|
"trueNegative": 1040,
|
|
"truePositive": 1269,
|
|
"wilsonPrecisionLower95": 0.9587
|
|
},
|
|
"scheduleNegotiation": {
|
|
"f1": 0.9834,
|
|
"falseNegative": 1,
|
|
"falsePositive": 2,
|
|
"precision": 0.978,
|
|
"predictedPositive": 91,
|
|
"recall": 0.9889,
|
|
"total": 2970,
|
|
"trueNegative": 2878,
|
|
"truePositive": 89,
|
|
"wilsonPrecisionLower95": 0.9234
|
|
},
|
|
"task": {
|
|
"f1": 0.7184,
|
|
"falseNegative": 308,
|
|
"falsePositive": 15,
|
|
"precision": 0.9649,
|
|
"predictedPositive": 427,
|
|
"recall": 0.5722,
|
|
"total": 2970,
|
|
"trueNegative": 2235,
|
|
"truePositive": 412,
|
|
"wilsonPrecisionLower95": 0.9429
|
|
}
|
|
},
|
|
"recordCount": 2970
|
|
}
|
|
}
|
|
]
|