Rocky
1398 lines
38 KiB
JSON
1398 lines
38 KiB
JSON
{
|
|
"binaryMacro" : {
|
|
"accuracy" : 0.9634,
|
|
"f1" : 0.824,
|
|
"precision" : 0.9518,
|
|
"recall" : 0.7564
|
|
},
|
|
"binaryMacroByLanguage" : {
|
|
"en" : {
|
|
"accuracy" : 0.967,
|
|
"f1" : 0.8428,
|
|
"precision" : 0.9683,
|
|
"recall" : 0.7824
|
|
},
|
|
"zh-Hans" : {
|
|
"accuracy" : 0.9599,
|
|
"f1" : 0.8026,
|
|
"precision" : 0.9435,
|
|
"recall" : 0.7305
|
|
}
|
|
},
|
|
"classifiers" : [
|
|
{
|
|
"falseNegativeExamples" : [
|
|
{
|
|
"confidence" : 0.7785,
|
|
"family" : "follow_up_reminder_golden",
|
|
"id" : "golden-en-0189",
|
|
"language" : "en",
|
|
"text" : "Follow up with the vendor about the delivery date next Monday."
|
|
},
|
|
{
|
|
"confidence" : 0.7766,
|
|
"family" : "task_statement",
|
|
"id" : "golden-zh-Hans-0002",
|
|
"language" : "zh-Hans",
|
|
"text" : "小王负责整理会议纪要，今天发到项目群。"
|
|
},
|
|
{
|
|
"confidence" : 0.6713,
|
|
"family" : "follow_up_reminder_golden",
|
|
"id" : "golden-zh-Hans-0175",
|
|
"language" : "zh-Hans",
|
|
"text" : "别忘了周五检查退款有没有到账。"
|
|
},
|
|
{
|
|
"confidence" : 0.6483,
|
|
"family" : "follow_up_reminder_golden",
|
|
"id" : "golden-en-0188",
|
|
"language" : "en",
|
|
"text" : "When approval arrives, contact the applicant and explain the reason."
|
|
},
|
|
{
|
|
"confidence" : 0.4819,
|
|
"family" : "task_question",
|
|
"id" : "golden-zh-Hans-0007",
|
|
"language" : "zh-Hans",
|
|
"text" : "这些材料可以在月底前准备好吗？"
|
|
}
|
|
],
|
|
"falsePositiveExamples" : [
|
|
{
|
|
"confidence" : 0.8675,
|
|
"family" : "confirmation_decision_boundary",
|
|
"id" : "golden-en-0172",
|
|
"language" : "en",
|
|
"text" : "OK, send me the contract for review first."
|
|
}
|
|
],
|
|
"id" : "task",
|
|
"metrics" : {
|
|
"accuracy" : 0.9575,
|
|
"f1" : 0.8163,
|
|
"falseNegative" : 8,
|
|
"falsePositive" : 1,
|
|
"precision" : 0.9524,
|
|
"recall" : 0.7143,
|
|
"total" : 212,
|
|
"trueNegative" : 183,
|
|
"truePositive" : 20
|
|
},
|
|
"metricsByLanguage" : {
|
|
"en" : {
|
|
"accuracy" : 0.9717,
|
|
"f1" : 0.8889,
|
|
"falseNegative" : 2,
|
|
"falsePositive" : 1,
|
|
"precision" : 0.9231,
|
|
"recall" : 0.8571,
|
|
"total" : 106,
|
|
"trueNegative" : 91,
|
|
"truePositive" : 12
|
|
},
|
|
"zh-Hans" : {
|
|
"accuracy" : 0.9434,
|
|
"f1" : 0.7273,
|
|
"falseNegative" : 6,
|
|
"falsePositive" : 0,
|
|
"precision" : 1,
|
|
"recall" : 0.5714,
|
|
"total" : 106,
|
|
"trueNegative" : 92,
|
|
"truePositive" : 8
|
|
}
|
|
},
|
|
"thresholdAt90Precision" : {
|
|
"metrics" : {
|
|
"accuracy" : 0.9575,
|
|
"f1" : 0.8163,
|
|
"falseNegative" : 8,
|
|
"falsePositive" : 1,
|
|
"precision" : 0.9524,
|
|
"recall" : 0.7143,
|
|
"total" : 212,
|
|
"trueNegative" : 183,
|
|
"truePositive" : 20
|
|
},
|
|
"threshold" : 0.81
|
|
},
|
|
"thresholdAt95Precision" : {
|
|
"metrics" : {
|
|
"accuracy" : 0.9575,
|
|
"f1" : 0.8163,
|
|
"falseNegative" : 8,
|
|
"falsePositive" : 1,
|
|
"precision" : 0.9524,
|
|
"recall" : 0.7143,
|
|
"total" : 212,
|
|
"trueNegative" : 183,
|
|
"truePositive" : 20
|
|
},
|
|
"threshold" : 0.81
|
|
},
|
|
"thresholdsAt90PrecisionByLanguage" : {
|
|
"en" : {
|
|
"metrics" : {
|
|
"accuracy" : 0.9717,
|
|
"f1" : 0.8889,
|
|
"falseNegative" : 2,
|
|
"falsePositive" : 1,
|
|
"precision" : 0.9231,
|
|
"recall" : 0.8571,
|
|
"total" : 106,
|
|
"trueNegative" : 91,
|
|
"truePositive" : 12
|
|
},
|
|
"threshold" : 0.81
|
|
},
|
|
"zh-Hans" : {
|
|
"metrics" : {
|
|
"accuracy" : 0.9623,
|
|
"f1" : 0.8462,
|
|
"falseNegative" : 3,
|
|
"falsePositive" : 1,
|
|
"precision" : 0.9167,
|
|
"recall" : 0.7857,
|
|
"total" : 106,
|
|
"trueNegative" : 91,
|
|
"truePositive" : 11
|
|
},
|
|
"threshold" : 0.48
|
|
}
|
|
},
|
|
"thresholdsAt95PrecisionByLanguage" : {
|
|
"en" : {
|
|
"metrics" : {
|
|
"accuracy" : 0.9623,
|
|
"f1" : 0.8333,
|
|
"falseNegative" : 4,
|
|
"falsePositive" : 0,
|
|
"precision" : 1,
|
|
"recall" : 0.7143,
|
|
"total" : 106,
|
|
"trueNegative" : 92,
|
|
"truePositive" : 10
|
|
},
|
|
"threshold" : 0.87
|
|
},
|
|
"zh-Hans" : {
|
|
"metrics" : {
|
|
"accuracy" : 0.9623,
|
|
"f1" : 0.8333,
|
|
"falseNegative" : 4,
|
|
"falsePositive" : 0,
|
|
"precision" : 1,
|
|
"recall" : 0.7143,
|
|
"total" : 106,
|
|
"trueNegative" : 92,
|
|
"truePositive" : 10
|
|
},
|
|
"threshold" : 0.61
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"falseNegativeExamples" : [
|
|
{
|
|
"confidence" : 0.5821,
|
|
"family" : "complaint_request",
|
|
"id" : "golden-en-0065",
|
|
"language" : "en",
|
|
"text" : "Support has not replied for three days. When will this be handled?"
|
|
},
|
|
{
|
|
"confidence" : 0.5408,
|
|
"family" : "schedule_negotiation_golden",
|
|
"id" : "golden-en-0106",
|
|
"language" : "en",
|
|
"text" : "The client is unavailable, so how about next Monday afternoon instead?"
|
|
}
|
|
],
|
|
"falsePositiveExamples" : [
|
|
{
|
|
"confidence" : 0.9558,
|
|
"family" : "schedule_negotiation_golden",
|
|
"id" : "golden-en-0108",
|
|
"language" : "en",
|
|
"text" : "If Friday does not work, I can do Saturday morning."
|
|
},
|
|
{
|
|
"confidence" : 0.9299,
|
|
"family" : "schedule_negotiation_boundary",
|
|
"id" : "golden-zh-Hans-0117",
|
|
"language" : "zh-Hans",
|
|
"text" : "邀请你周五下午参加评审，时间不变。"
|
|
},
|
|
{
|
|
"confidence" : 0.7663,
|
|
"family" : "neutral_fact",
|
|
"id" : "golden-en-0079",
|
|
"language" : "en",
|
|
"text" : "The store is open from nine in the morning until ten at night."
|
|
},
|
|
{
|
|
"confidence" : 0.6369,
|
|
"family" : "neutral_fact",
|
|
"id" : "golden-zh-Hans-0033",
|
|
"language" : "zh-Hans",
|
|
"text" : "门店营业时间是上午九点到晚上十点。"
|
|
}
|
|
],
|
|
"id" : "question",
|
|
"metrics" : {
|
|
"accuracy" : 0.9717,
|
|
"f1" : 0.9333,
|
|
"falseNegative" : 2,
|
|
"falsePositive" : 4,
|
|
"precision" : 0.913,
|
|
"recall" : 0.9545,
|
|
"total" : 212,
|
|
"trueNegative" : 164,
|
|
"truePositive" : 42
|
|
},
|
|
"metricsByLanguage" : {
|
|
"en" : {
|
|
"accuracy" : 0.9623,
|
|
"f1" : 0.9091,
|
|
"falseNegative" : 2,
|
|
"falsePositive" : 2,
|
|
"precision" : 0.9091,
|
|
"recall" : 0.9091,
|
|
"total" : 106,
|
|
"trueNegative" : 82,
|
|
"truePositive" : 20
|
|
},
|
|
"zh-Hans" : {
|
|
"accuracy" : 0.9811,
|
|
"f1" : 0.9565,
|
|
"falseNegative" : 0,
|
|
"falsePositive" : 2,
|
|
"precision" : 0.9167,
|
|
"recall" : 1,
|
|
"total" : 106,
|
|
"trueNegative" : 82,
|
|
"truePositive" : 22
|
|
}
|
|
},
|
|
"thresholdAt90Precision" : {
|
|
"metrics" : {
|
|
"accuracy" : 0.9764,
|
|
"f1" : 0.9451,
|
|
"falseNegative" : 1,
|
|
"falsePositive" : 4,
|
|
"precision" : 0.9149,
|
|
"recall" : 0.9773,
|
|
"total" : 212,
|
|
"trueNegative" : 164,
|
|
"truePositive" : 43
|
|
},
|
|
"threshold" : 0.55
|
|
},
|
|
"thresholdAt95Precision" : {
|
|
"metrics" : {
|
|
"accuracy" : 0.9764,
|
|
"f1" : 0.9425,
|
|
"falseNegative" : 3,
|
|
"falsePositive" : 2,
|
|
"precision" : 0.9535,
|
|
"recall" : 0.9318,
|
|
"total" : 212,
|
|
"trueNegative" : 166,
|
|
"truePositive" : 41
|
|
},
|
|
"threshold" : 0.77
|
|
},
|
|
"thresholdsAt90PrecisionByLanguage" : {
|
|
"en" : {
|
|
"metrics" : {
|
|
"accuracy" : 0.9811,
|
|
"f1" : 0.9565,
|
|
"falseNegative" : 0,
|
|
"falsePositive" : 2,
|
|
"precision" : 0.9167,
|
|
"recall" : 1,
|
|
"total" : 106,
|
|
"trueNegative" : 82,
|
|
"truePositive" : 22
|
|
},
|
|
"threshold" : 0.51
|
|
},
|
|
"zh-Hans" : {
|
|
"metrics" : {
|
|
"accuracy" : 0.9906,
|
|
"f1" : 0.9778,
|
|
"falseNegative" : 0,
|
|
"falsePositive" : 1,
|
|
"precision" : 0.9565,
|
|
"recall" : 1,
|
|
"total" : 106,
|
|
"trueNegative" : 83,
|
|
"truePositive" : 22
|
|
},
|
|
"threshold" : 0.64
|
|
}
|
|
},
|
|
"thresholdsAt95PrecisionByLanguage" : {
|
|
"en" : {
|
|
"metrics" : {
|
|
"accuracy" : 0.9623,
|
|
"f1" : 0.9048,
|
|
"falseNegative" : 3,
|
|
"falsePositive" : 1,
|
|
"precision" : 0.95,
|
|
"recall" : 0.8636,
|
|
"total" : 106,
|
|
"trueNegative" : 83,
|
|
"truePositive" : 19
|
|
},
|
|
"threshold" : 0.77
|
|
},
|
|
"zh-Hans" : {
|
|
"metrics" : {
|
|
"accuracy" : 0.9906,
|
|
"f1" : 0.9778,
|
|
"falseNegative" : 0,
|
|
"falsePositive" : 1,
|
|
"precision" : 0.9565,
|
|
"recall" : 1,
|
|
"total" : 106,
|
|
"trueNegative" : 83,
|
|
"truePositive" : 22
|
|
},
|
|
"threshold" : 0.64
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"falseNegativeExamples" : [
|
|
{
|
|
"confidence" : 0.833,
|
|
"family" : "invitation_question",
|
|
"id" : "golden-en-0059",
|
|
"language" : "en",
|
|
"text" : "Would you like to get coffee after work?"
|
|
},
|
|
{
|
|
"confidence" : 0.7794,
|
|
"family" : "invitation_question",
|
|
"id" : "golden-zh-Hans-0013",
|
|
"language" : "zh-Hans",
|
|
"text" : "下班后要不要一起喝杯咖啡？"
|
|
},
|
|
{
|
|
"confidence" : 0.7173,
|
|
"family" : "schedule_negotiation_boundary",
|
|
"id" : "golden-zh-Hans-0113",
|
|
"language" : "zh-Hans",
|
|
"text" : "项目启动会定在周一上午九点，请准时参加。"
|
|
},
|
|
{
|
|
"confidence" : 0.6164,
|
|
"family" : "schedule_negotiation_boundary",
|
|
"id" : "golden-en-0123",
|
|
"language" : "en",
|
|
"text" : "The kickoff is fixed for Monday at nine; please attend on time."
|
|
},
|
|
{
|
|
"confidence" : 0.3649,
|
|
"family" : "schedule_negotiation_boundary",
|
|
"id" : "golden-en-0132",
|
|
"language" : "en",
|
|
"text" : "Want to see a movie this weekend? The tickets are for Saturday at eight."
|
|
}
|
|
],
|
|
"falsePositiveExamples" : [
|
|
{
|
|
"confidence" : 0.8284,
|
|
"family" : "schedule_negotiation_golden",
|
|
"id" : "golden-zh-Hans-0096",
|
|
"language" : "zh-Hans",
|
|
"text" : "客户临时有事，咱们另约下周一下午怎么样？"
|
|
}
|
|
],
|
|
"id" : "invitation",
|
|
"metrics" : {
|
|
"accuracy" : 0.9623,
|
|
"f1" : 0.6923,
|
|
"falseNegative" : 7,
|
|
"falsePositive" : 1,
|
|
"precision" : 0.9,
|
|
"recall" : 0.5625,
|
|
"total" : 212,
|
|
"trueNegative" : 195,
|
|
"truePositive" : 9
|
|
},
|
|
"metricsByLanguage" : {
|
|
"en" : {
|
|
"accuracy" : 0.9623,
|
|
"f1" : 0.6667,
|
|
"falseNegative" : 4,
|
|
"falsePositive" : 0,
|
|
"precision" : 1,
|
|
"recall" : 0.5,
|
|
"total" : 106,
|
|
"trueNegative" : 98,
|
|
"truePositive" : 4
|
|
},
|
|
"zh-Hans" : {
|
|
"accuracy" : 0.9623,
|
|
"f1" : 0.7143,
|
|
"falseNegative" : 3,
|
|
"falsePositive" : 1,
|
|
"precision" : 0.8333,
|
|
"recall" : 0.625,
|
|
"total" : 106,
|
|
"trueNegative" : 97,
|
|
"truePositive" : 5
|
|
}
|
|
},
|
|
"thresholdAt90Precision" : {
|
|
"metrics" : {
|
|
"accuracy" : 0.9764,
|
|
"f1" : 0.8276,
|
|
"falseNegative" : 4,
|
|
"falsePositive" : 1,
|
|
"precision" : 0.9231,
|
|
"recall" : 0.75,
|
|
"total" : 212,
|
|
"trueNegative" : 195,
|
|
"truePositive" : 12
|
|
},
|
|
"threshold" : 0.67
|
|
},
|
|
"thresholdAt95Precision" : {
|
|
"metrics" : {
|
|
"accuracy" : 0.9717,
|
|
"f1" : 0.7692,
|
|
"falseNegative" : 6,
|
|
"falsePositive" : 0,
|
|
"precision" : 1,
|
|
"recall" : 0.625,
|
|
"total" : 212,
|
|
"trueNegative" : 196,
|
|
"truePositive" : 10
|
|
},
|
|
"threshold" : 0.83
|
|
},
|
|
"thresholdsAt90PrecisionByLanguage" : {
|
|
"en" : {
|
|
"metrics" : {
|
|
"accuracy" : 0.9811,
|
|
"f1" : 0.8571,
|
|
"falseNegative" : 2,
|
|
"falsePositive" : 0,
|
|
"precision" : 1,
|
|
"recall" : 0.75,
|
|
"total" : 106,
|
|
"trueNegative" : 98,
|
|
"truePositive" : 6
|
|
},
|
|
"threshold" : 0.54
|
|
},
|
|
"zh-Hans" : {
|
|
"metrics" : {
|
|
"accuracy" : 0.9717,
|
|
"f1" : 0.7692,
|
|
"falseNegative" : 3,
|
|
"falsePositive" : 0,
|
|
"precision" : 1,
|
|
"recall" : 0.625,
|
|
"total" : 106,
|
|
"trueNegative" : 98,
|
|
"truePositive" : 5
|
|
},
|
|
"threshold" : 0.83
|
|
}
|
|
},
|
|
"thresholdsAt95PrecisionByLanguage" : {
|
|
"en" : {
|
|
"metrics" : {
|
|
"accuracy" : 0.9811,
|
|
"f1" : 0.8571,
|
|
"falseNegative" : 2,
|
|
"falsePositive" : 0,
|
|
"precision" : 1,
|
|
"recall" : 0.75,
|
|
"total" : 106,
|
|
"trueNegative" : 98,
|
|
"truePositive" : 6
|
|
},
|
|
"threshold" : 0.54
|
|
},
|
|
"zh-Hans" : {
|
|
"metrics" : {
|
|
"accuracy" : 0.9717,
|
|
"f1" : 0.7692,
|
|
"falseNegative" : 3,
|
|
"falsePositive" : 0,
|
|
"precision" : 1,
|
|
"recall" : 0.625,
|
|
"total" : 106,
|
|
"trueNegative" : 98,
|
|
"truePositive" : 5
|
|
},
|
|
"threshold" : 0.83
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"falseNegativeExamples" : [
|
|
{
|
|
"confidence" : 0.802,
|
|
"family" : "complaint_request",
|
|
"id" : "golden-zh-Hans-0017",
|
|
"language" : "zh-Hans",
|
|
"text" : "账号无故被锁了，能请你们今天恢复吗？"
|
|
},
|
|
{
|
|
"confidence" : 0.7533,
|
|
"family" : "complaint_statement",
|
|
"id" : "golden-en-0062",
|
|
"language" : "en",
|
|
"text" : "The package arrived damaged and the cup inside was broken."
|
|
},
|
|
{
|
|
"confidence" : 0.6877,
|
|
"family" : "complaint_statement",
|
|
"id" : "golden-en-0060",
|
|
"language" : "en",
|
|
"text" : "The app has crashed constantly since the update, and I lost important data."
|
|
},
|
|
{
|
|
"confidence" : 0.686,
|
|
"family" : "complaint_request",
|
|
"id" : "golden-zh-Hans-0019",
|
|
"language" : "zh-Hans",
|
|
"text" : "客服已经三天没有回复了，请问什么时候能处理？"
|
|
},
|
|
{
|
|
"confidence" : 0.6192,
|
|
"family" : "complaint_statement",
|
|
"id" : "golden-zh-Hans-0016",
|
|
"language" : "zh-Hans",
|
|
"text" : "收到的商品外包装破损，里面的杯子也碎了。"
|
|
}
|
|
],
|
|
"falsePositiveExamples" : [
|
|
|
|
],
|
|
"id" : "complaint",
|
|
"metrics" : {
|
|
"accuracy" : 0.9623,
|
|
"f1" : 0.5,
|
|
"falseNegative" : 8,
|
|
"falsePositive" : 0,
|
|
"precision" : 1,
|
|
"recall" : 0.3333,
|
|
"total" : 212,
|
|
"trueNegative" : 200,
|
|
"truePositive" : 4
|
|
},
|
|
"metricsByLanguage" : {
|
|
"en" : {
|
|
"accuracy" : 0.9623,
|
|
"f1" : 0.5,
|
|
"falseNegative" : 4,
|
|
"falsePositive" : 0,
|
|
"precision" : 1,
|
|
"recall" : 0.3333,
|
|
"total" : 106,
|
|
"trueNegative" : 100,
|
|
"truePositive" : 2
|
|
},
|
|
"zh-Hans" : {
|
|
"accuracy" : 0.9623,
|
|
"f1" : 0.5,
|
|
"falseNegative" : 4,
|
|
"falsePositive" : 0,
|
|
"precision" : 1,
|
|
"recall" : 0.3333,
|
|
"total" : 106,
|
|
"trueNegative" : 100,
|
|
"truePositive" : 2
|
|
}
|
|
},
|
|
"thresholdAt90Precision" : {
|
|
"metrics" : {
|
|
"accuracy" : 0.9717,
|
|
"f1" : 0.6667,
|
|
"falseNegative" : 6,
|
|
"falsePositive" : 0,
|
|
"precision" : 1,
|
|
"recall" : 0.5,
|
|
"total" : 212,
|
|
"trueNegative" : 200,
|
|
"truePositive" : 6
|
|
},
|
|
"threshold" : 0.73
|
|
},
|
|
"thresholdAt95Precision" : {
|
|
"metrics" : {
|
|
"accuracy" : 0.9717,
|
|
"f1" : 0.6667,
|
|
"falseNegative" : 6,
|
|
"falsePositive" : 0,
|
|
"precision" : 1,
|
|
"recall" : 0.5,
|
|
"total" : 212,
|
|
"trueNegative" : 200,
|
|
"truePositive" : 6
|
|
},
|
|
"threshold" : 0.73
|
|
},
|
|
"thresholdsAt90PrecisionByLanguage" : {
|
|
"en" : {
|
|
"metrics" : {
|
|
"accuracy" : 0.9717,
|
|
"f1" : 0.6667,
|
|
"falseNegative" : 3,
|
|
"falsePositive" : 0,
|
|
"precision" : 1,
|
|
"recall" : 0.5,
|
|
"total" : 106,
|
|
"trueNegative" : 100,
|
|
"truePositive" : 3
|
|
},
|
|
"threshold" : 0.71
|
|
},
|
|
"zh-Hans" : {
|
|
"metrics" : {
|
|
"accuracy" : 0.9717,
|
|
"f1" : 0.6667,
|
|
"falseNegative" : 3,
|
|
"falsePositive" : 0,
|
|
"precision" : 1,
|
|
"recall" : 0.5,
|
|
"total" : 106,
|
|
"trueNegative" : 100,
|
|
"truePositive" : 3
|
|
},
|
|
"threshold" : 0.73
|
|
}
|
|
},
|
|
"thresholdsAt95PrecisionByLanguage" : {
|
|
"en" : {
|
|
"metrics" : {
|
|
"accuracy" : 0.9717,
|
|
"f1" : 0.6667,
|
|
"falseNegative" : 3,
|
|
"falsePositive" : 0,
|
|
"precision" : 1,
|
|
"recall" : 0.5,
|
|
"total" : 106,
|
|
"trueNegative" : 100,
|
|
"truePositive" : 3
|
|
},
|
|
"threshold" : 0.71
|
|
},
|
|
"zh-Hans" : {
|
|
"metrics" : {
|
|
"accuracy" : 0.9717,
|
|
"f1" : 0.6667,
|
|
"falseNegative" : 3,
|
|
"falsePositive" : 0,
|
|
"precision" : 1,
|
|
"recall" : 0.5,
|
|
"total" : 106,
|
|
"trueNegative" : 100,
|
|
"truePositive" : 3
|
|
},
|
|
"threshold" : 0.73
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"falseNegativeExamples" : [
|
|
{
|
|
"confidence" : 0.2919,
|
|
"family" : "schedule_negotiation_golden",
|
|
"id" : "golden-zh-Hans-0102",
|
|
"language" : "zh-Hans",
|
|
"text" : "我提议延期一天，新的截止时间定在周四如何？"
|
|
}
|
|
],
|
|
"falsePositiveExamples" : [
|
|
{
|
|
"confidence" : 0.5813,
|
|
"family" : "task_statement",
|
|
"id" : "golden-zh-Hans-0003",
|
|
"language" : "zh-Hans",
|
|
"text" : "记得把发票抬头改好以后重新提交。"
|
|
},
|
|
{
|
|
"confidence" : 0.5377,
|
|
"family" : "task_question",
|
|
"id" : "golden-zh-Hans-0006",
|
|
"language" : "zh-Hans",
|
|
"text" : "方便帮我约一下周三下午的会议室吗？"
|
|
}
|
|
],
|
|
"id" : "scheduleNegotiation",
|
|
"metrics" : {
|
|
"accuracy" : 0.9858,
|
|
"f1" : 0.9268,
|
|
"falseNegative" : 1,
|
|
"falsePositive" : 2,
|
|
"precision" : 0.9048,
|
|
"recall" : 0.95,
|
|
"total" : 212,
|
|
"trueNegative" : 190,
|
|
"truePositive" : 19
|
|
},
|
|
"metricsByLanguage" : {
|
|
"en" : {
|
|
"accuracy" : 1,
|
|
"f1" : 1,
|
|
"falseNegative" : 0,
|
|
"falsePositive" : 0,
|
|
"precision" : 1,
|
|
"recall" : 1,
|
|
"total" : 106,
|
|
"trueNegative" : 96,
|
|
"truePositive" : 10
|
|
},
|
|
"zh-Hans" : {
|
|
"accuracy" : 0.9717,
|
|
"f1" : 0.8571,
|
|
"falseNegative" : 1,
|
|
"falsePositive" : 2,
|
|
"precision" : 0.8182,
|
|
"recall" : 0.9,
|
|
"total" : 106,
|
|
"trueNegative" : 94,
|
|
"truePositive" : 9
|
|
}
|
|
},
|
|
"thresholdAt90Precision" : {
|
|
"metrics" : {
|
|
"accuracy" : 0.9953,
|
|
"f1" : 0.9744,
|
|
"falseNegative" : 1,
|
|
"falsePositive" : 0,
|
|
"precision" : 1,
|
|
"recall" : 0.95,
|
|
"total" : 212,
|
|
"trueNegative" : 192,
|
|
"truePositive" : 19
|
|
},
|
|
"threshold" : 0.59
|
|
},
|
|
"thresholdAt95Precision" : {
|
|
"metrics" : {
|
|
"accuracy" : 0.9953,
|
|
"f1" : 0.9744,
|
|
"falseNegative" : 1,
|
|
"falsePositive" : 0,
|
|
"precision" : 1,
|
|
"recall" : 0.95,
|
|
"total" : 212,
|
|
"trueNegative" : 192,
|
|
"truePositive" : 19
|
|
},
|
|
"threshold" : 0.59
|
|
},
|
|
"thresholdsAt90PrecisionByLanguage" : {
|
|
"en" : {
|
|
"metrics" : {
|
|
"accuracy" : 1,
|
|
"f1" : 1,
|
|
"falseNegative" : 0,
|
|
"falsePositive" : 0,
|
|
"precision" : 1,
|
|
"recall" : 1,
|
|
"total" : 106,
|
|
"trueNegative" : 96,
|
|
"truePositive" : 10
|
|
},
|
|
"threshold" : 0.44
|
|
},
|
|
"zh-Hans" : {
|
|
"metrics" : {
|
|
"accuracy" : 0.9906,
|
|
"f1" : 0.9474,
|
|
"falseNegative" : 1,
|
|
"falsePositive" : 0,
|
|
"precision" : 1,
|
|
"recall" : 0.9,
|
|
"total" : 106,
|
|
"trueNegative" : 96,
|
|
"truePositive" : 9
|
|
},
|
|
"threshold" : 0.59
|
|
}
|
|
},
|
|
"thresholdsAt95PrecisionByLanguage" : {
|
|
"en" : {
|
|
"metrics" : {
|
|
"accuracy" : 1,
|
|
"f1" : 1,
|
|
"falseNegative" : 0,
|
|
"falsePositive" : 0,
|
|
"precision" : 1,
|
|
"recall" : 1,
|
|
"total" : 106,
|
|
"trueNegative" : 96,
|
|
"truePositive" : 10
|
|
},
|
|
"threshold" : 0.44
|
|
},
|
|
"zh-Hans" : {
|
|
"metrics" : {
|
|
"accuracy" : 0.9906,
|
|
"f1" : 0.9474,
|
|
"falseNegative" : 1,
|
|
"falsePositive" : 0,
|
|
"precision" : 1,
|
|
"recall" : 0.9,
|
|
"total" : 106,
|
|
"trueNegative" : 96,
|
|
"truePositive" : 9
|
|
},
|
|
"threshold" : 0.59
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"falseNegativeExamples" : [
|
|
{
|
|
"confidence" : 0.7062,
|
|
"family" : "confirmation_decision_golden",
|
|
"id" : "golden-en-0143",
|
|
"language" : "en",
|
|
"text" : "Between the two versions, I choose the second. Ship that one."
|
|
},
|
|
{
|
|
"confidence" : 0.6585,
|
|
"family" : "confirmation_decision_golden",
|
|
"id" : "golden-zh-Hans-0141",
|
|
"language" : "zh-Hans",
|
|
"text" : "这三个候选里定小李负责，我确认。"
|
|
},
|
|
{
|
|
"confidence" : 0.3811,
|
|
"family" : "confirmation_decision_golden",
|
|
"id" : "golden-zh-Hans-0138",
|
|
"language" : "zh-Hans",
|
|
"text" : "决定先修稳定性问题，其他需求放到下个版本。"
|
|
}
|
|
],
|
|
"falsePositiveExamples" : [
|
|
|
|
],
|
|
"id" : "confirmationDecision",
|
|
"metrics" : {
|
|
"accuracy" : 0.9858,
|
|
"f1" : 0.9189,
|
|
"falseNegative" : 3,
|
|
"falsePositive" : 0,
|
|
"precision" : 1,
|
|
"recall" : 0.85,
|
|
"total" : 212,
|
|
"trueNegative" : 192,
|
|
"truePositive" : 17
|
|
},
|
|
"metricsByLanguage" : {
|
|
"en" : {
|
|
"accuracy" : 0.9906,
|
|
"f1" : 0.9474,
|
|
"falseNegative" : 1,
|
|
"falsePositive" : 0,
|
|
"precision" : 1,
|
|
"recall" : 0.9,
|
|
"total" : 106,
|
|
"trueNegative" : 96,
|
|
"truePositive" : 9
|
|
},
|
|
"zh-Hans" : {
|
|
"accuracy" : 0.9811,
|
|
"f1" : 0.8889,
|
|
"falseNegative" : 2,
|
|
"falsePositive" : 0,
|
|
"precision" : 1,
|
|
"recall" : 0.8,
|
|
"total" : 106,
|
|
"trueNegative" : 96,
|
|
"truePositive" : 8
|
|
}
|
|
},
|
|
"thresholdAt90Precision" : {
|
|
"metrics" : {
|
|
"accuracy" : 0.9858,
|
|
"f1" : 0.9231,
|
|
"falseNegative" : 2,
|
|
"falsePositive" : 1,
|
|
"precision" : 0.9474,
|
|
"recall" : 0.9,
|
|
"total" : 212,
|
|
"trueNegative" : 191,
|
|
"truePositive" : 18
|
|
},
|
|
"threshold" : 0.7
|
|
},
|
|
"thresholdAt95Precision" : {
|
|
"metrics" : {
|
|
"accuracy" : 0.9858,
|
|
"f1" : 0.9189,
|
|
"falseNegative" : 3,
|
|
"falsePositive" : 0,
|
|
"precision" : 1,
|
|
"recall" : 0.85,
|
|
"total" : 212,
|
|
"trueNegative" : 192,
|
|
"truePositive" : 17
|
|
},
|
|
"threshold" : 0.72
|
|
},
|
|
"thresholdsAt90PrecisionByLanguage" : {
|
|
"en" : {
|
|
"metrics" : {
|
|
"accuracy" : 1,
|
|
"f1" : 1,
|
|
"falseNegative" : 0,
|
|
"falsePositive" : 0,
|
|
"precision" : 1,
|
|
"recall" : 1,
|
|
"total" : 106,
|
|
"trueNegative" : 96,
|
|
"truePositive" : 10
|
|
},
|
|
"threshold" : 0.67
|
|
},
|
|
"zh-Hans" : {
|
|
"metrics" : {
|
|
"accuracy" : 0.9811,
|
|
"f1" : 0.8889,
|
|
"falseNegative" : 2,
|
|
"falsePositive" : 0,
|
|
"precision" : 1,
|
|
"recall" : 0.8,
|
|
"total" : 106,
|
|
"trueNegative" : 96,
|
|
"truePositive" : 8
|
|
},
|
|
"threshold" : 0.72
|
|
}
|
|
},
|
|
"thresholdsAt95PrecisionByLanguage" : {
|
|
"en" : {
|
|
"metrics" : {
|
|
"accuracy" : 1,
|
|
"f1" : 1,
|
|
"falseNegative" : 0,
|
|
"falsePositive" : 0,
|
|
"precision" : 1,
|
|
"recall" : 1,
|
|
"total" : 106,
|
|
"trueNegative" : 96,
|
|
"truePositive" : 10
|
|
},
|
|
"threshold" : 0.67
|
|
},
|
|
"zh-Hans" : {
|
|
"metrics" : {
|
|
"accuracy" : 0.9811,
|
|
"f1" : 0.8889,
|
|
"falseNegative" : 2,
|
|
"falsePositive" : 0,
|
|
"precision" : 1,
|
|
"recall" : 0.8,
|
|
"total" : 106,
|
|
"trueNegative" : 96,
|
|
"truePositive" : 8
|
|
},
|
|
"threshold" : 0.72
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"falseNegativeExamples" : [
|
|
{
|
|
"confidence" : 0.6716,
|
|
"family" : "follow_up_reminder_golden",
|
|
"id" : "golden-zh-Hans-0179",
|
|
"language" : "zh-Hans",
|
|
"text" : "下周一跟进供应商的交货日期。"
|
|
},
|
|
{
|
|
"confidence" : 0.5908,
|
|
"family" : "follow_up_reminder_golden",
|
|
"id" : "golden-zh-Hans-0178",
|
|
"language" : "zh-Hans",
|
|
"text" : "等审批结果出来，请联系申请人说明原因。"
|
|
},
|
|
{
|
|
"confidence" : 0.5591,
|
|
"family" : "task_statement",
|
|
"id" : "golden-en-0049",
|
|
"language" : "en",
|
|
"text" : "Remember to correct the invoice name and submit it again."
|
|
},
|
|
{
|
|
"confidence" : 0.473,
|
|
"family" : "follow_up_reminder_golden",
|
|
"id" : "golden-zh-Hans-0176",
|
|
"language" : "zh-Hans",
|
|
"text" : "会后需要整理三个决定并发到项目群。"
|
|
},
|
|
{
|
|
"confidence" : 0.3102,
|
|
"family" : "task_statement",
|
|
"id" : "golden-en-0050",
|
|
"language" : "en",
|
|
"text" : "First confirm the delivery address with the client, then update the order."
|
|
}
|
|
],
|
|
"falsePositiveExamples" : [
|
|
|
|
],
|
|
"id" : "followUpReminder",
|
|
"metrics" : {
|
|
"accuracy" : 0.9764,
|
|
"f1" : 0.8936,
|
|
"falseNegative" : 5,
|
|
"falsePositive" : 0,
|
|
"precision" : 1,
|
|
"recall" : 0.8077,
|
|
"total" : 212,
|
|
"trueNegative" : 186,
|
|
"truePositive" : 21
|
|
},
|
|
"metricsByLanguage" : {
|
|
"en" : {
|
|
"accuracy" : 0.9811,
|
|
"f1" : 0.9167,
|
|
"falseNegative" : 2,
|
|
"falsePositive" : 0,
|
|
"precision" : 1,
|
|
"recall" : 0.8462,
|
|
"total" : 106,
|
|
"trueNegative" : 93,
|
|
"truePositive" : 11
|
|
},
|
|
"zh-Hans" : {
|
|
"accuracy" : 0.9717,
|
|
"f1" : 0.8696,
|
|
"falseNegative" : 3,
|
|
"falsePositive" : 0,
|
|
"precision" : 1,
|
|
"recall" : 0.7692,
|
|
"total" : 106,
|
|
"trueNegative" : 93,
|
|
"truePositive" : 10
|
|
}
|
|
},
|
|
"thresholdAt90Precision" : {
|
|
"metrics" : {
|
|
"accuracy" : 0.9811,
|
|
"f1" : 0.9167,
|
|
"falseNegative" : 4,
|
|
"falsePositive" : 0,
|
|
"precision" : 1,
|
|
"recall" : 0.8462,
|
|
"total" : 212,
|
|
"trueNegative" : 186,
|
|
"truePositive" : 22
|
|
},
|
|
"threshold" : 0.64
|
|
},
|
|
"thresholdAt95Precision" : {
|
|
"metrics" : {
|
|
"accuracy" : 0.9811,
|
|
"f1" : 0.9167,
|
|
"falseNegative" : 4,
|
|
"falsePositive" : 0,
|
|
"precision" : 1,
|
|
"recall" : 0.8462,
|
|
"total" : 212,
|
|
"trueNegative" : 186,
|
|
"truePositive" : 22
|
|
},
|
|
"threshold" : 0.64
|
|
},
|
|
"thresholdsAt90PrecisionByLanguage" : {
|
|
"en" : {
|
|
"metrics" : {
|
|
"accuracy" : 0.9811,
|
|
"f1" : 0.9167,
|
|
"falseNegative" : 2,
|
|
"falsePositive" : 0,
|
|
"precision" : 1,
|
|
"recall" : 0.8462,
|
|
"total" : 106,
|
|
"trueNegative" : 93,
|
|
"truePositive" : 11
|
|
},
|
|
"threshold" : 0.64
|
|
},
|
|
"zh-Hans" : {
|
|
"metrics" : {
|
|
"accuracy" : 0.9811,
|
|
"f1" : 0.9167,
|
|
"falseNegative" : 2,
|
|
"falsePositive" : 0,
|
|
"precision" : 1,
|
|
"recall" : 0.8462,
|
|
"total" : 106,
|
|
"trueNegative" : 93,
|
|
"truePositive" : 11
|
|
},
|
|
"threshold" : 0.62
|
|
}
|
|
},
|
|
"thresholdsAt95PrecisionByLanguage" : {
|
|
"en" : {
|
|
"metrics" : {
|
|
"accuracy" : 0.9811,
|
|
"f1" : 0.9167,
|
|
"falseNegative" : 2,
|
|
"falsePositive" : 0,
|
|
"precision" : 1,
|
|
"recall" : 0.8462,
|
|
"total" : 106,
|
|
"trueNegative" : 93,
|
|
"truePositive" : 11
|
|
},
|
|
"threshold" : 0.64
|
|
},
|
|
"zh-Hans" : {
|
|
"metrics" : {
|
|
"accuracy" : 0.9811,
|
|
"f1" : 0.9167,
|
|
"falseNegative" : 2,
|
|
"falsePositive" : 0,
|
|
"precision" : 1,
|
|
"recall" : 0.8462,
|
|
"total" : 106,
|
|
"trueNegative" : 93,
|
|
"truePositive" : 11
|
|
},
|
|
"threshold" : 0.62
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"falseNegativeExamples" : [
|
|
{
|
|
"confidence" : 0.7661,
|
|
"family" : "confirmation_decision_golden",
|
|
"id" : "golden-zh-Hans-0142",
|
|
"language" : "zh-Hans",
|
|
"text" : "可以签字了，我批准这份最终合同。"
|
|
},
|
|
{
|
|
"confidence" : 0.7324,
|
|
"family" : "complaint_statement",
|
|
"id" : "golden-zh-Hans-0016",
|
|
"language" : "zh-Hans",
|
|
"text" : "收到的商品外包装破损，里面的杯子也碎了。"
|
|
},
|
|
{
|
|
"confidence" : 0.6908,
|
|
"family" : "follow_up_reminder_golden",
|
|
"id" : "golden-en-0189",
|
|
"language" : "en",
|
|
"text" : "Follow up with the vendor about the delivery date next Monday."
|
|
},
|
|
{
|
|
"confidence" : 0.6896,
|
|
"family" : "confirmation_decision_golden",
|
|
"id" : "golden-en-0143",
|
|
"language" : "en",
|
|
"text" : "Between the two versions, I choose the second. Ship that one."
|
|
},
|
|
{
|
|
"confidence" : 0.632,
|
|
"family" : "schedule_negotiation_boundary",
|
|
"id" : "golden-en-0123",
|
|
"language" : "en",
|
|
"text" : "The kickoff is fixed for Monday at nine; please attend on time."
|
|
}
|
|
],
|
|
"falsePositiveExamples" : [
|
|
{
|
|
"confidence" : 0.9938,
|
|
"family" : "follow_up_reminder_boundary",
|
|
"id" : "golden-en-0210",
|
|
"language" : "en",
|
|
"text" : "There is no owner and no next step."
|
|
},
|
|
{
|
|
"confidence" : 0.9468,
|
|
"family" : "neutral_fact",
|
|
"id" : "golden-en-0079",
|
|
"language" : "en",
|
|
"text" : "The store is open from nine in the morning until ten at night."
|
|
},
|
|
{
|
|
"confidence" : 0.9452,
|
|
"family" : "self_plan",
|
|
"id" : "golden-zh-Hans-0021",
|
|
"language" : "zh-Hans",
|
|
"text" : "明天我要去财务核对发票，不用你帮忙。"
|
|
},
|
|
{
|
|
"confidence" : 0.9416,
|
|
"family" : "confirmation_decision_boundary",
|
|
"id" : "golden-en-0169",
|
|
"language" : "en",
|
|
"text" : "Thanks for explaining; I still need to compare them."
|
|
},
|
|
{
|
|
"confidence" : 0.8797,
|
|
"family" : "schedule_negotiation_boundary",
|
|
"id" : "golden-en-0130",
|
|
"language" : "en",
|
|
"text" : "Lunch is already booked for twelve thirty."
|
|
}
|
|
],
|
|
"id" : "replyableMessage",
|
|
"metrics" : {
|
|
"accuracy" : 0.9057,
|
|
"f1" : 0.9107,
|
|
"falseNegative" : 14,
|
|
"falsePositive" : 6,
|
|
"precision" : 0.9444,
|
|
"recall" : 0.8793,
|
|
"total" : 212,
|
|
"trueNegative" : 90,
|
|
"truePositive" : 102
|
|
},
|
|
"metricsByLanguage" : {
|
|
"en" : {
|
|
"accuracy" : 0.9057,
|
|
"f1" : 0.9138,
|
|
"falseNegative" : 5,
|
|
"falsePositive" : 5,
|
|
"precision" : 0.9138,
|
|
"recall" : 0.9138,
|
|
"total" : 106,
|
|
"trueNegative" : 43,
|
|
"truePositive" : 53
|
|
},
|
|
"zh-Hans" : {
|
|
"accuracy" : 0.9057,
|
|
"f1" : 0.9074,
|
|
"falseNegative" : 9,
|
|
"falsePositive" : 1,
|
|
"precision" : 0.98,
|
|
"recall" : 0.8448,
|
|
"total" : 106,
|
|
"trueNegative" : 47,
|
|
"truePositive" : 49
|
|
}
|
|
},
|
|
"thresholdAt90Precision" : {
|
|
"metrics" : {
|
|
"accuracy" : 0.9151,
|
|
"f1" : 0.9231,
|
|
"falseNegative" : 8,
|
|
"falsePositive" : 10,
|
|
"precision" : 0.9153,
|
|
"recall" : 0.931,
|
|
"total" : 212,
|
|
"trueNegative" : 86,
|
|
"truePositive" : 108
|
|
},
|
|
"threshold" : 0.58
|
|
},
|
|
"thresholdAt95Precision" : {
|
|
"metrics" : {
|
|
"accuracy" : 0.8538,
|
|
"f1" : 0.8517,
|
|
"falseNegative" : 27,
|
|
"falsePositive" : 4,
|
|
"precision" : 0.957,
|
|
"recall" : 0.7672,
|
|
"total" : 212,
|
|
"trueNegative" : 92,
|
|
"truePositive" : 89
|
|
},
|
|
"threshold" : 0.88
|
|
},
|
|
"thresholdsAt90PrecisionByLanguage" : {
|
|
"en" : {
|
|
"metrics" : {
|
|
"accuracy" : 0.9245,
|
|
"f1" : 0.9333,
|
|
"falseNegative" : 2,
|
|
"falsePositive" : 6,
|
|
"precision" : 0.9032,
|
|
"recall" : 0.9655,
|
|
"total" : 106,
|
|
"trueNegative" : 42,
|
|
"truePositive" : 56
|
|
},
|
|
"threshold" : 0.58
|
|
},
|
|
"zh-Hans" : {
|
|
"metrics" : {
|
|
"accuracy" : 0.9057,
|
|
"f1" : 0.9123,
|
|
"falseNegative" : 6,
|
|
"falsePositive" : 4,
|
|
"precision" : 0.9286,
|
|
"recall" : 0.8966,
|
|
"total" : 106,
|
|
"trueNegative" : 44,
|
|
"truePositive" : 52
|
|
},
|
|
"threshold" : 0.55
|
|
}
|
|
},
|
|
"thresholdsAt95PrecisionByLanguage" : {
|
|
"en" : {
|
|
"metrics" : {
|
|
"accuracy" : 0.7547,
|
|
"f1" : 0.7174,
|
|
"falseNegative" : 25,
|
|
"falsePositive" : 1,
|
|
"precision" : 0.9706,
|
|
"recall" : 0.569,
|
|
"total" : 106,
|
|
"trueNegative" : 47,
|
|
"truePositive" : 33
|
|
},
|
|
"threshold" : 0.95
|
|
},
|
|
"zh-Hans" : {
|
|
"metrics" : {
|
|
"accuracy" : 0.9245,
|
|
"f1" : 0.9273,
|
|
"falseNegative" : 7,
|
|
"falsePositive" : 1,
|
|
"precision" : 0.9808,
|
|
"recall" : 0.8793,
|
|
"total" : 106,
|
|
"trueNegative" : 47,
|
|
"truePositive" : 51
|
|
},
|
|
"threshold" : 0.72
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"corpusRecordCount" : 212,
|
|
"exactTrainingOverlapCount" : 212,
|
|
"familyCount" : 20,
|
|
"generatedAt" : "2026-08-26T15:26:18Z",
|
|
"languageCounts" : {
|
|
"en" : 106,
|
|
"zh-Hans" : 106
|
|
},
|
|
"manifestSchemaVersion" : 2,
|
|
"seed" : 20260826,
|
|
"sentiment" : {
|
|
"accuracy" : 0.7689,
|
|
"correct" : 163,
|
|
"macroF1" : 0.8405,
|
|
"perLabelF1" : {
|
|
"negative" : 0.6471,
|
|
"neutral" : 0.8743,
|
|
"positive" : 1
|
|
},
|
|
"total" : 212,
|
|
"unknown" : 44,
|
|
"unknownRate" : 0.2075
|
|
}
|
|
} |