{ "classifiers" : [ { "candidates" : [ { "acceptedForAutomaticRouting" : true, "algorithm" : "maxEnt", "balancedTrainingCount" : 3600, "balancedValidationCount" : 900, "binaryByLanguage" : { "en" : { "accuracy" : 0.8551, "f1" : 0.4321, "falseNegative" : 163, "falsePositive" : 0, "precision" : 1, "recall" : 0.2756, "total" : 1125, "trueNegative" : 900, "truePositive" : 62 }, "zh-Hans" : { "accuracy" : 0.9084, "f1" : 0.7049, "falseNegative" : 102, "falsePositive" : 1, "precision" : 0.9919, "recall" : 0.5467, "total" : 1125, "trueNegative" : 899, "truePositive" : 123 } }, "confidenceThresholdsByLanguage" : { "en" : 0.98, "zh-Hans" : 0.97 }, "goldenBinary" : { "accuracy" : 0.9104, "f1" : 0.4865, "falseNegative" : 19, "falsePositive" : 0, "precision" : 1, "recall" : 0.3214, "total" : 212, "trueNegative" : 184, "truePositive" : 9 }, "goldenBinaryByLanguage" : { "en" : { "accuracy" : 0.8962, "f1" : 0.3529, "falseNegative" : 11, "falsePositive" : 0, "precision" : 1, "recall" : 0.2143, "total" : 106, "trueNegative" : 92, "truePositive" : 3 }, "zh-Hans" : { "accuracy" : 0.9245, "f1" : 0.6, "falseNegative" : 8, "falsePositive" : 0, "precision" : 1, "recall" : 0.4286, "total" : 106, "trueNegative" : 92, "truePositive" : 6 } }, "goldenFalseNegativeExamples" : [ "[en] Alex owns the meeting notes and should post them in the project channel today. (score=0.9363)", "[en] Remember to correct the invoice name and submit it again. (score=0.9748)", "[en] Can you double-check the quotation tomorrow morning? (score=0.9475)", "[en] Would you book a meeting room for Wednesday afternoon? (score=0.9038)", "[en] Could these documents be ready by the end of the month? (score=0.9492)", "[en] Once the client replies, send the final quote to Finance. (score=0.9706)", "[en] Do not forget to check whether the refund arrives on Friday. (score=0.9391)", "[en] After the meeting, summarize the three decisions in the project channel. (score=0.8265)", "[en] When approval arrives, contact the applicant and explain the reason. (score=0.6483)", "[en] Follow up with the vendor about the delivery date next Monday. (score=0.7785)", "[en] Next action: create the release tag after the tests pass. (score=0.8613)", "[zh-Hans] 小王负责整理会议纪要，今天发到项目群。 (score=0.7766)" ], "goldenFalsePositiveExamples" : [ ], "modelBytes" : 25395, "runtimeAssetIndependent" : true, "testBinary" : { "accuracy" : 0.8818, "f1" : 0.5818, "falseNegative" : 265, "falsePositive" : 1, "precision" : 0.9946, "recall" : 0.4111, "total" : 2250, "trueNegative" : 1799, "truePositive" : 185 }, "testFalseNegativeExamples" : [ "[en] Also, Reminder: call the client back before the next meeting. (score=0.7781)", "[en] One more thing: Reminder: book the follow-up visit by Friday. (score=0.665)", "[en] Reminder: check the invoice status by the end of the month. (score=0.8507)", "[en] Reminder: call the client back by the end of the month. (score=0.8371)", "[en] A quick note: Reminder: call the client back before the end of today. (score=0.7565)", "[en] One more thing: Reminder: create the release tag before 3 PM. (score=0.7932)", "[en] Reminder: contact the applicant by the end of the month. (score=0.8331)", "[en] Also, Make review the approval result the next action and do it by the end of the month. (score=0.9599)", "[en] A quick note: Make check the vendor delivery date the next action and do it by the end of the month. (score=0.9286)", "[en] Reminder: contact the applicant after you receive this message. (score=0.809)", "[en] Make summarize the meeting decisions the next action and do it after you receive this message. (score=0.9672)", "[en] A quick note: Reminder: verify the refund status before the next meeting. (score=0.7376)" ], "testFalsePositiveExamples" : [ "[zh-Hans] 个人行动项：给客户回电话，计划下次会议前完成。 (score=0.981)" ], "threshold" : 0.97, "validationBinary" : { "accuracy" : 0.9271, "f1" : 0.7807, "falseNegative" : 158, "falsePositive" : 6, "precision" : 0.9799, "recall" : 0.6489, "total" : 2250, "trueNegative" : 1794, "truePositive" : 292 } } ], "id" : "task", "labels" : [ "notTask", "task" ], "positiveLabel" : "task", "selectedAlgorithm" : "maxEnt", "selectedModelFile" : "TaskIntentClassifier.mlmodel" }, { "candidates" : [ { "acceptedForAutomaticRouting" : true, "algorithm" : "maxEnt", "balancedTrainingCount" : 4320, "balancedValidationCount" : 1080, "binaryByLanguage" : { "en" : { "accuracy" : 1, "f1" : 1, "falseNegative" : 0, "falsePositive" : 0, "precision" : 1, "recall" : 1, "total" : 1125, "trueNegative" : 855, "truePositive" : 270 }, "zh-Hans" : { "accuracy" : 0.9813, "f1" : 0.9595, "falseNegative" : 21, "falsePositive" : 0, "precision" : 1, "recall" : 0.9222, "total" : 1125, "trueNegative" : 855, "truePositive" : 249 } }, "confidenceThresholdsByLanguage" : { "en" : 0.6, "zh-Hans" : 0.6 }, "goldenBinary" : { "accuracy" : 0.9717, "f1" : 0.9333, "falseNegative" : 2, "falsePositive" : 4, "precision" : 0.913, "recall" : 0.9545, "total" : 212, "trueNegative" : 164, "truePositive" : 42 }, "goldenBinaryByLanguage" : { "en" : { "accuracy" : 0.9623, "f1" : 0.9091, "falseNegative" : 2, "falsePositive" : 2, "precision" : 0.9091, "recall" : 0.9091, "total" : 106, "trueNegative" : 82, "truePositive" : 20 }, "zh-Hans" : { "accuracy" : 0.9811, "f1" : 0.9565, "falseNegative" : 0, "falsePositive" : 2, "precision" : 0.9167, "recall" : 1, "total" : 106, "trueNegative" : 82, "truePositive" : 22 } }, "goldenFalseNegativeExamples" : [ "[en] Support has not replied for three days. When will this be handled? (score=0.5821)", "[en] The client is unavailable, so how about next Monday afternoon instead? (score=0.5408)" ], "goldenFalsePositiveExamples" : [ "[en] The store is open from nine in the morning until ten at night. (score=0.7663)", "[en] If Friday does not work, I can do Saturday morning. (score=0.9558)", "[zh-Hans] 门店营业时间是上午九点到晚上十点。 (score=0.6369)", "[zh-Hans] 邀请你周五下午参加评审，时间不变。 (score=0.9299)" ], "modelBytes" : 26879, "runtimeAssetIndependent" : true, "testBinary" : { "accuracy" : 0.9907, "f1" : 0.9802, "falseNegative" : 21, "falsePositive" : 0, "precision" : 1, "recall" : 0.9611, "total" : 2250, "trueNegative" : 1710, "truePositive" : 519 }, "testFalseNegativeExamples" : [ "[zh-Hans] 另外，我需要明确答复：预约记录消失了，你们准备如何解决？ (score=0.4701)", "[zh-Hans] 补充一下，我需要明确答复：预约记录消失了，你们准备如何解决？ (score=0.5047)", "[zh-Hans] 另外，我需要明确答复：发票信息写错了，你们准备如何解决？ (score=0.5631)", "[zh-Hans] 对于订单状态，这次有没有明确答案？ (score=0.4057)", "[zh-Hans] 另外，对于交付日期，对新用户来说有没有明确答案？ (score=0.5786)", "[zh-Hans] 顺便说一下，对于订单状态，目前有没有明确答案？ (score=0.2613)", "[zh-Hans] 还有一件事，对于交付日期，按照刚才的消息有没有明确答案？ (score=0.4831)", "[zh-Hans] 补充一下，对于会议安排，在最新版本里有没有明确答案？ (score=0.5506)", "[zh-Hans] 顺便说一下，对于报价变化，对新用户来说有没有明确答案？ (score=0.5343)", "[zh-Hans] 补充一下，对于项目预算，对新用户来说有没有明确答案？ (score=0.5484)", "[zh-Hans] 顺便说一下，对于项目预算，按照刚才的消息有没有明确答案？ (score=0.4389)", "[zh-Hans] 还有一件事，对于活动规则，对新用户来说有没有明确答案？ (score=0.4204)" ], "testFalsePositiveExamples" : [ ], "threshold" : 0.6, "validationBinary" : { "accuracy" : 0.9742, "f1" : 0.9432, "falseNegative" : 58, "falsePositive" : 0, "precision" : 1, "recall" : 0.8926, "total" : 2250, "trueNegative" : 1710, "truePositive" : 482 } } ], "id" : "question", "labels" : [ "notQuestion", "question" ], "positiveLabel" : "question", "selectedAlgorithm" : "maxEnt", "selectedModelFile" : "QuestionIntentClassifier.mlmodel" }, { "candidates" : [ { "acceptedForAutomaticRouting" : true, "algorithm" : "maxEnt", "balancedTrainingCount" : 1440, "balancedValidationCount" : 360, "binaryByLanguage" : { "en" : { "accuracy" : 0.9751, "f1" : 0.8158, "falseNegative" : 28, "falsePositive" : 0, "precision" : 1, "recall" : 0.6889, "total" : 1125, "trueNegative" : 1035, "truePositive" : 62 }, "zh-Hans" : { "accuracy" : 1, "f1" : 1, "falseNegative" : 0, "falsePositive" : 0, "precision" : 1, "recall" : 1, "total" : 1125, "trueNegative" : 1035, "truePositive" : 90 } }, "confidenceThresholdsByLanguage" : { "en" : 0.85, "zh-Hans" : 0.82 }, "goldenBinary" : { "accuracy" : 0.9623, "f1" : 0.6923, "falseNegative" : 7, "falsePositive" : 1, "precision" : 0.9, "recall" : 0.5625, "total" : 212, "trueNegative" : 195, "truePositive" : 9 }, "goldenBinaryByLanguage" : { "en" : { "accuracy" : 0.9623, "f1" : 0.6667, "falseNegative" : 4, "falsePositive" : 0, "precision" : 1, "recall" : 0.5, "total" : 106, "trueNegative" : 98, "truePositive" : 4 }, "zh-Hans" : { "accuracy" : 0.9623, "f1" : 0.7143, "falseNegative" : 3, "falsePositive" : 1, "precision" : 0.8333, "recall" : 0.625, "total" : 106, "trueNegative" : 97, "truePositive" : 5 } }, "goldenFalseNegativeExamples" : [ "[en] I'd like to invite you to Monday's project kickoff. Are you available? (score=0.3539)", "[en] Would you like to get coffee after work? (score=0.833)", "[en] The kickoff is fixed for Monday at nine; please attend on time. (score=0.6164)", "[en] Want to see a movie this weekend? The tickets are for Saturday at eight. (score=0.3649)", "[zh-Hans] 下班后要不要一起喝杯咖啡？ (score=0.7794)", "[zh-Hans] 项目启动会定在周一上午九点，请准时参加。 (score=0.7173)", "[zh-Hans] 周末一起看电影吗？票是周六晚八点的。 (score=0.2566)" ], "goldenFalsePositiveExamples" : [ "[zh-Hans] 客户临时有事，咱们另约下周一下午怎么样？ (score=0.8284)" ], "modelBytes" : 20757, "runtimeAssetIndependent" : true, "testBinary" : { "accuracy" : 0.9876, "f1" : 0.9157, "falseNegative" : 28, "falsePositive" : 0, "precision" : 1, "recall" : 0.8444, "total" : 2250, "trueNegative" : 2070, "truePositive" : 152 }, "testFalseNegativeExamples" : [ "[en] I saved you a spot to attend the design review next Monday at noon at the client office. How does that sound? (score=0.4163)", "[en] A quick note: I saved you a spot to get coffee tomorrow at 3 PM at the station. How does that sound? (score=0.1822)", "[en] I saved you a spot to attend the design review tomorrow at 3 PM at downstairs from the office. How does that sound? (score=0.4503)", "[en] I saved you a spot to meet this weekend Wednesday evening at Meeting Room 5. How does that sound? (score=0.7564)", "[en] I saved you a spot to attend an appointment after work at the campus cafe. How does that sound? (score=0.6277)", "[en] A quick note: I saved you a spot to have dinner Friday morning at the campus cafe. How does that sound? (score=0.3269)", "[en] One more thing: I saved you a spot to have dinner after work at the Wangjing branch. How does that sound? (score=0.3145)", "[en] Also, I saved you a spot to attend the design review before the end of the month at the Wangjing branch. How does that sound? (score=0.2919)", "[en] Also, I saved you a spot to attend an appointment next Monday at noon at the station. How does that sound? (score=0.2984)", "[en] A quick note: I saved you a spot to attend an appointment at seven tonight at the Wangjing branch. How does that sound? (score=0.35)", "[en] I saved you a spot to have dinner next Monday at noon at the usual place. How does that sound? (score=0.485)", "[en] I saved you a spot to visit the client before the end of the month at the client office. How does that sound? (score=0.3426)" ], "testFalsePositiveExamples" : [ ], "threshold" : 0.82, "validationBinary" : { "accuracy" : 0.9813, "f1" : 0.8704, "falseNegative" : 39, "falsePositive" : 3, "precision" : 0.9792, "recall" : 0.7833, "total" : 2250, "trueNegative" : 2067, "truePositive" : 141 } } ], "id" : "invitation", "labels" : [ "notInvitation", "invitation" ], "positiveLabel" : "invitation", "selectedAlgorithm" : "maxEnt", "selectedModelFile" : "InvitationIntentClassifier.mlmodel" }, { "candidates" : [ { "acceptedForAutomaticRouting" : false, "algorithm" : "maxEnt", "balancedTrainingCount" : 2160, "balancedValidationCount" : 540, "binaryByLanguage" : { "en" : { "accuracy" : 0.9858, "f1" : 0.937, "falseNegative" : 16, "falsePositive" : 0, "precision" : 1, "recall" : 0.8815, "total" : 1125, "trueNegative" : 990, "truePositive" : 119 }, "zh-Hans" : { "accuracy" : 0.9911, "f1" : 0.964, "falseNegative" : 1, "falsePositive" : 9, "precision" : 0.9371, "recall" : 0.9926, "total" : 1125, "trueNegative" : 981, "truePositive" : 134 } }, "confidenceThresholdsByLanguage" : { "en" : 0.6, "zh-Hans" : 0.6 }, "goldenBinary" : { "accuracy" : 0.9764, "f1" : 0.7826, "falseNegative" : 3, "falsePositive" : 2, "precision" : 0.8182, "recall" : 0.75, "total" : 212, "trueNegative" : 198, "truePositive" : 9 }, "goldenBinaryByLanguage" : { "en" : { "accuracy" : 0.9717, "f1" : 0.7273, "falseNegative" : 2, "falsePositive" : 1, "precision" : 0.8, "recall" : 0.6667, "total" : 106, "trueNegative" : 99, "truePositive" : 4 }, "zh-Hans" : { "accuracy" : 0.9811, "f1" : 0.8333, "falseNegative" : 1, "falsePositive" : 1, "precision" : 0.8333, "recall" : 0.8333, "total" : 106, "trueNegative" : 99, "truePositive" : 5 } }, "goldenFalseNegativeExamples" : [ "[en] My account was locked for no reason. Can you restore it today? (score=0.3536)", "[en] Support has not replied for three days. When will this be handled? (score=0.179)", "[zh-Hans] 消息一直发送失败，请尽快给我一个解决方案。 (score=0.4351)" ], "goldenFalsePositiveExamples" : [ "[en] We will see what happens later; no reminder is needed now. (score=0.7067)", "[zh-Hans] 事情终于解决了，我现在轻松多了。 (score=0.7202)" ], "modelBytes" : 22186, "runtimeAssetIndependent" : true, "testBinary" : { "accuracy" : 0.9884, "f1" : 0.9511, "falseNegative" : 17, "falsePositive" : 9, "precision" : 0.9656, "recall" : 0.937, "total" : 2250, "trueNegative" : 1971, "truePositive" : 253 }, "testFalseNegativeExamples" : [ "[en] Just to add, I need a clear answer: the page loads extremely slowly. How are you going to resolve it? (score=0.4323)", "[en] I need a clear answer: the invoice information is wrong. How are you going to resolve it? (score=0.5818)", "[en] Just to add, I need a clear answer: the app keeps crashing. How are you going to resolve it? (score=0.3776)", "[en] Also, I need a clear answer: the app keeps crashing. How are you going to resolve it? (score=0.3915)", "[en] One more thing: I need a clear answer: the order was charged twice. How are you going to resolve it? (score=0.4142)", "[en] A quick note: I need a clear answer: messages never send. How are you going to resolve it? (score=0.5379)", "[en] A quick note: I need a clear answer: the item arrived damaged. How are you going to resolve it? (score=0.4788)", "[en] One more thing: I need a clear answer: the app keeps crashing. How are you going to resolve it? (score=0.3501)", "[en] Also, I need a clear answer: the page loads extremely slowly. How are you going to resolve it? (score=0.4468)", "[en] I need a clear answer: the app keeps crashing. How are you going to resolve it? (score=0.5487)", "[en] Also, I need a clear answer: the invoice information is wrong. How are you going to resolve it? (score=0.4241)", "[en] Just to add, I need a clear answer: the order was charged twice. How are you going to resolve it? (score=0.4432)" ], "testFalsePositiveExamples" : [ "[zh-Hans] 说起来也巧，结果居然成了，而且感觉还挺有意思。 (score=0.6802)", "[zh-Hans] 还有一件事，说起来也巧，这个真的有点好笑，而且今天真的累坏了。 (score=0.6685)", "[zh-Hans] 对于订单状态，这次有没有明确答案？ (score=0.9107)", "[zh-Hans] 另外，对于交付日期，对新用户来说有没有明确答案？ (score=0.6471)", "[zh-Hans] 顺便说一下，对于订单状态，目前有没有明确答案？ (score=0.7345)", "[zh-Hans] 对于订单状态，目前有没有明确答案？ (score=0.8481)", "[zh-Hans] 另外，对于账户权限，这次有没有明确答案？ (score=0.8156)", "[zh-Hans] 对于项目预算，目前有没有明确答案？ (score=0.7395)", "[zh-Hans] 对于活动规则，这次有没有明确答案？ (score=0.8127)" ], "threshold" : 0.6, "validationBinary" : { "accuracy" : 0.9978, "f1" : 0.9908, "falseNegative" : 2, "falsePositive" : 3, "precision" : 0.9889, "recall" : 0.9926, "total" : 2250, "trueNegative" : 1977, "truePositive" : 268 } } ], "id" : "complaint", "labels" : [ "notComplaint", "complaint" ], "positiveLabel" : "complaint", "selectedAlgorithm" : "maxEnt", "selectedModelFile" : "ComplaintIntentClassifier.mlmodel" }, { "candidates" : [ { "acceptedForAutomaticRouting" : false, "algorithm" : "maxEnt", "balancedTrainingCount" : 720, "balancedValidationCount" : 180, "binaryByLanguage" : { "en" : { "accuracy" : 0.9991, "f1" : 0.989, "falseNegative" : 0, "falsePositive" : 1, "precision" : 0.9783, "recall" : 1, "total" : 1125, "trueNegative" : 1079, "truePositive" : 45 }, "zh-Hans" : { "accuracy" : 0.9964, "f1" : 0.9556, "falseNegative" : 2, "falsePositive" : 2, "precision" : 0.9556, "recall" : 0.9556, "total" : 1125, "trueNegative" : 1078, "truePositive" : 43 } }, "confidenceThresholdsByLanguage" : { "en" : 0.53, "zh-Hans" : 0.53 }, "goldenBinary" : { "accuracy" : 0.9764, "f1" : 0.8837, "falseNegative" : 1, "falsePositive" : 4, "precision" : 0.8261, "recall" : 0.95, "total" : 212, "trueNegative" : 188, "truePositive" : 19 }, "goldenBinaryByLanguage" : { "en" : { "accuracy" : 1, "f1" : 1, "falseNegative" : 0, "falsePositive" : 0, "precision" : 1, "recall" : 1, "total" : 106, "trueNegative" : 96, "truePositive" : 10 }, "zh-Hans" : { "accuracy" : 0.9528, "f1" : 0.7826, "falseNegative" : 1, "falsePositive" : 4, "precision" : 0.6923, "recall" : 0.9, "total" : 106, "trueNegative" : 92, "truePositive" : 9 } }, "goldenFalseNegativeExamples" : [ "[zh-Hans] 我提议延期一天，新的截止时间定在周四如何？ (score=0.4017)" ], "goldenFalsePositiveExamples" : [ "[zh-Hans] 记得把发票抬头改好以后重新提交。 (score=0.7242)", "[zh-Hans] 你能明天上午把报价单核对一遍吗？ (score=0.8139)", "[zh-Hans] 方便帮我约一下周三下午的会议室吗？ (score=0.6747)", "[zh-Hans] 门店营业时间是上午九点到晚上十点。 (score=0.636)" ], "modelBytes" : 17408, "runtimeAssetIndependent" : true, "testBinary" : { "accuracy" : 0.9978, "f1" : 0.9724, "falseNegative" : 2, "falsePositive" : 3, "precision" : 0.967, "recall" : 0.9778, "total" : 2250, "trueNegative" : 2157, "truePositive" : 88 }, "testFalseNegativeExamples" : [ "[zh-Hans] 还有一件事，关于看电影，我提议改期到下周一早上，你觉得呢？ (score=0.4251)", "[zh-Hans] 另外，关于看电影，我提议改期到下周一早上，你觉得呢？ (score=0.4196)" ], "testFalsePositiveExamples" : [ "[en] Invitation confirmed for next Monday at noon at the station; can you join us to attend an appointment? (score=0.5816)", "[zh-Hans] 不知道你明天下午三点是否有空，要不要在线上会议室周末聚会？ (score=0.6057)", "[zh-Hans] 不知道你周五上午是否有空，要不要在火车站开会？ (score=0.5337)" ], "threshold" : 0.53, "validationBinary" : { "accuracy" : 1, "f1" : 1, "falseNegative" : 0, "falsePositive" : 0, "precision" : 1, "recall" : 1, "total" : 2250, "trueNegative" : 2160, "truePositive" : 90 } } ], "id" : "scheduleNegotiation", "labels" : [ "notScheduleNegotiation", "scheduleNegotiation" ], "positiveLabel" : "scheduleNegotiation", "selectedAlgorithm" : "maxEnt", "selectedModelFile" : "ScheduleNegotiationIntentClassifier.mlmodel" }, { "candidates" : [ { "acceptedForAutomaticRouting" : true, "algorithm" : "maxEnt", "balancedTrainingCount" : 1440, "balancedValidationCount" : 360, "binaryByLanguage" : { "en" : { "accuracy" : 0.968, "f1" : 0.75, "falseNegative" : 36, "falsePositive" : 0, "precision" : 1, "recall" : 0.6, "total" : 1125, "trueNegative" : 1035, "truePositive" : 54 }, "zh-Hans" : { "accuracy" : 0.9822, "f1" : 0.875, "falseNegative" : 20, "falsePositive" : 0, "precision" : 1, "recall" : 0.7778, "total" : 1125, "trueNegative" : 1035, "truePositive" : 70 } }, "confidenceThresholdsByLanguage" : { "en" : 0.85, "zh-Hans" : 0.84 }, "goldenBinary" : { "accuracy" : 0.9717, "f1" : 0.8235, "falseNegative" : 6, "falsePositive" : 0, "precision" : 1, "recall" : 0.7, "total" : 212, "trueNegative" : 192, "truePositive" : 14 }, "goldenBinaryByLanguage" : { "en" : { "accuracy" : 0.9811, "f1" : 0.8889, "falseNegative" : 2, "falsePositive" : 0, "precision" : 1, "recall" : 0.8, "total" : 106, "trueNegative" : 96, "truePositive" : 8 }, "zh-Hans" : { "accuracy" : 0.9623, "f1" : 0.75, "falseNegative" : 4, "falsePositive" : 0, "precision" : 1, "recall" : 0.6, "total" : 106, "trueNegative" : 96, "truePositive" : 6 } }, "goldenFalseNegativeExamples" : [ "[en] Between the two versions, I choose the second. Ship that one. (score=0.7062)", "[en] You may sign it; I approve the final contract. (score=0.7205)", "[zh-Hans] 预算我批准了，可以开始采购。 (score=0.7251)", "[zh-Hans] 决定先修稳定性问题，其他需求放到下个版本。 (score=0.3811)", "[zh-Hans] 这三个候选里定小李负责，我确认。 (score=0.6585)", "[zh-Hans] 可以签字了，我批准这份最终合同。 (score=0.7677)" ], "goldenFalsePositiveExamples" : [ ], "modelBytes" : 19331, "runtimeAssetIndependent" : true, "testBinary" : { "accuracy" : 0.9751, "f1" : 0.8158, "falseNegative" : 56, "falsePositive" : 0, "precision" : 1, "recall" : 0.6889, "total" : 2250, "trueNegative" : 2070, "truePositive" : 124 }, "testFalseNegativeExamples" : [ "[en] Just to add, Final call: adopt the monthly plan and proceed before the next meeting. (score=0.782)", "[en] A quick note: Final call: adopt vendor B and proceed after you receive this message. (score=0.7459)", "[en] Final call: adopt the second design and proceed after you receive this message. (score=0.7604)", "[en] Also, Final call: adopt the blue version and proceed this week. (score=0.8277)", "[en] Just to add, Final call: adopt the basic release first and proceed this week. (score=0.8485)", "[en] A quick note: Final call: adopt the second design and proceed after you receive this message. (score=0.7266)", "[en] Just to add, Final call: adopt vendor B and proceed by the end of the month. (score=0.7647)", "[en] Final call: adopt vendor B and proceed before the end of today. (score=0.8445)", "[en] Also, Final call: adopt the monthly plan and proceed after you receive this message. (score=0.5835)", "[en] Just to add, Final call: adopt the basic release first and proceed after you receive this message. (score=0.7193)", "[en] Also, I have made the decision: the payment method is approved. (score=0.772)", "[en] Just to add, Finance is the confirmed owner; this is the formal conclusion. (score=0.5379)" ], "testFalsePositiveExamples" : [ ], "threshold" : 0.84, "validationBinary" : { "accuracy" : 0.9716, "f1" : 0.7881, "falseNegative" : 61, "falsePositive" : 3, "precision" : 0.9754, "recall" : 0.6611, "total" : 2250, "trueNegative" : 2067, "truePositive" : 119 } } ], "id" : "confirmationDecision", "labels" : [ "notConfirmationDecision", "confirmationDecision" ], "positiveLabel" : "confirmationDecision", "selectedAlgorithm" : "maxEnt", "selectedModelFile" : "ConfirmationDecisionIntentClassifier.mlmodel" }, { "candidates" : [ { "acceptedForAutomaticRouting" : true, "algorithm" : "maxEnt", "balancedTrainingCount" : 2160, "balancedValidationCount" : 540, "binaryByLanguage" : { "en" : { "accuracy" : 0.9804, "f1" : 0.9113, "falseNegative" : 22, "falsePositive" : 0, "precision" : 1, "recall" : 0.837, "total" : 1125, "trueNegative" : 990, "truePositive" : 113 }, "zh-Hans" : { "accuracy" : 0.9716, "f1" : 0.8678, "falseNegative" : 30, "falsePositive" : 2, "precision" : 0.9813, "recall" : 0.7778, "total" : 1125, "trueNegative" : 988, "truePositive" : 105 } }, "confidenceThresholdsByLanguage" : { "en" : 0.88, "zh-Hans" : 0.87 }, "goldenBinary" : { "accuracy" : 0.9528, "f1" : 0.7619, "falseNegative" : 10, "falsePositive" : 0, "precision" : 1, "recall" : 0.6154, "total" : 212, "trueNegative" : 186, "truePositive" : 16 }, "goldenBinaryByLanguage" : { "en" : { "accuracy" : 0.9623, "f1" : 0.8182, "falseNegative" : 4, "falsePositive" : 0, "precision" : 1, "recall" : 0.6923, "total" : 106, "trueNegative" : 93, "truePositive" : 9 }, "zh-Hans" : { "accuracy" : 0.9434, "f1" : 0.7, "falseNegative" : 6, "falsePositive" : 0, "precision" : 1, "recall" : 0.5385, "total" : 106, "trueNegative" : 93, "truePositive" : 7 } }, "goldenFalseNegativeExamples" : [ "[en] Remember to correct the invoice name and submit it again. (score=0.5591)", "[en] First confirm the delivery address with the client, then update the order. (score=0.3102)", "[en] After the meeting, summarize the three decisions in the project channel. (score=0.8453)", "[en] When approval arrives, contact the applicant and explain the reason. (score=0.8415)", "[zh-Hans] 记得把发票抬头改好以后重新提交。 (score=0.8331)", "[zh-Hans] 先记一下：月底前我自己联系客户。 (score=0.7792)", "[zh-Hans] 客户回复后，下一步把最终报价发给财务。 (score=0.8046)", "[zh-Hans] 会后需要整理三个决定并发到项目群。 (score=0.473)", "[zh-Hans] 等审批结果出来，请联系申请人说明原因。 (score=0.5908)", "[zh-Hans] 下周一跟进供应商的交货日期。 (score=0.6716)" ], "goldenFalsePositiveExamples" : [ ], "modelBytes" : 23455, "runtimeAssetIndependent" : true, "testBinary" : { "accuracy" : 0.976, "f1" : 0.8898, "falseNegative" : 52, "falsePositive" : 2, "precision" : 0.9909, "recall" : 0.8074, "total" : 2250, "trueNegative" : 1978, "truePositive" : 218 }, "testFalseNegativeExamples" : [ "[en] A quick note: I need to remember to record my temperature before the end of today. (score=0.8766)", "[en] Just to add, I need to remember to check the vendor delivery date by the end of the month. (score=0.8744)", "[en] Just to add, I need to remember to send the final version before 3 PM. (score=0.6876)", "[en] Just to add, I need to remember to check the vendor delivery date before 3 PM. (score=0.8592)", "[en] A quick note: I need to remember to update the project channel before 3 PM. (score=0.8674)", "[en] A quick note: I need to remember to review the shipment status by Friday. (score=0.8297)", "[en] A quick note: Revisit the leave policy after a few days and share the latest status. (score=0.6859)", "[en] Also, Revisit the required documents after a few days and share the latest status. (score=0.5351)", "[en] Revisit the account permissions after a few days and share the latest status. (score=0.4891)", "[en] Also, Revisit the order status after a few days and share the latest status. (score=0.3482)", "[en] One more thing: Revisit the required documents after a few days and share the latest status. (score=0.6254)", "[en] Just to add, Revisit the travel plan after a few days and share the latest status. (score=0.4133)" ], "testFalsePositiveExamples" : [ "[zh-Hans] 请假制度没有确定下一步，以后有机会再联系。 (score=0.8868)", "[zh-Hans] 资料要求没有确定下一步，有空或许跟进。 (score=0.8877)" ], "threshold" : 0.87, "validationBinary" : { "accuracy" : 0.9636, "f1" : 0.8248, "falseNegative" : 77, "falsePositive" : 5, "precision" : 0.9747, "recall" : 0.7148, "total" : 2250, "trueNegative" : 1975, "truePositive" : 193 } } ], "id" : "followUpReminder", "labels" : [ "notFollowUpReminder", "followUpReminder" ], "positiveLabel" : "followUpReminder", "selectedAlgorithm" : "maxEnt", "selectedModelFile" : "FollowUpReminderIntentClassifier.mlmodel" }, { "candidates" : [ { "acceptedForAutomaticRouting" : true, "algorithm" : "maxEnt", "balancedTrainingCount" : 5720, "balancedValidationCount" : 1620, "binaryByLanguage" : { "en" : { "accuracy" : 0.9262, "f1" : 0.9406, "falseNegative" : 63, "falsePositive" : 20, "precision" : 0.9705, "recall" : 0.9125, "total" : 1125, "trueNegative" : 385, "truePositive" : 657 }, "zh-Hans" : { "accuracy" : 0.9244, "f1" : 0.9382, "falseNegative" : 75, "falsePositive" : 10, "precision" : 0.9847, "recall" : 0.8958, "total" : 1125, "trueNegative" : 395, "truePositive" : 645 } }, "confidenceThresholdsByLanguage" : { "en" : 0.75, "zh-Hans" : 0.79 }, "goldenBinary" : { "accuracy" : 0.9057, "f1" : 0.9107, "falseNegative" : 14, "falsePositive" : 6, "precision" : 0.9444, "recall" : 0.8793, "total" : 212, "trueNegative" : 90, "truePositive" : 102 }, "goldenBinaryByLanguage" : { "en" : { "accuracy" : 0.9057, "f1" : 0.9138, "falseNegative" : 5, "falsePositive" : 5, "precision" : 0.9138, "recall" : 0.9138, "total" : 106, "trueNegative" : 43, "truePositive" : 53 }, "zh-Hans" : { "accuracy" : 0.9057, "f1" : 0.9074, "falseNegative" : 9, "falsePositive" : 1, "precision" : 0.98, "recall" : 0.8448, "total" : 106, "trueNegative" : 47, "truePositive" : 49 } }, "goldenFalseNegativeExamples" : [ "[en] My flight changed; can pickup move to nine tonight? (score=0.3099)", "[en] The kickoff is fixed for Monday at nine; please attend on time. (score=0.632)", "[en] Between the two versions, I choose the second. Ship that one. (score=0.6896)", "[en] After the meeting, summarize the three decisions in the project channel. (score=0.5006)", "[en] Follow up with the vendor about the delivery date next Monday. (score=0.6908)", "[zh-Hans] 这些材料可以在月底前准备好吗？ (score=0.47)", "[zh-Hans] 收到的商品外包装破损，里面的杯子也碎了。 (score=0.7324)", "[zh-Hans] 客服已经三天没有回复了，请问什么时候能处理？ (score=0.5888)", "[zh-Hans] 如果周五不行，我可以配合周六上午。 (score=0.4275)", "[zh-Hans] 预算我批准了，可以开始采购。 (score=0.4476)", "[zh-Hans] 这三个候选里定小李负责，我确认。 (score=0.2995)", "[zh-Hans] 可以签字了，我批准这份最终合同。 (score=0.7661)" ], "goldenFalsePositiveExamples" : [ "[en] The store is open from nine in the morning until ten at night. (score=0.9468)", "[en] Lunch is already booked for twelve thirty. (score=0.8797)", "[en] Thanks for explaining; I still need to compare them. (score=0.9416)", "[en] Leave it for now and mention it whenever it comes up. (score=0.8451)", "[en] There is no owner and no next step. (score=0.9938)", "[zh-Hans] 明天我要去财务核对发票，不用你帮忙。 (score=0.9452)" ], "modelBytes" : 26927, "runtimeAssetIndependent" : true, "testBinary" : { "accuracy" : 0.9253, "f1" : 0.9394, "falseNegative" : 138, "falsePositive" : 30, "precision" : 0.9775, "recall" : 0.9042, "total" : 2250, "trueNegative" : 780, "truePositive" : 1302 }, "testFalseNegativeExamples" : [ "[en] Also, I have made the decision: the payment method is approved. (score=0.605)", "[en] Also, I have made the decision: the leave policy is approved. (score=0.65)", "[en] One more thing: I have made the decision: the delivery date is approved. (score=0.6617)", "[en] Just to add, I have made the decision: the required documents is approved. (score=0.6346)", "[en] A quick note: I have made the decision: the required documents is approved. (score=0.7057)", "[en] Also, I have made the decision: the refund process is approved. (score=0.7208)", "[en] I have made the decision: the required documents is approved. (score=0.7018)", "[en] Just to add, I have made the decision: the delivery date is approved. (score=0.6094)", "[en] Just to add, I have made the decision: the order status is approved. (score=0.5233)", "[en] Also, I have made the decision: the pricing change is approved. (score=0.6589)", "[en] I have made the decision: the event rules is approved. (score=0.5794)", "[en] I have made the decision: the order status is approved. (score=0.598)" ], "testFalsePositiveExamples" : [ "[en] I need to remember to book the follow-up visit this week. (score=0.8899)", "[en] One more thing: I need to remember to check the invoice status this week. (score=0.9443)", "[en] I need to remember to summarize the meeting decisions before the end of today. (score=0.9474)", "[en] A quick note: I need to remember to verify the refund status before the next meeting. (score=0.9599)", "[en] I need to remember to check the invoice status before 3 PM. (score=0.9522)", "[en] Also, I need to remember to call the client back before the next meeting. (score=0.918)", "[en] I need to remember to book the follow-up visit by Friday. (score=0.8857)", "[en] One more thing: I need to remember to book the follow-up visit before the next meeting. (score=0.884)", "[en] I need to remember to update the project channel this week. (score=0.9813)", "[en] A quick note: I need to remember to record my temperature before the end of today. (score=0.9567)", "[en] Also, I need to remember to check the vendor delivery date by the end of the month. (score=0.9574)", "[en] I need to remember to check the vendor delivery date before the next meeting. (score=0.9549)" ], "threshold" : 0.75, "validationBinary" : { "accuracy" : 0.9209, "f1" : 0.936, "falseNegative" : 139, "falsePositive" : 39, "precision" : 0.9709, "recall" : 0.9035, "total" : 2250, "trueNegative" : 771, "truePositive" : 1301 } } ], "id" : "replyableMessage", "labels" : [ "notReplyableMessage", "replyableMessage" ], "positiveLabel" : "replyableMessage", "selectedAlgorithm" : "maxEnt", "selectedModelFile" : "ConversationalReplyIntentClassifier.mlmodel" }, { "candidates" : [ { "acceptedForAutomaticRouting" : true, "algorithm" : "maxEnt", "balancedTrainingCount" : 1080, "balancedValidationCount" : 270, "goldenMulticlass" : { "accuracy" : 0.9151, "confusion" : { "negative" : { "negative" : 16, "neutral" : 1, "positive" : 1 }, "neutral" : { "negative" : 12, "neutral" : 172, "positive" : 4 }, "positive" : { "negative" : 0, "neutral" : 0, "positive" : 6 } }, "macroF1" : 0.7848, "perLabel" : { "negative" : { "accuracy" : 0.934, "f1" : 0.6957, "falseNegative" : 2, "falsePositive" : 12, "precision" : 0.5714, "recall" : 0.8889, "total" : 212, "trueNegative" : 182, "truePositive" : 16 }, "neutral" : { "accuracy" : 0.9198, "f1" : 0.9529, "falseNegative" : 16, "falsePositive" : 1, "precision" : 0.9942, "recall" : 0.9149, "total" : 212, "trueNegative" : 23, "truePositive" : 172 }, "positive" : { "accuracy" : 0.9764, "f1" : 0.7059, "falseNegative" : 0, "falsePositive" : 5, "precision" : 0.5455, "recall" : 1, "total" : 212, "trueNegative" : 201, "truePositive" : 6 } }, "total" : 212 }, "modelBytes" : 20878, "multiclassByLanguage" : { "en" : { "accuracy" : 0.9964, "confusion" : { "negative" : { "negative" : 180, "neutral" : 0, "positive" : 0 }, "neutral" : { "negative" : 4, "neutral" : 896, "positive" : 0 }, "positive" : { "negative" : 0, "neutral" : 0, "positive" : 45 } }, "macroF1" : 0.9956, "perLabel" : { "negative" : { "accuracy" : 0.9964, "f1" : 0.989, "falseNegative" : 0, "falsePositive" : 4, "precision" : 0.9783, "recall" : 1, "total" : 1125, "trueNegative" : 941, "truePositive" : 180 }, "neutral" : { "accuracy" : 0.9964, "f1" : 0.9978, "falseNegative" : 4, "falsePositive" : 0, "precision" : 1, "recall" : 0.9956, "total" : 1125, "trueNegative" : 225, "truePositive" : 896 }, "positive" : { "accuracy" : 1, "f1" : 1, "falseNegative" : 0, "falsePositive" : 0, "precision" : 1, "recall" : 1, "total" : 1125, "trueNegative" : 1080, "truePositive" : 45 } }, "total" : 1125 }, "zh-Hans" : { "accuracy" : 0.9698, "confusion" : { "negative" : { "negative" : 164, "neutral" : 16, "positive" : 0 }, "neutral" : { "negative" : 18, "neutral" : 882, "positive" : 0 }, "positive" : { "negative" : 0, "neutral" : 0, "positive" : 45 } }, "macroF1" : 0.9624, "perLabel" : { "negative" : { "accuracy" : 0.9698, "f1" : 0.9061, "falseNegative" : 16, "falsePositive" : 18, "precision" : 0.9011, "recall" : 0.9111, "total" : 1125, "trueNegative" : 927, "truePositive" : 164 }, "neutral" : { "accuracy" : 0.9698, "f1" : 0.9811, "falseNegative" : 18, "falsePositive" : 16, "precision" : 0.9822, "recall" : 0.98, "total" : 1125, "trueNegative" : 209, "truePositive" : 882 }, "positive" : { "accuracy" : 1, "f1" : 1, "falseNegative" : 0, "falsePositive" : 0, "precision" : 1, "recall" : 1, "total" : 1125, "trueNegative" : 1080, "truePositive" : 45 } }, "total" : 1125 } }, "runtimeAssetIndependent" : true, "testMulticlass" : { "accuracy" : 0.9831, "confusion" : { "negative" : { "negative" : 344, "neutral" : 16, "positive" : 0 }, "neutral" : { "negative" : 22, "neutral" : 1778, "positive" : 0 }, "positive" : { "negative" : 0, "neutral" : 0, "positive" : 90 } }, "macroF1" : 0.979, "perLabel" : { "negative" : { "accuracy" : 0.9831, "f1" : 0.9477, "falseNegative" : 16, "falsePositive" : 22, "precision" : 0.9399, "recall" : 0.9556, "total" : 2250, "trueNegative" : 1868, "truePositive" : 344 }, "neutral" : { "accuracy" : 0.9831, "f1" : 0.9894, "falseNegative" : 22, "falsePositive" : 16, "precision" : 0.9911, "recall" : 0.9878, "total" : 2250, "trueNegative" : 434, "truePositive" : 1778 }, "positive" : { "accuracy" : 1, "f1" : 1, "falseNegative" : 0, "falsePositive" : 0, "precision" : 1, "recall" : 1, "total" : 2250, "trueNegative" : 2160, "truePositive" : 90 } }, "total" : 2250 }, "validationMulticlass" : { "accuracy" : 0.9876, "confusion" : { "negative" : { "negative" : 359, "neutral" : 1, "positive" : 0 }, "neutral" : { "negative" : 26, "neutral" : 1773, "positive" : 1 }, "positive" : { "negative" : 0, "neutral" : 0, "positive" : 90 } }, "macroF1" : 0.9835, "perLabel" : { "negative" : { "accuracy" : 0.988, "f1" : 0.9638, "falseNegative" : 1, "falsePositive" : 26, "precision" : 0.9325, "recall" : 0.9972, "total" : 2250, "trueNegative" : 1864, "truePositive" : 359 }, "neutral" : { "accuracy" : 0.9876, "f1" : 0.9922, "falseNegative" : 27, "falsePositive" : 1, "precision" : 0.9994, "recall" : 0.985, "total" : 2250, "trueNegative" : 449, "truePositive" : 1773 }, "positive" : { "accuracy" : 0.9996, "f1" : 0.9945, "falseNegative" : 0, "falsePositive" : 1, "precision" : 0.989, "recall" : 1, "total" : 2250, "trueNegative" : 2159, "truePositive" : 90 } }, "total" : 2250 } } ], "id" : "sentiment", "labels" : [ "negative", "neutral", "positive" ], "selectedAlgorithm" : "maxEnt", "selectedModelFile" : "SentimentClassifier.mlmodel" } ], "corpusCount" : 13332, "corpusPath" : "ModelTraining/ClipboardSemantics/clipboard_semantic_corpus.jsonl", "generatedAt" : "2026-08-26T15:18:57Z", "goldenCount" : 212, "selectionPolicy" : "Validation only: global and per-language binary thresholds require precision >= 0.97, then maximize recall; languages with fewer than 20 examples per class fall back to the global threshold. sentiment prioritizes macro-F1. Automatic routing also requires a self-contained maxEnt model because BERT embedding assets are not guaranteed in extensions. Test and golden data gate deployment but never tune model weights.", "testCount" : 2250, "trainingCount" : 8620, "validationCount" : 2250 }