{ "binaryMacro" : { "accuracy" : 0.8812, "f1" : 0.2783, "precision" : 0.2948, "recall" : 0.332 }, "binaryMacroByLanguage" : { "en" : { "accuracy" : 0.8905, "f1" : 0.2891, "precision" : 0.3107, "recall" : 0.2941 }, "zh-Hans" : { "accuracy" : 0.8767, "f1" : 0.2752, "precision" : 0.2969, "recall" : 0.3327 } }, "classifiers" : [ { "falseNegativeExamples" : [ { "confidence" : 0.9596, "family" : "online_massive_intent_30", "id" : "comprehensive-massive-zh-CN-8182", "language" : "zh-Hans", "sourceDataset" : "AmazonScience/MASSIVE zh-CN test", "text" : "你能删除牙医预约吗" }, { "confidence" : 0.8947, "family" : "online_massive_intent_33", "id" : "comprehensive-massive-en-US-16891", "language" : "en", "sourceDataset" : "AmazonScience/MASSIVE en-US test", "text" : "send john smith and email" }, { "confidence" : 0.8831, "family" : "online_massive_intent_33", "id" : "comprehensive-massive-en-US-16438", "language" : "en", "sourceDataset" : "AmazonScience/MASSIVE en-US test", "text" : "i need this email sent now" }, { "confidence" : 0.8786, "family" : "online_enron_intent_pos", "id" : "comprehensive-enron-intent_pos-892", "language" : "en", "sourceDataset" : "Verified Enron Intent Dataset", "text" : "Please send it to me again so I can follow up." }, { "confidence" : 0.8759, "family" : "online_goemotions_reddit", "id" : "comprehensive-goemotions-ee451sg", "language" : "en", "sourceDataset" : "Google Research GoEmotions test", "text" : "As a Ringwraith this offends me. Please build me a snowfort." } ], "falsePositiveExamples" : [ { "confidence" : 0.9998, "family" : "online_cped_disagreement", "id" : "comprehensive-cped-34_224_008", "language" : "zh-Hans", "sourceDataset" : "CPED test split", "text" : "别过两天过两天就晚了" }, { "confidence" : 0.9991, "family" : "online_cped_question", "id" : "comprehensive-cped-27_214_016", "language" : "zh-Hans", "sourceDataset" : "CPED test split", "text" : "不成帮你找一帮群众演员围着你给你当鲜花" }, { "confidence" : 0.999, "family" : "online_enron_intent_neg", "id" : "comprehensive-enron-intent_neg-507", "language" : "en", "sourceDataset" : "Verified Enron Intent Dataset", "text" : "Hi Cam, If you’re not ready to share everything you’re doing on Pinterest with the rest of the world, you might want to check out secret boards." }, { "confidence" : 0.999, "family" : "online_enron_intent_neg", "id" : "comprehensive-enron-intent_neg-999", "language" : "en", "sourceDataset" : "Verified Enron Intent Dataset", "text" : "If you have any questions, comments or suggestions please feel free to get in touch with me any time by replying to this email and if you haven't had a chance to see the new site, go to $LINK" }, { "confidence" : 0.9989, "family" : "online_enron_intent_neg", "id" : "comprehensive-enron-intent_neg-923", "language" : "en", "sourceDataset" : "Verified Enron Intent Dataset", "text" : "If you continue to receive emails after this period, please click below to contact Consumer Services:" } ], "id" : "task", "metrics" : { "accuracy" : 0.9116, "f1" : 0.1194, "falseNegative" : 1531, "falsePositive" : 2111, "precision" : 0.1047, "recall" : 0.1389, "total" : 41195, "trueNegative" : 37306, "truePositive" : 247 }, "metricsByLanguage" : { "en" : { "accuracy" : 0.8871, "f1" : 0.1753, "falseNegative" : 961, "falsePositive" : 526, "precision" : 0.231, "recall" : 0.1412, "total" : 13176, "trueNegative" : 11531, "truePositive" : 158 }, "zh-Hans" : { "accuracy" : 0.9231, "f1" : 0.0763, "falseNegative" : 570, "falsePositive" : 1585, "precision" : 0.0532, "recall" : 0.1351, "total" : 28019, "trueNegative" : 25775, "truePositive" : 89 } }, "metricsBySource" : { "AmazonScience/MASSIVE en-US test" : { "accuracy" : 0.7833, "f1" : 0.1392, "falseNegative" : 489, "falsePositive" : 154, "precision" : 0.2524, "recall" : 0.0961, "total" : 2967, "trueNegative" : 2272, "truePositive" : 52 }, "AmazonScience/MASSIVE zh-CN test" : { "accuracy" : 0.794, "f1" : 0.1767, "falseNegative" : 415, "falsePositive" : 172, "precision" : 0.2681, "recall" : 0.1318, "total" : 2850, "trueNegative" : 2200, "truePositive" : 63 }, "CFPB via hpe-ai/customer-complaints" : { "accuracy" : 0.788, "f1" : 0, "falseNegative" : 53, "falsePositive" : 0, "precision" : 0, "recall" : 0, "total" : 250, "trueNegative" : 197, "truePositive" : 0 }, "CPED test split" : { "accuracy" : 0.9379, "f1" : 0.0324, "falseNegative" : 142, "falsePositive" : 1413, "precision" : 0.0181, "recall" : 0.1548, "total" : 25049, "trueNegative" : 23468, "truePositive" : 26 }, "Google Research GoEmotions test" : { "accuracy" : 0.9773, "f1" : 0.0611, "falseNegative" : 46, "falsePositive" : 77, "precision" : 0.0494, "recall" : 0.08, "total" : 5418, "trueNegative" : 5291, "truePositive" : 4 }, "Meituan-Dianping/ASAP test" : { "accuracy" : 0.8917, "f1" : 0, "falseNegative" : 13, "falsePositive" : 0, "precision" : 0, "recall" : 0, "total" : 120, "trueNegative" : 107, "truePositive" : 0 }, "Sulak2020 GitHub issues multi-repo dataset" : { "accuracy" : 0.8984, "f1" : 0, "falseNegative" : 90, "falsePositive" : 12, "precision" : 0, "recall" : 0, "total" : 1004, "trueNegative" : 902, "truePositive" : 0 }, "Verified Enron Intent Dataset" : { "accuracy" : 0.84, "f1" : 0.2649, "falseNegative" : 283, "falsePositive" : 283, "precision" : 0.2649, "recall" : 0.2649, "total" : 3537, "trueNegative" : 2869, "truePositive" : 102 } }, "thresholdsAt90PrecisionByLanguage" : { }, "thresholdsAt95PrecisionByLanguage" : { } }, { "falseNegativeExamples" : [ { "confidence" : 0.5999, "family" : "online_cped_statement-non-opinion", "id" : "comprehensive-cped-30_071_003", "language" : "zh-Hans", "sourceDataset" : "CPED test split", "text" : "谁说不是啊" }, { "confidence" : 0.5995, "family" : "online_cped_question", "id" : "comprehensive-cped-36_086_003", "language" : "zh-Hans", "sourceDataset" : "CPED test split", "text" : "不是你干的吧" }, { "confidence" : 0.5994, "family" : "online_cped_question", "id" : "comprehensive-cped-35_072_009", "language" : "zh-Hans", "sourceDataset" : "CPED test split", "text" : "你不喜欢小老鼠" }, { "confidence" : 0.5994, "family" : "online_cped_question", "id" : "comprehensive-cped-27_534_000", "language" : "zh-Hans", "sourceDataset" : "CPED test split", "text" : "那是因为您" }, { "confidence" : 0.5994, "family" : "online_cped_question", "id" : "comprehensive-cped-30_081_008", "language" : "zh-Hans", "sourceDataset" : "CPED test split", "text" : "怎么推 柯南看多了吧" } ], "falsePositiveExamples" : [ { "confidence" : 0.9996, "family" : "online_goemotions_reddit", "id" : "comprehensive-goemotions-efficjb", "language" : "en", "sourceDataset" : "Google Research GoEmotions test", "text" : "\"\"\"are you related?\"\" \"\"No\"\" \"\"Are you ?\"\"\"" }, { "confidence" : 0.9996, "family" : "online_enron_intent_pos", "id" : "comprehensive-enron-intent_pos-549", "language" : "en", "sourceDataset" : "Verified Enron Intent Dataset", "text" : "If you happen to be in Houston on November 5th we would enjoy having you join us for the walk." }, { "confidence" : 0.9989, "family" : "online_goemotions_reddit", "id" : "comprehensive-goemotions-eetmkpj", "language" : "en", "sourceDataset" : "Google Research GoEmotions test", "text" : "\"Could you explain what you mean by \"\"both sides\"\" crap?\"" }, { "confidence" : 0.9988, "family" : "online_cped_statement-non-opinion", "id" : "comprehensive-cped-37_157_002", "language" : "zh-Hans", "sourceDataset" : "CPED test split", "text" : "你哥我想帮你想帮你成为咱们家的第一名老大" }, { "confidence" : 0.9987, "family" : "online_cped_statement-non-opinion", "id" : "comprehensive-cped-31_013_001", "language" : "zh-Hans", "sourceDataset" : "CPED test split", "text" : "美嘉 我有话想对你说 我们 我们能不能 能不能别" } ], "id" : "question", "metrics" : { "accuracy" : 0.735, "f1" : 0.4765, "falseNegative" : 4872, "falsePositive" : 6043, "precision" : 0.4512, "recall" : 0.5049, "total" : 41195, "trueNegative" : 25312, "truePositive" : 4968 }, "metricsByLanguage" : { "en" : { "accuracy" : 0.7847, "f1" : 0.4688, "falseNegative" : 1404, "falsePositive" : 1433, "precision" : 0.4663, "recall" : 0.4714, "total" : 13176, "trueNegative" : 9087, "truePositive" : 1252 }, "zh-Hans" : { "accuracy" : 0.7117, "f1" : 0.4792, "falseNegative" : 3468, "falsePositive" : 4610, "precision" : 0.4463, "recall" : 0.5173, "total" : 28019, "trueNegative" : 16225, "truePositive" : 3716 } }, "metricsBySource" : { "AmazonScience/MASSIVE en-US test" : { "accuracy" : 0.5271, "f1" : 0.4132, "falseNegative" : 1122, "falsePositive" : 281, "precision" : 0.6374, "recall" : 0.3057, "total" : 2967, "trueNegative" : 1070, "truePositive" : 494 }, "AmazonScience/MASSIVE zh-CN test" : { "accuracy" : 0.5709, "f1" : 0.4776, "falseNegative" : 1003, "falsePositive" : 220, "precision" : 0.7176, "recall" : 0.3579, "total" : 2850, "trueNegative" : 1068, "truePositive" : 559 }, "CFPB via hpe-ai/customer-complaints" : { "accuracy" : 0.92, "f1" : 0, "falseNegative" : 17, "falsePositive" : 3, "precision" : 0, "recall" : 0, "total" : 250, "trueNegative" : 230, "truePositive" : 0 }, "CPED test split" : { "accuracy" : 0.7277, "f1" : 0.4804, "falseNegative" : 2432, "falsePositive" : 4390, "precision" : 0.4181, "recall" : 0.5646, "total" : 25049, "trueNegative" : 15073, "truePositive" : 3154 }, "Google Research GoEmotions test" : { "accuracy" : 0.8773, "f1" : 0.5063, "falseNegative" : 191, "falsePositive" : 474, "precision" : 0.4184, "recall" : 0.641, "total" : 5418, "trueNegative" : 4412, "truePositive" : 341 }, "Meituan-Dianping/ASAP test" : { "accuracy" : 0.725, "f1" : 0.1538, "falseNegative" : 33, "falsePositive" : 0, "precision" : 1, "recall" : 0.0833, "total" : 120, "trueNegative" : 84, "truePositive" : 3 }, "Sulak2020 GitHub issues multi-repo dataset" : { "accuracy" : 0.7719, "f1" : 0.2776, "falseNegative" : 12, "falsePositive" : 217, "precision" : 0.1686, "recall" : 0.7857, "total" : 1004, "trueNegative" : 731, "truePositive" : 44 }, "Verified Enron Intent Dataset" : { "accuracy" : 0.853, "f1" : 0.5893, "falseNegative" : 62, "falsePositive" : 458, "precision" : 0.4489, "recall" : 0.8575, "total" : 3537, "trueNegative" : 2644, "truePositive" : 373 } }, "thresholdsAt90PrecisionByLanguage" : { }, "thresholdsAt95PrecisionByLanguage" : { } }, { "falseNegativeExamples" : [ { "confidence" : 0.8191, "family" : "online_cped_question", "id" : "comprehensive-cped-30_280_003", "language" : "zh-Hans", "sourceDataset" : "CPED test split", "text" : "要不咱们一起去泰国吃免费的大餐 和晓昭一起" }, { "confidence" : 0.8161, "family" : "online_cped_statement-opinion", "id" : "comprehensive-cped-36_080_020", "language" : "zh-Hans", "sourceDataset" : "CPED test split", "text" : "一起吃一起住" }, { "confidence" : 0.8146, "family" : "online_cped_statement-opinion", "id" : "comprehensive-cped-28_188_011", "language" : "zh-Hans", "sourceDataset" : "CPED test split", "text" : "一起喝盐汽水" }, { "confidence" : 0.8144, "family" : "online_cped_question", "id" : "comprehensive-cped-27_523_002", "language" : "zh-Hans", "sourceDataset" : "CPED test split", "text" : "票你个头加 勒比班白痴的票你要不要" }, { "confidence" : 0.7985, "family" : "online_enron_intent_neg", "id" : "comprehensive-enron-intent_neg-1567", "language" : "en", "sourceDataset" : "Verified Enron Intent Dataset", "text" : "Please read: You are invited to enter the $1,000 Apple Sweepstakes." } ], "falsePositiveExamples" : [ { "confidence" : 0.9865, "family" : "online_cped_statement-non-opinion", "id" : "comprehensive-cped-30_037_000", "language" : "zh-Hans", "sourceDataset" : "CPED test split", "text" : "接下来就让我们一起来回忆一下你上次情绪失控的全过程" }, { "confidence" : 0.9861, "family" : "online_cped_statement-non-opinion", "id" : "comprehensive-cped-29_276_016", "language" : "zh-Hans", "sourceDataset" : "CPED test split", "text" : "你要仔细想一想儿子" }, { "confidence" : 0.9853, "family" : "online_cped_question", "id" : "comprehensive-cped-30_269_007", "language" : "zh-Hans", "sourceDataset" : "CPED test split", "text" : "你是想让我们全部重新来一遍吗" }, { "confidence" : 0.9837, "family" : "online_cped_statement-opinion", "id" : "comprehensive-cped-36_047_008", "language" : "zh-Hans", "sourceDataset" : "CPED test split", "text" : "你就来我们家" }, { "confidence" : 0.9822, "family" : "online_cped_statement-non-opinion", "id" : "comprehensive-cped-28_184_004", "language" : "zh-Hans", "sourceDataset" : "CPED test split", "text" : "我们想用你的名字来命名" } ], "id" : "invitation", "metrics" : { "accuracy" : 0.9813, "f1" : 0.0875, "falseNegative" : 97, "falsePositive" : 675, "precision" : 0.052, "recall" : 0.2761, "total" : 41195, "trueNegative" : 40386, "truePositive" : 37 }, "metricsByLanguage" : { "en" : { "accuracy" : 0.9902, "f1" : 0.1569, "falseNegative" : 49, "falsePositive" : 80, "precision" : 0.1304, "recall" : 0.1967, "total" : 13176, "trueNegative" : 13035, "truePositive" : 12 }, "zh-Hans" : { "accuracy" : 0.9771, "f1" : 0.0722, "falseNegative" : 48, "falsePositive" : 595, "precision" : 0.0403, "recall" : 0.3425, "total" : 28019, "trueNegative" : 27351, "truePositive" : 25 } }, "metricsBySource" : { "AmazonScience/MASSIVE en-US test" : { "accuracy" : 0.9956, "f1" : 0, "falseNegative" : 0, "falsePositive" : 13, "precision" : 0, "recall" : 0, "total" : 2967, "trueNegative" : 2954, "truePositive" : 0 }, "AmazonScience/MASSIVE zh-CN test" : { "accuracy" : 0.9937, "f1" : 0, "falseNegative" : 0, "falsePositive" : 18, "precision" : 0, "recall" : 0, "total" : 2850, "trueNegative" : 2832, "truePositive" : 0 }, "CFPB via hpe-ai/customer-complaints" : { "accuracy" : 1, "f1" : 0, "falseNegative" : 0, "falsePositive" : 0, "precision" : 0, "recall" : 0, "total" : 250, "trueNegative" : 250, "truePositive" : 0 }, "CPED test split" : { "accuracy" : 0.975, "f1" : 0.0741, "falseNegative" : 48, "falsePositive" : 577, "precision" : 0.0415, "recall" : 0.3425, "total" : 25049, "trueNegative" : 24399, "truePositive" : 25 }, "Google Research GoEmotions test" : { "accuracy" : 0.995, "f1" : 0, "falseNegative" : 0, "falsePositive" : 27, "precision" : 0, "recall" : 0, "total" : 5418, "trueNegative" : 5391, "truePositive" : 0 }, "Meituan-Dianping/ASAP test" : { "accuracy" : 1, "f1" : 0, "falseNegative" : 0, "falsePositive" : 0, "precision" : 0, "recall" : 0, "total" : 120, "trueNegative" : 120, "truePositive" : 0 }, "Sulak2020 GitHub issues multi-repo dataset" : { "accuracy" : 0.998, "f1" : 0, "falseNegative" : 0, "falsePositive" : 2, "precision" : 0, "recall" : 0, "total" : 1004, "trueNegative" : 1002, "truePositive" : 0 }, "Verified Enron Intent Dataset" : { "accuracy" : 0.9754, "f1" : 0.2162, "falseNegative" : 49, "falsePositive" : 38, "precision" : 0.24, "recall" : 0.1967, "total" : 3537, "trueNegative" : 3438, "truePositive" : 12 } }, "thresholdAt90Precision" : { "metrics" : { "accuracy" : 0.9968, "f1" : 0.0294, "falseNegative" : 132, "falsePositive" : 0, "precision" : 1, "recall" : 0.0149, "total" : 41195, "trueNegative" : 41061, "truePositive" : 2 }, "threshold" : 0.99 }, "thresholdAt95Precision" : { "metrics" : { "accuracy" : 0.9968, "f1" : 0.0294, "falseNegative" : 132, "falsePositive" : 0, "precision" : 1, "recall" : 0.0149, "total" : 41195, "trueNegative" : 41061, "truePositive" : 2 }, "threshold" : 0.99 }, "thresholdsAt90PrecisionByLanguage" : { "en" : { "metrics" : { "accuracy" : 0.9955, "f1" : 0.0635, "falseNegative" : 59, "falsePositive" : 0, "precision" : 1, "recall" : 0.0328, "total" : 13176, "trueNegative" : 13115, "truePositive" : 2 }, "threshold" : 0.98 }, "zh-Hans" : { "metrics" : { "accuracy" : 0.9974, "f1" : 0.027, "falseNegative" : 72, "falsePositive" : 0, "precision" : 1, "recall" : 0.0137, "total" : 28019, "trueNegative" : 27946, "truePositive" : 1 }, "threshold" : 0.99 } }, "thresholdsAt95PrecisionByLanguage" : { "en" : { "metrics" : { "accuracy" : 0.9955, "f1" : 0.0635, "falseNegative" : 59, "falsePositive" : 0, "precision" : 1, "recall" : 0.0328, "total" : 13176, "trueNegative" : 13115, "truePositive" : 2 }, "threshold" : 0.98 }, "zh-Hans" : { "metrics" : { "accuracy" : 0.9974, "f1" : 0.027, "falseNegative" : 72, "falsePositive" : 0, "precision" : 1, "recall" : 0.0137, "total" : 28019, "trueNegative" : 27946, "truePositive" : 1 }, "threshold" : 0.99 } } }, { "falseNegativeExamples" : [ { "confidence" : 0.838, "family" : "online_github_issue_title", "id" : "comprehensive-github-1833854030", "language" : "en", "sourceDataset" : "Sulak2020 GitHub issues multi-repo dataset", "text" : "load_dataset() fails with streamlit caching inside docker" }, { "confidence" : 0.838, "family" : "online_github_issue_title", "id" : "comprehensive-github-3218044656", "language" : "en", "sourceDataset" : "Sulak2020 GitHub issues multi-repo dataset", "text" : "Toxicity fails with datasets 4.0.0" }, { "confidence" : 0.8218, "family" : "online_github_issue_title", "id" : "comprehensive-github-2052332528", "language" : "en", "sourceDataset" : "Sulak2020 GitHub issues multi-repo dataset", "text" : "Loading HF Hub Dataset (private org repo) fails to load all features" }, { "confidence" : 0.8148, "family" : "online_cped_question", "id" : "comprehensive-cped-37_041_002", "language" : "zh-Hans", "sourceDataset" : "CPED test split", "text" : "怎么还不睡" }, { "confidence" : 0.8128, "family" : "online_cped_statement-non-opinion", "id" : "comprehensive-cped-37_050_002", "language" : "zh-Hans", "sourceDataset" : "CPED test split", "text" : "你们昨天又考了一次试" } ], "falsePositiveExamples" : [ { "confidence" : 0.9871, "family" : "online_cped_statement-non-opinion", "id" : "comprehensive-cped-27_450_006", "language" : "zh-Hans", "sourceDataset" : "CPED test split", "text" : "他为了等我一直到现在都还未婚" }, { "confidence" : 0.9869, "family" : "online_cped_statement-non-opinion", "id" : "comprehensive-cped-30_098_003", "language" : "zh-Hans", "sourceDataset" : "CPED test split", "text" : "从高中到现在有好多好多年了" }, { "confidence" : 0.9814, "family" : "online_massive_intent_44", "id" : "comprehensive-massive-zh-CN-16668", "language" : "zh-Hans", "sourceDataset" : "AmazonScience/MASSIVE zh-CN test", "text" : "从今天四点到现在收到任何邮件了吗" }, { "confidence" : 0.9813, "family" : "online_cped_statement-non-opinion", "id" : "comprehensive-cped-29_076_011", "language" : "zh-Hans", "sourceDataset" : "CPED test split", "text" : "我到现在还没有" }, { "confidence" : 0.9804, "family" : "online_cped_statement-non-opinion", "id" : "comprehensive-cped-34_326_000", "language" : "zh-Hans", "sourceDataset" : "CPED test split", "text" : "我都看见了 开大奔那个人来送你 那样儿还 还不错" } ], "id" : "complaint", "metrics" : { "accuracy" : 0.9627, "f1" : 0.0376, "falseNegative" : 674, "falsePositive" : 861, "precision" : 0.0337, "recall" : 0.0426, "total" : 41195, "trueNegative" : 39630, "truePositive" : 30 }, "metricsByLanguage" : { "en" : { "accuracy" : 0.9572, "f1" : 0.0662, "falseNegative" : 414, "falsePositive" : 150, "precision" : 0.1176, "recall" : 0.0461, "total" : 13176, "trueNegative" : 12592, "truePositive" : 20 }, "zh-Hans" : { "accuracy" : 0.9653, "f1" : 0.0202, "falseNegative" : 260, "falsePositive" : 711, "precision" : 0.0139, "recall" : 0.037, "total" : 28019, "trueNegative" : 27038, "truePositive" : 10 } }, "metricsBySource" : { "AmazonScience/MASSIVE en-US test" : { "accuracy" : 0.9919, "f1" : 0, "falseNegative" : 0, "falsePositive" : 24, "precision" : 0, "recall" : 0, "total" : 2967, "trueNegative" : 2943, "truePositive" : 0 }, "AmazonScience/MASSIVE zh-CN test" : { "accuracy" : 0.9909, "f1" : 0, "falseNegative" : 0, "falsePositive" : 26, "precision" : 0, "recall" : 0, "total" : 2850, "trueNegative" : 2824, "truePositive" : 0 }, "CFPB via hpe-ai/customer-complaints" : { "accuracy" : 0.052, "f1" : 0.0989, "falseNegative" : 237, "falsePositive" : 0, "precision" : 1, "recall" : 0.052, "total" : 250, "trueNegative" : 0, "truePositive" : 13 }, "CPED test split" : { "accuracy" : 0.9669, "f1" : 0.0143, "falseNegative" : 144, "falsePositive" : 685, "precision" : 0.0087, "recall" : 0.04, "total" : 25049, "trueNegative" : 24214, "truePositive" : 6 }, "Google Research GoEmotions test" : { "accuracy" : 0.9819, "f1" : 0.02, "falseNegative" : 19, "falsePositive" : 79, "precision" : 0.0125, "recall" : 0.05, "total" : 5418, "trueNegative" : 5319, "truePositive" : 1 }, "Meituan-Dianping/ASAP test" : { "accuracy" : 0.0333, "f1" : 0.0645, "falseNegative" : 116, "falsePositive" : 0, "precision" : 1, "recall" : 0.0333, "total" : 120, "trueNegative" : 0, "truePositive" : 4 }, "Sulak2020 GitHub issues multi-repo dataset" : { "accuracy" : 0.8177, "f1" : 0.0615, "falseNegative" : 158, "falsePositive" : 25, "precision" : 0.1935, "recall" : 0.0366, "total" : 1004, "trueNegative" : 815, "truePositive" : 6 }, "Verified Enron Intent Dataset" : { "accuracy" : 0.9938, "f1" : 0, "falseNegative" : 0, "falsePositive" : 22, "precision" : 0, "recall" : 0, "total" : 3537, "trueNegative" : 3515, "truePositive" : 0 } }, "thresholdAt90Precision" : { "metrics" : { "accuracy" : 0.9829, "f1" : 0.0028, "falseNegative" : 703, "falsePositive" : 0, "precision" : 1, "recall" : 0.0014, "total" : 41195, "trueNegative" : 40491, "truePositive" : 1 }, "threshold" : 0.99 }, "thresholdAt95Precision" : { "metrics" : { "accuracy" : 0.9829, "f1" : 0.0028, "falseNegative" : 703, "falsePositive" : 0, "precision" : 1, "recall" : 0.0014, "total" : 41195, "trueNegative" : 40491, "truePositive" : 1 }, "threshold" : 0.99 }, "thresholdsAt90PrecisionByLanguage" : { "en" : { "metrics" : { "accuracy" : 0.9671, "f1" : 0.0046, "falseNegative" : 433, "falsePositive" : 0, "precision" : 1, "recall" : 0.0023, "total" : 13176, "trueNegative" : 12742, "truePositive" : 1 }, "threshold" : 0.99 } }, "thresholdsAt95PrecisionByLanguage" : { "en" : { "metrics" : { "accuracy" : 0.9671, "f1" : 0.0046, "falseNegative" : 433, "falsePositive" : 0, "precision" : 1, "recall" : 0.0023, "total" : 13176, "trueNegative" : 12742, "truePositive" : 1 }, "threshold" : 0.99 } } }, { "falseNegativeExamples" : [ { "confidence" : 0.6664, "family" : "online_github_issue_title", "id" : "comprehensive-github-2089230766", "language" : "en", "sourceDataset" : "Sulak2020 GitHub issues multi-repo dataset", "text" : "datasets map `cache_file_name` does not work" }, { "confidence" : 0.6664, "family" : "online_github_issue_title", "id" : "comprehensive-github-2103780697", "language" : "en", "sourceDataset" : "Sulak2020 GitHub issues multi-repo dataset", "text" : "multi-GPU map does not work" }, { "confidence" : 0.6664, "family" : "online_github_issue_title", "id" : "comprehensive-github-3503446288", "language" : "en", "sourceDataset" : "Sulak2020 GitHub issues multi-repo dataset", "text" : "Caching does not work when using python3.14" }, { "confidence" : 0.663, "family" : "online_enron_intent_pos", "id" : "comprehensive-enron-intent_pos-452", "language" : "en", "sourceDataset" : "Verified Enron Intent Dataset", "text" : "I would be happy to reschedule at your team's convenience." }, { "confidence" : 0.6163, "family" : "online_github_issue_title", "id" : "comprehensive-github-2069922151", "language" : "en", "sourceDataset" : "Sulak2020 GitHub issues multi-repo dataset", "text" : "keep_in_memory=True does not seem to work" } ], "falsePositiveExamples" : [ { "confidence" : 0.974, "family" : "online_massive_intent_48", "id" : "comprehensive-massive-zh-CN-612", "language" : "zh-Hans", "sourceDataset" : "AmazonScience/MASSIVE zh-CN test", "text" : "下个周六早上八点起床闹钟响铃" }, { "confidence" : 0.9567, "family" : "online_massive_intent_50", "id" : "comprehensive-massive-en-US-7291", "language" : "en", "sourceDataset" : "AmazonScience/MASSIVE en-US test", "text" : "block one hour from ten tomorrow morning" }, { "confidence" : 0.9543, "family" : "online_enron_intent_pos", "id" : "comprehensive-enron-intent_pos-714", "language" : "en", "sourceDataset" : "Verified Enron Intent Dataset", "text" : "Maybe on Ben's layover we can have a discussion about where we go from here from a schedule standpoint." }, { "confidence" : 0.9537, "family" : "online_goemotions_reddit", "id" : "comprehensive-goemotions-edxtibo", "language" : "en", "sourceDataset" : "Google Research GoEmotions test", "text" : "Funny how we get more anxious about our heroes than their victims" }, { "confidence" : 0.9443, "family" : "online_goemotions_reddit", "id" : "comprehensive-goemotions-edhb7vy", "language" : "en", "sourceDataset" : "Google Research GoEmotions test", "text" : "Great! I hope we can make our loan system work better" } ], "id" : "scheduleNegotiation", "metrics" : { "accuracy" : 0.928, "f1" : 0.0074, "falseNegative" : 60, "falsePositive" : 2908, "precision" : 0.0038, "recall" : 0.1549, "total" : 41195, "trueNegative" : 38216, "truePositive" : 11 }, "metricsByLanguage" : { "en" : { "accuracy" : 0.9746, "f1" : 0.0616, "falseNegative" : 56, "falsePositive" : 279, "precision" : 0.0379, "recall" : 0.1642, "total" : 13176, "trueNegative" : 12830, "truePositive" : 11 }, "zh-Hans" : { "accuracy" : 0.906, "f1" : 0, "falseNegative" : 4, "falsePositive" : 2629, "precision" : 0, "recall" : 0, "total" : 28019, "trueNegative" : 25386, "truePositive" : 0 } }, "metricsBySource" : { "AmazonScience/MASSIVE en-US test" : { "accuracy" : 0.9717, "f1" : 0, "falseNegative" : 0, "falsePositive" : 84, "precision" : 0, "recall" : 0, "total" : 2967, "trueNegative" : 2883, "truePositive" : 0 }, "AmazonScience/MASSIVE zh-CN test" : { "accuracy" : 0.8439, "f1" : 0, "falseNegative" : 0, "falsePositive" : 445, "precision" : 0, "recall" : 0, "total" : 2850, "trueNegative" : 2405, "truePositive" : 0 }, "CFPB via hpe-ai/customer-complaints" : { "accuracy" : 1, "f1" : 0, "falseNegative" : 0, "falsePositive" : 0, "precision" : 0, "recall" : 0, "total" : 250, "trueNegative" : 250, "truePositive" : 0 }, "CPED test split" : { "accuracy" : 0.9127, "f1" : 0, "falseNegative" : 4, "falsePositive" : 2184, "precision" : 0, "recall" : 0, "total" : 25049, "trueNegative" : 22861, "truePositive" : 0 }, "Google Research GoEmotions test" : { "accuracy" : 0.9836, "f1" : 0, "falseNegative" : 0, "falsePositive" : 89, "precision" : 0, "recall" : 0, "total" : 5418, "trueNegative" : 5329, "truePositive" : 0 }, "Meituan-Dianping/ASAP test" : { "accuracy" : 1, "f1" : 0, "falseNegative" : 0, "falsePositive" : 0, "precision" : 0, "recall" : 0, "total" : 120, "trueNegative" : 120, "truePositive" : 0 }, "Sulak2020 GitHub issues multi-repo dataset" : { "accuracy" : 0.9373, "f1" : 0.0308, "falseNegative" : 24, "falsePositive" : 39, "precision" : 0.025, "recall" : 0.04, "total" : 1004, "trueNegative" : 940, "truePositive" : 1 }, "Verified Enron Intent Dataset" : { "accuracy" : 0.972, "f1" : 0.1681, "falseNegative" : 32, "falsePositive" : 67, "precision" : 0.1299, "recall" : 0.2381, "total" : 3537, "trueNegative" : 3428, "truePositive" : 10 } }, "thresholdAt90Precision" : { "metrics" : { "accuracy" : 0.9983, "f1" : 0.0278, "falseNegative" : 70, "falsePositive" : 0, "precision" : 1, "recall" : 0.0141, "total" : 41195, "trueNegative" : 41124, "truePositive" : 1 }, "threshold" : 0.98 }, "thresholdAt95Precision" : { "metrics" : { "accuracy" : 0.9983, "f1" : 0.0278, "falseNegative" : 70, "falsePositive" : 0, "precision" : 1, "recall" : 0.0141, "total" : 41195, "trueNegative" : 41124, "truePositive" : 1 }, "threshold" : 0.98 }, "thresholdsAt90PrecisionByLanguage" : { "en" : { "metrics" : { "accuracy" : 0.9952, "f1" : 0.1127, "falseNegative" : 63, "falsePositive" : 0, "precision" : 1, "recall" : 0.0597, "total" : 13176, "trueNegative" : 13109, "truePositive" : 4 }, "threshold" : 0.96 } }, "thresholdsAt95PrecisionByLanguage" : { "en" : { "metrics" : { "accuracy" : 0.9952, "f1" : 0.1127, "falseNegative" : 63, "falsePositive" : 0, "precision" : 1, "recall" : 0.0597, "total" : 13176, "trueNegative" : 13109, "truePositive" : 4 }, "threshold" : 0.96 } } }, { "falseNegativeExamples" : [ { "confidence" : 0.6969, "family" : "online_cped_statement-non-opinion", "id" : "comprehensive-cped-30_028_000", "language" : "zh-Hans", "sourceDataset" : "CPED test split", "text" : "不是我选的 是硬币选的" }, { "confidence" : 0.6719, "family" : "online_cped_statement-opinion", "id" : "comprehensive-cped-31_099_005", "language" : "zh-Hans", "sourceDataset" : "CPED test split", "text" : "我就同意收留它 怎么样" }, { "confidence" : 0.6593, "family" : "online_cped_statement-non-opinion", "id" : "comprehensive-cped-27_514_003", "language" : "zh-Hans", "sourceDataset" : "CPED test split", "text" : "我选了宛瑜做的" }, { "confidence" : 0.6522, "family" : "online_cped_statement-opinion", "id" : "comprehensive-cped-27_121_009", "language" : "zh-Hans", "sourceDataset" : "CPED test split", "text" : "所以我选择了爱情公寓" }, { "confidence" : 0.6295, "family" : "online_cped_statement-non-opinion", "id" : "comprehensive-cped-29_393_000", "language" : "zh-Hans", "sourceDataset" : "CPED test split", "text" : "关谷你就按照我说的 摆出绝不让步的态度" } ], "falsePositiveExamples" : [ { "confidence" : 0.972, "family" : "online_enron_intent_neg", "id" : "comprehensive-enron-intent_neg-1619", "language" : "en", "sourceDataset" : "Verified Enron Intent Dataset", "text" : "Pretty much as soon as I can get all my shit together." }, { "confidence" : 0.9659, "family" : "online_massive_intent_43", "id" : "comprehensive-massive-en-US-3819", "language" : "en", "sourceDataset" : "AmazonScience/MASSIVE en-US test", "text" : "please rate current song as five stars" }, { "confidence" : 0.9621, "family" : "online_enron_intent_neg", "id" : "comprehensive-enron-intent_neg-586", "language" : "en", "sourceDataset" : "Verified Enron Intent Dataset", "text" : "I can make other days work as well." }, { "confidence" : 0.9613, "family" : "online_enron_intent_neg", "id" : "comprehensive-enron-intent_neg-2151", "language" : "en", "sourceDataset" : "Verified Enron Intent Dataset", "text" : "We can make the salary retroactive as part of the contract." }, { "confidence" : 0.9488, "family" : "online_enron_intent_neg", "id" : "comprehensive-enron-intent_neg-1371", "language" : "en", "sourceDataset" : "Verified Enron Intent Dataset", "text" : "My schedule is extremely flexible, and I would like to start working as soon as possible." } ], "id" : "confirmationDecision", "metrics" : { "accuracy" : 0.9824, "f1" : 0.0189, "falseNegative" : 62, "falsePositive" : 664, "precision" : 0.0104, "recall" : 0.1014, "total" : 41195, "trueNegative" : 40462, "truePositive" : 7 }, "metricsByLanguage" : { "en" : { "accuracy" : 0.9802, "f1" : 0, "falseNegative" : 12, "falsePositive" : 249, "precision" : 0, "recall" : 0, "total" : 13176, "trueNegative" : 12915, "truePositive" : 0 }, "zh-Hans" : { "accuracy" : 0.9834, "f1" : 0.0292, "falseNegative" : 50, "falsePositive" : 415, "precision" : 0.0166, "recall" : 0.1228, "total" : 28019, "trueNegative" : 27547, "truePositive" : 7 } }, "metricsBySource" : { "AmazonScience/MASSIVE en-US test" : { "accuracy" : 0.9835, "f1" : 0, "falseNegative" : 0, "falsePositive" : 49, "precision" : 0, "recall" : 0, "total" : 2967, "trueNegative" : 2918, "truePositive" : 0 }, "AmazonScience/MASSIVE zh-CN test" : { "accuracy" : 0.986, "f1" : 0, "falseNegative" : 0, "falsePositive" : 40, "precision" : 0, "recall" : 0, "total" : 2850, "trueNegative" : 2810, "truePositive" : 0 }, "CFPB via hpe-ai/customer-complaints" : { "accuracy" : 0.984, "f1" : 0, "falseNegative" : 0, "falsePositive" : 4, "precision" : 0, "recall" : 0, "total" : 250, "trueNegative" : 246, "truePositive" : 0 }, "CPED test split" : { "accuracy" : 0.983, "f1" : 0.0319, "falseNegative" : 50, "falsePositive" : 375, "precision" : 0.0183, "recall" : 0.1228, "total" : 25049, "trueNegative" : 24617, "truePositive" : 7 }, "Google Research GoEmotions test" : { "accuracy" : 0.978, "f1" : 0, "falseNegative" : 0, "falsePositive" : 119, "precision" : 0, "recall" : 0, "total" : 5418, "trueNegative" : 5299, "truePositive" : 0 }, "Meituan-Dianping/ASAP test" : { "accuracy" : 1, "f1" : 0, "falseNegative" : 0, "falsePositive" : 0, "precision" : 0, "recall" : 0, "total" : 120, "trueNegative" : 120, "truePositive" : 0 }, "Sulak2020 GitHub issues multi-repo dataset" : { "accuracy" : 0.9801, "f1" : 0, "falseNegative" : 2, "falsePositive" : 18, "precision" : 0, "recall" : 0, "total" : 1004, "trueNegative" : 984, "truePositive" : 0 }, "Verified Enron Intent Dataset" : { "accuracy" : 0.9805, "f1" : 0, "falseNegative" : 10, "falsePositive" : 59, "precision" : 0, "recall" : 0, "total" : 3537, "trueNegative" : 3468, "truePositive" : 0 } }, "thresholdAt90Precision" : { "metrics" : { "accuracy" : 0.9984, "f1" : 0.0563, "falseNegative" : 67, "falsePositive" : 0, "precision" : 1, "recall" : 0.029, "total" : 41195, "trueNegative" : 41126, "truePositive" : 2 }, "threshold" : 0.98 }, "thresholdAt95Precision" : { "metrics" : { "accuracy" : 0.9984, "f1" : 0.0563, "falseNegative" : 67, "falsePositive" : 0, "precision" : 1, "recall" : 0.029, "total" : 41195, "trueNegative" : 41126, "truePositive" : 2 }, "threshold" : 0.98 }, "thresholdsAt90PrecisionByLanguage" : { "zh-Hans" : { "metrics" : { "accuracy" : 0.998, "f1" : 0.0678, "falseNegative" : 55, "falsePositive" : 0, "precision" : 1, "recall" : 0.0351, "total" : 28019, "trueNegative" : 27962, "truePositive" : 2 }, "threshold" : 0.95 } }, "thresholdsAt95PrecisionByLanguage" : { "zh-Hans" : { "metrics" : { "accuracy" : 0.998, "f1" : 0.0678, "falseNegative" : 55, "falsePositive" : 0, "precision" : 1, "recall" : 0.0351, "total" : 28019, "trueNegative" : 27962, "truePositive" : 2 }, "threshold" : 0.95 } } }, { "falseNegativeExamples" : [ { "confidence" : 0.728, "family" : "online_massive_intent_59", "id" : "comprehensive-massive-zh-CN-10699", "language" : "zh-Hans", "sourceDataset" : "AmazonScience/MASSIVE zh-CN test", "text" : "提醒我清单上有什么项目" }, { "confidence" : 0.7253, "family" : "online_cped_statement-non-opinion", "id" : "comprehensive-cped-29_166_002", "language" : "zh-Hans", "sourceDataset" : "CPED test split", "text" : "不过展博倒是提醒我了" }, { "confidence" : 0.7241, "family" : "online_massive_intent_50", "id" : "comprehensive-massive-zh-CN-7916", "language" : "zh-Hans", "sourceDataset" : "AmazonScience/MASSIVE zh-CN test", "text" : "提醒我在下午六点倒垃圾" }, { "confidence" : 0.7241, "family" : "online_massive_intent_50", "id" : "comprehensive-massive-zh-CN-8598", "language" : "zh-Hans", "sourceDataset" : "AmazonScience/MASSIVE zh-CN test", "text" : "提醒我下午六点在机场接迈克" }, { "confidence" : 0.7194, "family" : "online_enron_intent_pos", "id" : "comprehensive-enron-intent_pos-1103", "language" : "en", "sourceDataset" : "Verified Enron Intent Dataset", "text" : "We shall hold an internal follow-up meeting next Monday to review potential projects on which we could cooperate in the future." } ], "falsePositiveExamples" : [ { "confidence" : 0.9997, "family" : "online_cped_disagreement", "id" : "comprehensive-cped-34_224_008", "language" : "zh-Hans", "sourceDataset" : "CPED test split", "text" : "别过两天过两天就晚了" }, { "confidence" : 0.9981, "family" : "online_cped_disagreement", "id" : "comprehensive-cped-37_151_009", "language" : "zh-Hans", "sourceDataset" : "CPED test split", "text" : "别别别别别 咱们俩先按兵不动" }, { "confidence" : 0.9962, "family" : "online_cped_statement-non-opinion", "id" : "comprehensive-cped-34_117_007", "language" : "zh-Hans", "sourceDataset" : "CPED test split", "text" : "别别别 咱一码归一码" }, { "confidence" : 0.9959, "family" : "online_massive_intent_50", "id" : "comprehensive-massive-en-US-6902", "language" : "en", "sourceDataset" : "AmazonScience/MASSIVE en-US test", "text" : "set a reminder up next month to get my oil changed" }, { "confidence" : 0.9958, "family" : "online_enron_intent_pos", "id" : "comprehensive-enron-intent_pos-492", "language" : "en", "sourceDataset" : "Verified Enron Intent Dataset", "text" : "I'm going to Las Vegas next week, so we'll have to get together some time after next week." } ], "id" : "followUpReminder", "metrics" : { "accuracy" : 0.9668, "f1" : 0.1502, "falseNegative" : 269, "falsePositive" : 1100, "precision" : 0.0991, "recall" : 0.3103, "total" : 41195, "trueNegative" : 39705, "truePositive" : 121 }, "metricsByLanguage" : { "en" : { "accuracy" : 0.9615, "f1" : 0.0523, "falseNegative" : 111, "falsePositive" : 396, "precision" : 0.0341, "recall" : 0.112, "total" : 13176, "trueNegative" : 12655, "truePositive" : 14 }, "zh-Hans" : { "accuracy" : 0.9692, "f1" : 0.1989, "falseNegative" : 158, "falsePositive" : 704, "precision" : 0.1319, "recall" : 0.4038, "total" : 28019, "trueNegative" : 27050, "truePositive" : 107 } }, "metricsBySource" : { "AmazonScience/MASSIVE en-US test" : { "accuracy" : 0.8992, "f1" : 0.08, "falseNegative" : 84, "falsePositive" : 215, "precision" : 0.057, "recall" : 0.134, "total" : 2967, "trueNegative" : 2655, "truePositive" : 13 }, "AmazonScience/MASSIVE zh-CN test" : { "accuracy" : 0.9368, "f1" : 0.5, "falseNegative" : 76, "falsePositive" : 104, "precision" : 0.4639, "recall" : 0.5422, "total" : 2850, "trueNegative" : 2580, "truePositive" : 90 }, "CFPB via hpe-ai/customer-complaints" : { "accuracy" : 1, "f1" : 0, "falseNegative" : 0, "falsePositive" : 0, "precision" : 0, "recall" : 0, "total" : 250, "trueNegative" : 250, "truePositive" : 0 }, "CPED test split" : { "accuracy" : 0.9728, "f1" : 0.0475, "falseNegative" : 82, "falsePositive" : 600, "precision" : 0.0276, "recall" : 0.1717, "total" : 25049, "trueNegative" : 24350, "truePositive" : 17 }, "Google Research GoEmotions test" : { "accuracy" : 0.9893, "f1" : 0, "falseNegative" : 0, "falsePositive" : 58, "precision" : 0, "recall" : 0, "total" : 5418, "trueNegative" : 5360, "truePositive" : 0 }, "Meituan-Dianping/ASAP test" : { "accuracy" : 1, "f1" : 0, "falseNegative" : 0, "falsePositive" : 0, "precision" : 0, "recall" : 0, "total" : 120, "trueNegative" : 120, "truePositive" : 0 }, "Sulak2020 GitHub issues multi-repo dataset" : { "accuracy" : 0.9771, "f1" : 0, "falseNegative" : 2, "falsePositive" : 21, "precision" : 0, "recall" : 0, "total" : 1004, "trueNegative" : 981, "truePositive" : 0 }, "Verified Enron Intent Dataset" : { "accuracy" : 0.9641, "f1" : 0.0155, "falseNegative" : 25, "falsePositive" : 102, "precision" : 0.0097, "recall" : 0.0385, "total" : 3537, "trueNegative" : 3409, "truePositive" : 1 } }, "thresholdsAt90PrecisionByLanguage" : { }, "thresholdsAt95PrecisionByLanguage" : { } }, { "falseNegativeExamples" : [ { "confidence" : 0.7899, "family" : "online_cped_statement-non-opinion", "id" : "comprehensive-cped-27_272_006", "language" : "zh-Hans", "sourceDataset" : "CPED test split", "text" : "而且学富五车还是中国第8代导演" }, { "confidence" : 0.7899, "family" : "online_cped_statement-non-opinion", "id" : "comprehensive-cped-34_369_001", "language" : "zh-Hans", "sourceDataset" : "CPED test split", "text" : "这招一出呢 你们家那些乱七八糟来相亲的" }, { "confidence" : 0.7899, "family" : "online_cped_statement-non-opinion", "id" : "comprehensive-cped-29_228_001", "language" : "zh-Hans", "sourceDataset" : "CPED test split", "text" : "我搞错了" }, { "confidence" : 0.7899, "family" : "online_cped_statement-opinion", "id" : "comprehensive-cped-31_129_007", "language" : "zh-Hans", "sourceDataset" : "CPED test split", "text" : "姑奶奶我错了" }, { "confidence" : 0.7898, "family" : "online_cped_question", "id" : "comprehensive-cped-27_130_012", "language" : "zh-Hans", "sourceDataset" : "CPED test split", "text" : "你知道吗" } ], "falsePositiveExamples" : [ { "confidence" : 1, "family" : "online_enron_intent_neg", "id" : "comprehensive-enron-intent_neg-2179", "language" : "en", "sourceDataset" : "Verified Enron Intent Dataset", "text" : "We have some grand ideas and need only your name and influence to make this a successful evening for TUTS!You and your wife may be as involved or not involved as you like!We would like to top the funds raised by this current year's event that was Co -Chaired by Chuck and Kim Watson .They helped TUTS to raise over $350,000." }, { "confidence" : 1, "family" : "online_enron_intent_neg", "id" : "comprehensive-enron-intent_neg-1396", "language" : "en", "sourceDataset" : "Verified Enron Intent Dataset", "text" : "Now let me tell you why this medallion and Archangel Michael, the most powerful of angels, can deliver the miracles, protection, guidance and love you need to change your life so you can live your dreams." }, { "confidence" : 1, "family" : "online_enron_intent_neg", "id" : "comprehensive-enron-intent_neg-2082", "language" : "en", "sourceDataset" : "Verified Enron Intent Dataset", "text" : "Two action items: a) Print out for John - Commercial Income & Summary worksheets for his comment and review & b) Start putting completed commercial packages together for distribution/further comment by the business units in the next couple of weeks - include detailed income statement, headcount, balance sheet (including expected increases,decreases, capital charge and depreciation), leave pages open for goals & objectives and target customer list." }, { "confidence" : 1, "family" : "online_enron_intent_neg", "id" : "comprehensive-enron-intent_neg-1740", "language" : "en", "sourceDataset" : "Verified Enron Intent Dataset", "text" : "Specifically, we buy/sell gas at the Exxon Katy Plant and were wondering the \"Force Majeure\" ramifications of the the Plant closing due to freeze-offs, i.e. can we claim Force Majeure in that instance and what remedies are provided for in the EOL contract language?" }, { "confidence" : 1, "family" : "online_enron_intent_neg", "id" : "comprehensive-enron-intent_neg-1741", "language" : "en", "sourceDataset" : "Verified Enron Intent Dataset", "text" : "Specifically, we buy/sell gas at the Exxon Katy Plant and were wondering the Force Majeure ramifications of the the Plant closing due to freeze-offs, i.e. can we claim Force Majeure in that instance and what remedies are provided for in the EOL contract language?" } ], "id" : "replyableMessage", "metrics" : { "accuracy" : 0.4626, "f1" : 0.6073, "falseNegative" : 20191, "falsePositive" : 1946, "precision" : 0.8979, "recall" : 0.4588, "total" : 41195, "trueNegative" : 1942, "truePositive" : 17116 }, "metricsByLanguage" : { "en" : { "accuracy" : 0.4794, "f1" : 0.6205, "falseNegative" : 5273, "falsePositive" : 1587, "precision" : 0.7794, "recall" : 0.5154, "total" : 13176, "trueNegative" : 708, "truePositive" : 5608 }, "zh-Hans" : { "accuracy" : 0.4548, "f1" : 0.601, "falseNegative" : 14918, "falsePositive" : 359, "precision" : 0.9697, "recall" : 0.4355, "total" : 28019, "trueNegative" : 1234, "truePositive" : 11508 } }, "metricsBySource" : { "AmazonScience/MASSIVE en-US test" : { "accuracy" : 0.4611, "f1" : 0.6311, "falseNegative" : 1599, "falsePositive" : 0, "precision" : 1, "recall" : 0.4611, "total" : 2967, "trueNegative" : 0, "truePositive" : 1368 }, "AmazonScience/MASSIVE zh-CN test" : { "accuracy" : 0.3396, "f1" : 0.5071, "falseNegative" : 1882, "falsePositive" : 0, "precision" : 1, "recall" : 0.3396, "total" : 2850, "trueNegative" : 0, "truePositive" : 968 }, "CFPB via hpe-ai/customer-complaints" : { "accuracy" : 0.668, "f1" : 0.801, "falseNegative" : 83, "falsePositive" : 0, "precision" : 1, "recall" : 0.668, "total" : 250, "trueNegative" : 0, "truePositive" : 167 }, "CPED test split" : { "accuracy" : 0.4657, "f1" : 0.6092, "falseNegative" : 13025, "falsePositive" : 359, "precision" : 0.9667, "recall" : 0.4447, "total" : 25049, "trueNegative" : 1234, "truePositive" : 10431 }, "Google Research GoEmotions test" : { "accuracy" : 0.5201, "f1" : 0.6843, "falseNegative" : 2600, "falsePositive" : 0, "precision" : 1, "recall" : 0.5201, "total" : 5418, "trueNegative" : 0, "truePositive" : 2818 }, "Meituan-Dianping/ASAP test" : { "accuracy" : 0.9083, "f1" : 0.952, "falseNegative" : 11, "falsePositive" : 0, "precision" : 1, "recall" : 0.9083, "total" : 120, "trueNegative" : 0, "truePositive" : 109 }, "Sulak2020 GitHub issues multi-repo dataset" : { "accuracy" : 0.2898, "f1" : 0.4494, "falseNegative" : 713, "falsePositive" : 0, "precision" : 1, "recall" : 0.2898, "total" : 1004, "trueNegative" : 0, "truePositive" : 291 }, "Verified Enron Intent Dataset" : { "accuracy" : 0.4727, "f1" : 0.5083, "falseNegative" : 278, "falsePositive" : 1587, "precision" : 0.3779, "recall" : 0.7762, "total" : 3537, "trueNegative" : 708, "truePositive" : 964 } }, "thresholdAt90Precision" : { "metrics" : { "accuracy" : 0.8901, "f1" : 0.9417, "falseNegative" : 707, "falsePositive" : 3821, "precision" : 0.9055, "recall" : 0.981, "total" : 41195, "trueNegative" : 67, "truePositive" : 36600 }, "threshold" : 0.05 }, "thresholdsAt90PrecisionByLanguage" : { "zh-Hans" : { "metrics" : { "accuracy" : 0.9275, "f1" : 0.9623, "falseNegative" : 458, "falsePositive" : 1574, "precision" : 0.9429, "recall" : 0.9827, "total" : 28019, "trueNegative" : 19, "truePositive" : 25968 }, "threshold" : 0.05 } }, "thresholdsAt95PrecisionByLanguage" : { "zh-Hans" : { "metrics" : { "accuracy" : 0.6745, "f1" : 0.7981, "falseNegative" : 8398, "falsePositive" : 721, "precision" : 0.9615, "recall" : 0.6822, "total" : 28019, "trueNegative" : 872, "truePositive" : 18028 }, "threshold" : 0.51 } } }, { "falseNegativeExamples" : [ ], "falsePositiveExamples" : [ ], "id" : "blessing", "metrics" : { "accuracy" : 1, "f1" : 1, "falseNegative" : 0, "falsePositive" : 0, "precision" : 1, "recall" : 1, "total" : 41195, "trueNegative" : 41109, "truePositive" : 86 }, "metricsByLanguage" : { "en" : { "accuracy" : 1, "f1" : 1, "falseNegative" : 0, "falsePositive" : 0, "precision" : 1, "recall" : 1, "total" : 13176, "trueNegative" : 13107, "truePositive" : 69 }, "zh-Hans" : { "accuracy" : 1, "f1" : 1, "falseNegative" : 0, "falsePositive" : 0, "precision" : 1, "recall" : 1, "total" : 28019, "trueNegative" : 28002, "truePositive" : 17 } }, "metricsBySource" : { "AmazonScience/MASSIVE en-US test" : { "accuracy" : 1, "f1" : 0, "falseNegative" : 0, "falsePositive" : 0, "precision" : 0, "recall" : 0, "total" : 2967, "trueNegative" : 2967, "truePositive" : 0 }, "AmazonScience/MASSIVE zh-CN test" : { "accuracy" : 1, "f1" : 0, "falseNegative" : 0, "falsePositive" : 0, "precision" : 0, "recall" : 0, "total" : 2850, "trueNegative" : 2850, "truePositive" : 0 }, "CFPB via hpe-ai/customer-complaints" : { "accuracy" : 1, "f1" : 0, "falseNegative" : 0, "falsePositive" : 0, "precision" : 0, "recall" : 0, "total" : 250, "trueNegative" : 250, "truePositive" : 0 }, "CPED test split" : { "accuracy" : 1, "f1" : 1, "falseNegative" : 0, "falsePositive" : 0, "precision" : 1, "recall" : 1, "total" : 25049, "trueNegative" : 25032, "truePositive" : 17 }, "Google Research GoEmotions test" : { "accuracy" : 1, "f1" : 1, "falseNegative" : 0, "falsePositive" : 0, "precision" : 1, "recall" : 1, "total" : 5418, "trueNegative" : 5352, "truePositive" : 66 }, "Meituan-Dianping/ASAP test" : { "accuracy" : 1, "f1" : 0, "falseNegative" : 0, "falsePositive" : 0, "precision" : 0, "recall" : 0, "total" : 120, "trueNegative" : 120, "truePositive" : 0 }, "Sulak2020 GitHub issues multi-repo dataset" : { "accuracy" : 1, "f1" : 1, "falseNegative" : 0, "falsePositive" : 0, "precision" : 1, "recall" : 1, "total" : 1004, "trueNegative" : 1002, "truePositive" : 2 }, "Verified Enron Intent Dataset" : { "accuracy" : 1, "f1" : 1, "falseNegative" : 0, "falsePositive" : 0, "precision" : 1, "recall" : 1, "total" : 3537, "trueNegative" : 3536, "truePositive" : 1 } }, "thresholdAt90Precision" : { "metrics" : { "accuracy" : 1, "f1" : 1, "falseNegative" : 0, "falsePositive" : 0, "precision" : 1, "recall" : 1, "total" : 41195, "trueNegative" : 41109, "truePositive" : 86 }, "threshold" : 0.05 }, "thresholdAt95Precision" : { "metrics" : { "accuracy" : 1, "f1" : 1, "falseNegative" : 0, "falsePositive" : 0, "precision" : 1, "recall" : 1, "total" : 41195, "trueNegative" : 41109, "truePositive" : 86 }, "threshold" : 0.05 }, "thresholdsAt90PrecisionByLanguage" : { "en" : { "metrics" : { "accuracy" : 1, "f1" : 1, "falseNegative" : 0, "falsePositive" : 0, "precision" : 1, "recall" : 1, "total" : 13176, "trueNegative" : 13107, "truePositive" : 69 }, "threshold" : 0.05 }, "zh-Hans" : { "metrics" : { "accuracy" : 1, "f1" : 1, "falseNegative" : 0, "falsePositive" : 0, "precision" : 1, "recall" : 1, "total" : 28019, "trueNegative" : 28002, "truePositive" : 17 }, "threshold" : 0.05 } }, "thresholdsAt95PrecisionByLanguage" : { "en" : { "metrics" : { "accuracy" : 1, "f1" : 1, "falseNegative" : 0, "falsePositive" : 0, "precision" : 1, "recall" : 1, "total" : 13176, "trueNegative" : 13107, "truePositive" : 69 }, "threshold" : 0.05 }, "zh-Hans" : { "metrics" : { "accuracy" : 1, "f1" : 1, "falseNegative" : 0, "falsePositive" : 0, "precision" : 1, "recall" : 1, "total" : 28019, "trueNegative" : 28002, "truePositive" : 17 }, "threshold" : 0.05 } } } ], "corpusRecordCount" : 41195, "exactTrainingOverlapCount" : 0, "familyCount" : 85, "generatedAt" : "2026-08-27T09:28:05Z", "languageCounts" : { "en" : 13176, "zh-Hans" : 28019 }, "manifestSchemaVersion" : 3, "seed" : 20260826, "sentiment" : { "accuracy" : 0.1726, "correct" : 7109, "macroF1" : 0.1614, "perLabelF1" : { "negative" : 0.0569, "neutral" : 0.4079, "positive" : 0.0194 }, "total" : 41195, "unknown" : 27306, "unknownRate" : 0.6628 }, "sentimentBySource" : { "AmazonScience/MASSIVE en-US test" : { "accuracy" : 0.2346, "correct" : 696, "macroF1" : 0.1627, "perLabelF1" : { "negative" : 0, "neutral" : 0.3829, "positive" : 0.1053 }, "total" : 2967, "unknown" : 2204, "unknownRate" : 0.7428 }, "AmazonScience/MASSIVE zh-CN test" : { "accuracy" : 0.2758, "correct" : 786, "macroF1" : 0.1452, "perLabelF1" : { "negative" : 0, "neutral" : 0.4357, "positive" : 0 }, "total" : 2850, "unknown" : 2026, "unknownRate" : 0.7109 }, "CFPB via hpe-ai/customer-complaints" : { "accuracy" : 0.156, "correct" : 39, "macroF1" : 0.09, "perLabelF1" : { "negative" : 0.2699, "neutral" : 0, "positive" : 0 }, "total" : 250, "unknown" : 40, "unknownRate" : 0.16 }, "CPED test split" : { "accuracy" : 0.0878, "correct" : 2199, "macroF1" : 0.1087, "perLabelF1" : { "negative" : 0.0494, "neutral" : 0.2702, "positive" : 0.0064 }, "total" : 25049, "unknown" : 17853, "unknownRate" : 0.7127 }, "Google Research GoEmotions test" : { "accuracy" : 0.1768, "correct" : 958, "macroF1" : 0.1768, "perLabelF1" : { "negative" : 0.0756, "neutral" : 0.3988, "positive" : 0.0561 }, "total" : 5418, "unknown" : 3170, "unknownRate" : 0.5851 }, "Meituan-Dianping/ASAP test" : { "accuracy" : 0.1167, "correct" : 14, "macroF1" : 0.0697, "perLabelF1" : { "negative" : 0.209, "neutral" : 0, "positive" : 0 }, "total" : 120, "unknown" : 3, "unknownRate" : 0.025 }, "Sulak2020 GitHub issues multi-repo dataset" : { "accuracy" : 0.1165, "correct" : 117, "macroF1" : 0.0891, "perLabelF1" : { "negative" : 0.0326, "neutral" : 0.2346, "positive" : 0 }, "total" : 1004, "unknown" : 846, "unknownRate" : 0.8426 }, "Verified Enron Intent Dataset" : { "accuracy" : 0.6503, "correct" : 2300, "macroF1" : 0.2627, "perLabelF1" : { "negative" : 0, "neutral" : 0.7881, "positive" : 0 }, "total" : 3537, "unknown" : 1164, "unknownRate" : 0.3291 } }, "verifierLayers" : [ { "acceptedForAutomaticRouting" : false, "deploymentMode" : "shadow", "id" : "action", "leaveOneSourceOut" : { "AmazonScience/MASSIVE en-US test" : { "correctRouted" : 2218, "expectedSpecialized" : 9796, "falseRouted" : 5564, "finalPrecision" : 0.285, "finalPrecisionWilsonLower95" : 0.2751, "finalRecall" : 0.2264, "routed" : 7782, "stageACandidates" : 23084, "stageARecall" : 0.7417, "stageBExactAccuracy" : 0.2543, "total" : 38228 }, "AmazonScience/MASSIVE zh-CN test" : { "correctRouted" : 2047, "expectedSpecialized" : 9910, "falseRouted" : 5361, "finalPrecision" : 0.2763, "finalPrecisionWilsonLower95" : 0.2663, "finalRecall" : 0.2066, "routed" : 7408, "stageACandidates" : 23774, "stageARecall" : 0.7776, "stageBExactAccuracy" : 0.2598, "total" : 38345 }, "CFPB via hpe-ai/customer-complaints" : { "correctRouted" : 2508, "expectedSpecialized" : 11676, "falseRouted" : 5701, "finalPrecision" : 0.3055, "finalPrecisionWilsonLower95" : 0.2956, "finalRecall" : 0.2148, "routed" : 8209, "stageACandidates" : 25413, "stageARecall" : 0.7603, "stageBExactAccuracy" : 0.2696, "total" : 40945 }, "CPED test split" : { "correctRouted" : 1059, "expectedSpecialized" : 6114, "falseRouted" : 884, "finalPrecision" : 0.545, "finalPrecisionWilsonLower95" : 0.5228, "finalRecall" : 0.1732, "routed" : 1943, "stageACandidates" : 8873, "stageARecall" : 0.6757, "stageBExactAccuracy" : 0.3496, "total" : 16146 }, "Google Research GoEmotions test" : { "correctRouted" : 2448, "expectedSpecialized" : 11341, "falseRouted" : 5587, "finalPrecision" : 0.3047, "finalPrecisionWilsonLower95" : 0.2947, "finalRecall" : 0.2159, "routed" : 8035, "stageACandidates" : 23224, "stageARecall" : 0.7483, "stageBExactAccuracy" : 0.2674, "total" : 35777 }, "Meituan-Dianping/ASAP test" : { "correctRouted" : 2526, "expectedSpecialized" : 11806, "falseRouted" : 5701, "finalPrecision" : 0.307, "finalPrecisionWilsonLower95" : 0.2972, "finalRecall" : 0.214, "routed" : 8227, "stageACandidates" : 25442, "stageARecall" : 0.7544, "stageBExactAccuracy" : 0.2703, "total" : 41075 }, "Sulak2020 GitHub issues multi-repo dataset" : { "correctRouted" : 2523, "expectedSpecialized" : 11621, "falseRouted" : 5691, "finalPrecision" : 0.3072, "finalPrecisionWilsonLower95" : 0.2973, "finalRecall" : 0.2171, "routed" : 8214, "stageACandidates" : 24733, "stageARecall" : 0.7484, "stageBExactAccuracy" : 0.2728, "total" : 40191 }, "Verified Enron Intent Dataset" : { "correctRouted" : 2402, "expectedSpecialized" : 11218, "falseRouted" : 5446, "finalPrecision" : 0.3061, "finalPrecisionWilsonLower95" : 0.296, "finalRecall" : 0.2141, "routed" : 7848, "stageACandidates" : 23628, "stageARecall" : 0.7443, "stageBExactAccuracy" : 0.269, "total" : 37658 } }, "metrics" : { "correctRouted" : 2533, "expectedSpecialized" : 11926, "falseRouted" : 5705, "finalPrecision" : 0.3075, "finalPrecisionWilsonLower95" : 0.2976, "finalRecall" : 0.2124, "routed" : 8238, "stageACandidates" : 25453, "stageARecall" : 0.7477, "stageBExactAccuracy" : 0.2705, "total" : 41195 }, "metricsByLanguage" : { "en" : { "correctRouted" : 566, "expectedSpecialized" : 3978, "falseRouted" : 536, "finalPrecision" : 0.5136, "finalPrecisionWilsonLower95" : 0.4841, "finalRecall" : 0.1423, "routed" : 1102, "stageACandidates" : 7183, "stageARecall" : 0.7313, "stageBExactAccuracy" : 0.3325, "total" : 13176 }, "zh-Hans" : { "correctRouted" : 1967, "expectedSpecialized" : 7948, "falseRouted" : 5169, "finalPrecision" : 0.2756, "finalPrecisionWilsonLower95" : 0.2654, "finalRecall" : 0.2475, "routed" : 7136, "stageACandidates" : 18270, "stageARecall" : 0.7559, "stageBExactAccuracy" : 0.2461, "total" : 28019 } }, "metricsBySource" : { "AmazonScience/MASSIVE en-US test" : { "correctRouted" : 315, "expectedSpecialized" : 2130, "falseRouted" : 141, "finalPrecision" : 0.6908, "finalPrecisionWilsonLower95" : 0.6469, "finalRecall" : 0.1479, "routed" : 456, "stageACandidates" : 2369, "stageARecall" : 0.7751, "stageBExactAccuracy" : 0.428, "total" : 2967 }, "AmazonScience/MASSIVE zh-CN test" : { "correctRouted" : 486, "expectedSpecialized" : 2016, "falseRouted" : 344, "finalPrecision" : 0.5855, "finalPrecisionWilsonLower95" : 0.5517, "finalRecall" : 0.2411, "routed" : 830, "stageACandidates" : 1679, "stageARecall" : 0.6007, "stageBExactAccuracy" : 0.4211, "total" : 2850 }, "CFPB via hpe-ai/customer-complaints" : { "correctRouted" : 25, "expectedSpecialized" : 250, "falseRouted" : 4, "finalPrecision" : 0.8621, "finalPrecisionWilsonLower95" : 0.6944, "finalRecall" : 0.1, "routed" : 29, "stageACandidates" : 40, "stageARecall" : 0.16, "stageBExactAccuracy" : 0.8, "total" : 250 }, "CPED test split" : { "correctRouted" : 1474, "expectedSpecialized" : 5812, "falseRouted" : 4821, "finalPrecision" : 0.2342, "finalPrecisionWilsonLower95" : 0.2239, "finalRecall" : 0.2536, "routed" : 6295, "stageACandidates" : 16580, "stageARecall" : 0.8235, "stageBExactAccuracy" : 0.2281, "total" : 25049 }, "Google Research GoEmotions test" : { "correctRouted" : 85, "expectedSpecialized" : 585, "falseRouted" : 118, "finalPrecision" : 0.4187, "finalPrecisionWilsonLower95" : 0.353, "finalRecall" : 0.1453, "routed" : 203, "stageACandidates" : 2229, "stageARecall" : 0.7368, "stageBExactAccuracy" : 0.3028, "total" : 5418 }, "Meituan-Dianping/ASAP test" : { "correctRouted" : 7, "expectedSpecialized" : 120, "falseRouted" : 4, "finalPrecision" : 0.6364, "finalPrecisionWilsonLower95" : 0.3538, "finalRecall" : 0.0583, "routed" : 11, "stageACandidates" : 11, "stageARecall" : 0.0917, "stageBExactAccuracy" : 0.6364, "total" : 120 }, "Sulak2020 GitHub issues multi-repo dataset" : { "correctRouted" : 10, "expectedSpecialized" : 305, "falseRouted" : 14, "finalPrecision" : 0.4167, "finalPrecisionWilsonLower95" : 0.2447, "finalRecall" : 0.0328, "routed" : 24, "stageACandidates" : 720, "stageARecall" : 0.7213, "stageBExactAccuracy" : 0.1917, "total" : 1004 }, "Verified Enron Intent Dataset" : { "correctRouted" : 131, "expectedSpecialized" : 708, "falseRouted" : 259, "finalPrecision" : 0.3359, "finalPrecisionWilsonLower95" : 0.2908, "finalRecall" : 0.185, "routed" : 390, "stageACandidates" : 1825, "stageARecall" : 0.8008, "stageBExactAccuracy" : 0.2899, "total" : 3537 } }, "warmMedianMilliseconds" : 0.1429, "warmP95Milliseconds" : 0.184 }, { "acceptedForAutomaticRouting" : false, "deploymentMode" : "shadow", "id" : "coordination", "leaveOneSourceOut" : { "AmazonScience/MASSIVE en-US test" : { "correctRouted" : 23, "expectedSpecialized" : 557, "falseRouted" : 145, "finalPrecision" : 0.1369, "finalPrecisionWilsonLower95" : 0.093, "finalRecall" : 0.0413, "routed" : 168, "stageACandidates" : 17464, "stageARecall" : 0.6732, "stageBExactAccuracy" : 0.4356, "total" : 38228 }, "AmazonScience/MASSIVE zh-CN test" : { "correctRouted" : 32, "expectedSpecialized" : 488, "falseRouted" : 203, "finalPrecision" : 0.1362, "finalPrecisionWilsonLower95" : 0.0981, "finalRecall" : 0.0656, "routed" : 235, "stageACandidates" : 17780, "stageARecall" : 0.6025, "stageBExactAccuracy" : 0.4465, "total" : 38345 }, "CFPB via hpe-ai/customer-complaints" : { "correctRouted" : 32, "expectedSpecialized" : 654, "falseRouted" : 203, "finalPrecision" : 0.1362, "finalPrecisionWilsonLower95" : 0.0981, "finalRecall" : 0.0489, "routed" : 235, "stageACandidates" : 19370, "stageARecall" : 0.6774, "stageBExactAccuracy" : 0.4424, "total" : 40945 }, "CPED test split" : { "correctRouted" : 32, "expectedSpecialized" : 425, "falseRouted" : 203, "finalPrecision" : 0.1362, "finalPrecisionWilsonLower95" : 0.0981, "finalRecall" : 0.0753, "routed" : 235, "stageACandidates" : 6813, "stageARecall" : 0.68, "stageBExactAccuracy" : 0.4728, "total" : 16146 }, "Google Research GoEmotions test" : { "correctRouted" : 32, "expectedSpecialized" : 654, "falseRouted" : 160, "finalPrecision" : 0.1667, "finalPrecisionWilsonLower95" : 0.1206, "finalRecall" : 0.0489, "routed" : 192, "stageACandidates" : 17699, "stageARecall" : 0.6774, "stageBExactAccuracy" : 0.4328, "total" : 35777 }, "Meituan-Dianping/ASAP test" : { "correctRouted" : 32, "expectedSpecialized" : 654, "falseRouted" : 203, "finalPrecision" : 0.1362, "finalPrecisionWilsonLower95" : 0.0981, "finalRecall" : 0.0489, "routed" : 235, "stageACandidates" : 19378, "stageARecall" : 0.6774, "stageBExactAccuracy" : 0.4424, "total" : 41075 }, "Sulak2020 GitHub issues multi-repo dataset" : { "correctRouted" : 32, "expectedSpecialized" : 625, "falseRouted" : 197, "finalPrecision" : 0.1397, "finalPrecisionWilsonLower95" : 0.1008, "finalRecall" : 0.0512, "routed" : 229, "stageACandidates" : 18772, "stageARecall" : 0.696, "stageBExactAccuracy" : 0.4437, "total" : 40191 }, "Verified Enron Intent Dataset" : { "correctRouted" : 9, "expectedSpecialized" : 521, "falseRouted" : 107, "finalPrecision" : 0.0776, "finalPrecisionWilsonLower95" : 0.0414, "finalRecall" : 0.0173, "routed" : 116, "stageACandidates" : 18370, "stageARecall" : 0.7274, "stageBExactAccuracy" : 0.4416, "total" : 37658 } }, "metrics" : { "correctRouted" : 32, "expectedSpecialized" : 654, "falseRouted" : 203, "finalPrecision" : 0.1362, "finalPrecisionWilsonLower95" : 0.0981, "finalRecall" : 0.0489, "routed" : 235, "stageACandidates" : 19378, "stageARecall" : 0.6774, "stageBExactAccuracy" : 0.4424, "total" : 41195 }, "metricsByLanguage" : { "en" : { "correctRouted" : 32, "expectedSpecialized" : 259, "falseRouted" : 203, "finalPrecision" : 0.1362, "finalPrecisionWilsonLower95" : 0.0981, "finalRecall" : 0.1236, "routed" : 235, "stageACandidates" : 5215, "stageARecall" : 0.5405, "stageBExactAccuracy" : 0.4961, "total" : 13176 }, "zh-Hans" : { "correctRouted" : 0, "expectedSpecialized" : 395, "falseRouted" : 0, "finalPrecision" : 0, "finalPrecisionWilsonLower95" : 0, "finalRecall" : 0, "routed" : 0, "stageACandidates" : 14163, "stageARecall" : 0.7671, "stageBExactAccuracy" : 0.4227, "total" : 28019 } }, "metricsBySource" : { "AmazonScience/MASSIVE en-US test" : { "correctRouted" : 9, "expectedSpecialized" : 97, "falseRouted" : 58, "finalPrecision" : 0.1343, "finalPrecisionWilsonLower95" : 0.0723, "finalRecall" : 0.0928, "routed" : 67, "stageACandidates" : 1914, "stageARecall" : 0.701, "stageBExactAccuracy" : 0.5047, "total" : 2967 }, "AmazonScience/MASSIVE zh-CN test" : { "correctRouted" : 0, "expectedSpecialized" : 166, "falseRouted" : 0, "finalPrecision" : 0, "finalPrecisionWilsonLower95" : 0, "finalRecall" : 0, "routed" : 0, "stageACandidates" : 1598, "stageARecall" : 0.8976, "stageBExactAccuracy" : 0.3967, "total" : 2850 }, "CFPB via hpe-ai/customer-complaints" : { "correctRouted" : 0, "expectedSpecialized" : 0, "falseRouted" : 0, "finalPrecision" : 0, "finalPrecisionWilsonLower95" : 0, "finalRecall" : 0, "routed" : 0, "stageACandidates" : 8, "stageARecall" : 0, "stageBExactAccuracy" : 0.375, "total" : 250 }, "CPED test split" : { "correctRouted" : 0, "expectedSpecialized" : 229, "falseRouted" : 0, "finalPrecision" : 0, "finalPrecisionWilsonLower95" : 0, "finalRecall" : 0, "routed" : 0, "stageACandidates" : 12565, "stageARecall" : 0.6725, "stageBExactAccuracy" : 0.4259, "total" : 25049 }, "Google Research GoEmotions test" : { "correctRouted" : 0, "expectedSpecialized" : 0, "falseRouted" : 43, "finalPrecision" : 0, "finalPrecisionWilsonLower95" : 0, "finalRecall" : 0, "routed" : 43, "stageACandidates" : 1679, "stageARecall" : 0, "stageBExactAccuracy" : 0.5438, "total" : 5418 }, "Meituan-Dianping/ASAP test" : { "correctRouted" : 0, "expectedSpecialized" : 0, "falseRouted" : 0, "finalPrecision" : 0, "finalPrecisionWilsonLower95" : 0, "finalRecall" : 0, "routed" : 0, "stageACandidates" : 0, "stageARecall" : 0, "stageBExactAccuracy" : 0, "total" : 120 }, "Sulak2020 GitHub issues multi-repo dataset" : { "correctRouted" : 0, "expectedSpecialized" : 29, "falseRouted" : 6, "finalPrecision" : 0, "finalPrecisionWilsonLower95" : 0, "finalRecall" : 0, "routed" : 6, "stageACandidates" : 606, "stageARecall" : 0.2759, "stageBExactAccuracy" : 0.4026, "total" : 1004 }, "Verified Enron Intent Dataset" : { "correctRouted" : 23, "expectedSpecialized" : 133, "falseRouted" : 96, "finalPrecision" : 0.1933, "finalPrecisionWilsonLower95" : 0.1324, "finalRecall" : 0.1729, "routed" : 119, "stageACandidates" : 1008, "stageARecall" : 0.4812, "stageBExactAccuracy" : 0.4573, "total" : 3537 } }, "warmMedianMilliseconds" : 0.1412, "warmP95Milliseconds" : 0.1755 } ] }