docs(semantics): record iterative retraining study
Co-authored-by: Rocky <hkgood@users.noreply.github.com>
This commit is contained in:
Cursor Agent
+8422
File diff suppressed because it is too large
Load Diff
+629
@@ -0,0 +1,629 @@
|
||||
{
|
||||
"automaticPromotionAllowed": false,
|
||||
"baseCorpus": "ModelTraining/ClipboardSemantics/clipboard_semantic_corpus.jsonl",
|
||||
"configuration": {
|
||||
"alpha": 2e-05,
|
||||
"augmentation": "none",
|
||||
"char_max": 5,
|
||||
"char_min": 2,
|
||||
"external_weight": 0.5700000000000001,
|
||||
"hard_example_weight": 1.0,
|
||||
"l1_ratio": 0.0,
|
||||
"max_char_features": 45000,
|
||||
"max_word_features": 22000,
|
||||
"min_df": 2,
|
||||
"pseudo_weight": 0.16499999999999998,
|
||||
"round": 1,
|
||||
"word_max": 2
|
||||
},
|
||||
"configurationsByIntent": {
|
||||
"blessing": {
|
||||
"alpha": 1.2e-06,
|
||||
"augmentation": "numeric",
|
||||
"char_max": 8,
|
||||
"char_min": 3,
|
||||
"external_weight": 0.93,
|
||||
"hard_example_weight": 2.7,
|
||||
"l1_ratio": 0.25,
|
||||
"max_char_features": 95000,
|
||||
"max_word_features": 45000,
|
||||
"min_df": 1,
|
||||
"pseudo_weight": 0.43499999999999994,
|
||||
"round": 19,
|
||||
"word_max": 3
|
||||
},
|
||||
"complaint": {
|
||||
"alpha": 2e-05,
|
||||
"augmentation": "none",
|
||||
"char_max": 5,
|
||||
"char_min": 2,
|
||||
"external_weight": 0.5700000000000001,
|
||||
"hard_example_weight": 1.0,
|
||||
"l1_ratio": 0.0,
|
||||
"max_char_features": 45000,
|
||||
"max_word_features": 22000,
|
||||
"min_df": 2,
|
||||
"pseudo_weight": 0.16499999999999998,
|
||||
"round": 1,
|
||||
"word_max": 2
|
||||
},
|
||||
"confirmationDecision": {
|
||||
"alpha": 5e-06,
|
||||
"augmentation": "prefix",
|
||||
"char_max": 6,
|
||||
"char_min": 2,
|
||||
"external_weight": 0.77,
|
||||
"hard_example_weight": 1.8,
|
||||
"l1_ratio": 0.1,
|
||||
"max_char_features": 70000,
|
||||
"max_word_features": 30000,
|
||||
"min_df": 2,
|
||||
"pseudo_weight": 0.31499999999999995,
|
||||
"round": 11,
|
||||
"word_max": 2
|
||||
},
|
||||
"followUpReminder": {
|
||||
"alpha": 2.5e-06,
|
||||
"augmentation": "prefix",
|
||||
"char_max": 7,
|
||||
"char_min": 3,
|
||||
"external_weight": 0.8300000000000001,
|
||||
"hard_example_weight": 2.2,
|
||||
"l1_ratio": 0.2,
|
||||
"max_char_features": 80000,
|
||||
"max_word_features": 40000,
|
||||
"min_df": 1,
|
||||
"pseudo_weight": 0.36,
|
||||
"round": 14,
|
||||
"word_max": 3
|
||||
},
|
||||
"invitation": {
|
||||
"alpha": 2e-05,
|
||||
"augmentation": "none",
|
||||
"char_max": 5,
|
||||
"char_min": 2,
|
||||
"external_weight": 0.5700000000000001,
|
||||
"hard_example_weight": 1.0,
|
||||
"l1_ratio": 0.0,
|
||||
"max_char_features": 45000,
|
||||
"max_word_features": 22000,
|
||||
"min_df": 2,
|
||||
"pseudo_weight": 0.16499999999999998,
|
||||
"round": 1,
|
||||
"word_max": 2
|
||||
},
|
||||
"question": {
|
||||
"alpha": 2e-05,
|
||||
"augmentation": "none",
|
||||
"char_max": 5,
|
||||
"char_min": 2,
|
||||
"external_weight": 0.5700000000000001,
|
||||
"hard_example_weight": 1.0,
|
||||
"l1_ratio": 0.0,
|
||||
"max_char_features": 45000,
|
||||
"max_word_features": 22000,
|
||||
"min_df": 2,
|
||||
"pseudo_weight": 0.16499999999999998,
|
||||
"round": 1,
|
||||
"word_max": 2
|
||||
},
|
||||
"replyableMessage": {
|
||||
"alpha": 2e-05,
|
||||
"augmentation": "none",
|
||||
"char_max": 5,
|
||||
"char_min": 2,
|
||||
"external_weight": 0.5700000000000001,
|
||||
"hard_example_weight": 1.0,
|
||||
"l1_ratio": 0.0,
|
||||
"max_char_features": 45000,
|
||||
"max_word_features": 22000,
|
||||
"min_df": 2,
|
||||
"pseudo_weight": 0.16499999999999998,
|
||||
"round": 1,
|
||||
"word_max": 2
|
||||
},
|
||||
"scheduleNegotiation": {
|
||||
"alpha": 6e-06,
|
||||
"augmentation": "punctuation",
|
||||
"char_max": 6,
|
||||
"char_min": 3,
|
||||
"external_weight": 0.6900000000000001,
|
||||
"hard_example_weight": 1.5,
|
||||
"l1_ratio": 0.1,
|
||||
"max_char_features": 65000,
|
||||
"max_word_features": 32000,
|
||||
"min_df": 2,
|
||||
"pseudo_weight": 0.255,
|
||||
"round": 7,
|
||||
"word_max": 3
|
||||
},
|
||||
"task": {
|
||||
"alpha": 2e-05,
|
||||
"augmentation": "none",
|
||||
"char_max": 5,
|
||||
"char_min": 2,
|
||||
"external_weight": 0.5700000000000001,
|
||||
"hard_example_weight": 1.0,
|
||||
"l1_ratio": 0.0,
|
||||
"max_char_features": 45000,
|
||||
"max_word_features": 22000,
|
||||
"min_df": 2,
|
||||
"pseudo_weight": 0.16499999999999998,
|
||||
"round": 1,
|
||||
"word_max": 2
|
||||
}
|
||||
},
|
||||
"openCorpus": "ModelTraining/ClipboardSemantics/open-training-corpus.jsonl",
|
||||
"reason": "Surrogate feature weights are not deployable NLModel assets and cannot bypass the existing macOS acceptance policy.",
|
||||
"requiredCommands": [
|
||||
"python3 Scripts/clipboard_semantics/generate_open_training_corpus.py",
|
||||
"xcrun swift Scripts/clipboard_semantics/train_models.swift --algorithms maxEnt --corpus ModelTraining/ClipboardSemantics/combined-training-corpus.jsonl",
|
||||
"xcrun swift Scripts/clipboard_semantics/evaluate_random_holdout.swift --corpus ModelTraining/ClipboardSemantics/random-holdout-corpus.jsonl"
|
||||
],
|
||||
"schemaVersion": 1,
|
||||
"seed": 20260827,
|
||||
"selectedRound": 1,
|
||||
"selectedRoundsByIntent": {
|
||||
"blessing": 19,
|
||||
"complaint": 1,
|
||||
"confirmationDecision": 11,
|
||||
"followUpReminder": 14,
|
||||
"invitation": 1,
|
||||
"question": 1,
|
||||
"replyableMessage": 1,
|
||||
"scheduleNegotiation": 7,
|
||||
"task": 1
|
||||
},
|
||||
"thresholds": {
|
||||
"blessing": {
|
||||
"abstained": true,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": true,
|
||||
"metrics": {
|
||||
"f1": 0.0,
|
||||
"falseNegative": 0,
|
||||
"falsePositive": 0,
|
||||
"precision": 0.0,
|
||||
"predictedPositive": 0,
|
||||
"recall": 0.0,
|
||||
"total": 0,
|
||||
"trueNegative": 0,
|
||||
"truePositive": 0,
|
||||
"wilsonPrecisionLower95": 0
|
||||
},
|
||||
"threshold": 1.0
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": true,
|
||||
"metrics": {
|
||||
"f1": 0.0,
|
||||
"falseNegative": 0,
|
||||
"falsePositive": 0,
|
||||
"precision": 0.0,
|
||||
"predictedPositive": 0,
|
||||
"recall": 0.0,
|
||||
"total": 0,
|
||||
"trueNegative": 0,
|
||||
"truePositive": 0,
|
||||
"wilsonPrecisionLower95": 0
|
||||
},
|
||||
"threshold": 1.0
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.0,
|
||||
"falseNegative": 0,
|
||||
"falsePositive": 0,
|
||||
"precision": 0.0,
|
||||
"predictedPositive": 0,
|
||||
"recall": 0.0,
|
||||
"total": 0,
|
||||
"trueNegative": 0,
|
||||
"truePositive": 0,
|
||||
"wilsonPrecisionLower95": 0
|
||||
},
|
||||
"threshold": 1.0
|
||||
},
|
||||
"complaint": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 1.0,
|
||||
"falseNegative": 0,
|
||||
"falsePositive": 0,
|
||||
"precision": 1.0,
|
||||
"predictedPositive": 180,
|
||||
"recall": 1.0,
|
||||
"total": 1485,
|
||||
"trueNegative": 1305,
|
||||
"truePositive": 180,
|
||||
"wilsonPrecisionLower95": 0.9791
|
||||
},
|
||||
"threshold": 0.5
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.9526,
|
||||
"falseNegative": 9,
|
||||
"falsePositive": 8,
|
||||
"precision": 0.9553,
|
||||
"predictedPositive": 179,
|
||||
"recall": 0.95,
|
||||
"total": 1485,
|
||||
"trueNegative": 1297,
|
||||
"truePositive": 171,
|
||||
"wilsonPrecisionLower95": 0.9143
|
||||
},
|
||||
"threshold": 0.54
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.9764,
|
||||
"falseNegative": 9,
|
||||
"falsePositive": 8,
|
||||
"precision": 0.9777,
|
||||
"predictedPositive": 359,
|
||||
"recall": 0.975,
|
||||
"total": 2970,
|
||||
"trueNegative": 2602,
|
||||
"truePositive": 351,
|
||||
"wilsonPrecisionLower95": 0.9567
|
||||
},
|
||||
"threshold": 0.54
|
||||
},
|
||||
"confirmationDecision": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.931,
|
||||
"falseNegative": 9,
|
||||
"falsePositive": 3,
|
||||
"precision": 0.9643,
|
||||
"predictedPositive": 84,
|
||||
"recall": 0.9,
|
||||
"total": 1485,
|
||||
"trueNegative": 1392,
|
||||
"truePositive": 81,
|
||||
"wilsonPrecisionLower95": 0.9002
|
||||
},
|
||||
"threshold": 0.97
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.7222,
|
||||
"falseNegative": 38,
|
||||
"falsePositive": 2,
|
||||
"precision": 0.963,
|
||||
"predictedPositive": 54,
|
||||
"recall": 0.5778,
|
||||
"total": 1485,
|
||||
"trueNegative": 1393,
|
||||
"truePositive": 52,
|
||||
"wilsonPrecisionLower95": 0.8746
|
||||
},
|
||||
"threshold": 0.86
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.8077,
|
||||
"falseNegative": 54,
|
||||
"falsePositive": 6,
|
||||
"precision": 0.9545,
|
||||
"predictedPositive": 132,
|
||||
"recall": 0.7,
|
||||
"total": 2970,
|
||||
"trueNegative": 2784,
|
||||
"truePositive": 126,
|
||||
"wilsonPrecisionLower95": 0.9044
|
||||
},
|
||||
"threshold": 0.95
|
||||
},
|
||||
"followUpReminder": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.8788,
|
||||
"falseNegative": 35,
|
||||
"falsePositive": 5,
|
||||
"precision": 0.9667,
|
||||
"predictedPositive": 150,
|
||||
"recall": 0.8056,
|
||||
"total": 1485,
|
||||
"trueNegative": 1300,
|
||||
"truePositive": 145,
|
||||
"wilsonPrecisionLower95": 0.9243
|
||||
},
|
||||
"threshold": 0.845
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.6113,
|
||||
"falseNegative": 99,
|
||||
"falsePositive": 4,
|
||||
"precision": 0.9529,
|
||||
"predictedPositive": 85,
|
||||
"recall": 0.45,
|
||||
"total": 1485,
|
||||
"trueNegative": 1301,
|
||||
"truePositive": 81,
|
||||
"wilsonPrecisionLower95": 0.8852
|
||||
},
|
||||
"threshold": 0.97
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.7682,
|
||||
"falseNegative": 128,
|
||||
"falsePositive": 12,
|
||||
"precision": 0.9508,
|
||||
"predictedPositive": 244,
|
||||
"recall": 0.6444,
|
||||
"total": 2970,
|
||||
"trueNegative": 2598,
|
||||
"truePositive": 232,
|
||||
"wilsonPrecisionLower95": 0.916
|
||||
},
|
||||
"threshold": 0.91
|
||||
},
|
||||
"invitation": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.8302,
|
||||
"falseNegative": 24,
|
||||
"falsePositive": 3,
|
||||
"precision": 0.9565,
|
||||
"predictedPositive": 69,
|
||||
"recall": 0.7333,
|
||||
"total": 1485,
|
||||
"trueNegative": 1392,
|
||||
"truePositive": 66,
|
||||
"wilsonPrecisionLower95": 0.8798
|
||||
},
|
||||
"threshold": 0.935
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.6765,
|
||||
"falseNegative": 44,
|
||||
"falsePositive": 0,
|
||||
"precision": 1.0,
|
||||
"predictedPositive": 46,
|
||||
"recall": 0.5111,
|
||||
"total": 1485,
|
||||
"trueNegative": 1395,
|
||||
"truePositive": 46,
|
||||
"wilsonPrecisionLower95": 0.9229
|
||||
},
|
||||
"threshold": 0.505
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.5476,
|
||||
"falseNegative": 111,
|
||||
"falsePositive": 3,
|
||||
"precision": 0.9583,
|
||||
"predictedPositive": 72,
|
||||
"recall": 0.3833,
|
||||
"total": 2970,
|
||||
"trueNegative": 2787,
|
||||
"truePositive": 69,
|
||||
"wilsonPrecisionLower95": 0.8845
|
||||
},
|
||||
"threshold": 0.935
|
||||
},
|
||||
"question": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.8445,
|
||||
"falseNegative": 76,
|
||||
"falsePositive": 12,
|
||||
"precision": 0.9522,
|
||||
"predictedPositive": 251,
|
||||
"recall": 0.7587,
|
||||
"total": 1485,
|
||||
"trueNegative": 1158,
|
||||
"truePositive": 239,
|
||||
"wilsonPrecisionLower95": 0.9183
|
||||
},
|
||||
"threshold": 0.725
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.8536,
|
||||
"falseNegative": 76,
|
||||
"falsePositive": 6,
|
||||
"precision": 0.9755,
|
||||
"predictedPositive": 245,
|
||||
"recall": 0.7587,
|
||||
"total": 1485,
|
||||
"trueNegative": 1164,
|
||||
"truePositive": 239,
|
||||
"wilsonPrecisionLower95": 0.9476
|
||||
},
|
||||
"threshold": 0.5
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.8729,
|
||||
"falseNegative": 122,
|
||||
"falsePositive": 26,
|
||||
"precision": 0.9513,
|
||||
"predictedPositive": 534,
|
||||
"recall": 0.8063,
|
||||
"total": 2970,
|
||||
"trueNegative": 2314,
|
||||
"truePositive": 508,
|
||||
"wilsonPrecisionLower95": 0.9296
|
||||
},
|
||||
"threshold": 0.515
|
||||
},
|
||||
"replyableMessage": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.948,
|
||||
"falseNegative": 52,
|
||||
"falsePositive": 46,
|
||||
"precision": 0.951,
|
||||
"predictedPositive": 939,
|
||||
"recall": 0.945,
|
||||
"total": 1485,
|
||||
"trueNegative": 494,
|
||||
"truePositive": 893,
|
||||
"wilsonPrecisionLower95": 0.9353
|
||||
},
|
||||
"threshold": 0.7
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.9424,
|
||||
"falseNegative": 62,
|
||||
"falsePositive": 46,
|
||||
"precision": 0.9505,
|
||||
"predictedPositive": 929,
|
||||
"recall": 0.9344,
|
||||
"total": 1485,
|
||||
"trueNegative": 494,
|
||||
"truePositive": 883,
|
||||
"wilsonPrecisionLower95": 0.9346
|
||||
},
|
||||
"threshold": 0.695
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.9454,
|
||||
"falseNegative": 115,
|
||||
"falsePositive": 90,
|
||||
"precision": 0.9517,
|
||||
"predictedPositive": 1865,
|
||||
"recall": 0.9392,
|
||||
"total": 2970,
|
||||
"trueNegative": 990,
|
||||
"truePositive": 1775,
|
||||
"wilsonPrecisionLower95": 0.9411
|
||||
},
|
||||
"threshold": 0.7
|
||||
},
|
||||
"scheduleNegotiation": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 1.0,
|
||||
"falseNegative": 0,
|
||||
"falsePositive": 0,
|
||||
"precision": 1.0,
|
||||
"predictedPositive": 45,
|
||||
"recall": 1.0,
|
||||
"total": 1485,
|
||||
"trueNegative": 1440,
|
||||
"truePositive": 45,
|
||||
"wilsonPrecisionLower95": 0.9213
|
||||
},
|
||||
"threshold": 0.66
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 1.0,
|
||||
"falseNegative": 0,
|
||||
"falsePositive": 0,
|
||||
"precision": 1.0,
|
||||
"predictedPositive": 45,
|
||||
"recall": 1.0,
|
||||
"total": 1485,
|
||||
"trueNegative": 1440,
|
||||
"truePositive": 45,
|
||||
"wilsonPrecisionLower95": 0.9213
|
||||
},
|
||||
"threshold": 0.55
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 1.0,
|
||||
"falseNegative": 0,
|
||||
"falsePositive": 0,
|
||||
"precision": 1.0,
|
||||
"predictedPositive": 90,
|
||||
"recall": 1.0,
|
||||
"total": 2970,
|
||||
"trueNegative": 2880,
|
||||
"truePositive": 90,
|
||||
"wilsonPrecisionLower95": 0.9591
|
||||
},
|
||||
"threshold": 0.66
|
||||
},
|
||||
"task": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.8403,
|
||||
"falseNegative": 89,
|
||||
"falsePositive": 14,
|
||||
"precision": 0.9509,
|
||||
"predictedPositive": 285,
|
||||
"recall": 0.7528,
|
||||
"total": 1485,
|
||||
"trueNegative": 1111,
|
||||
"truePositive": 271,
|
||||
"wilsonPrecisionLower95": 0.9192
|
||||
},
|
||||
"threshold": 0.835
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.8278,
|
||||
"falseNegative": 98,
|
||||
"falsePositive": 11,
|
||||
"precision": 0.9597,
|
||||
"predictedPositive": 273,
|
||||
"recall": 0.7278,
|
||||
"total": 1485,
|
||||
"trueNegative": 1114,
|
||||
"truePositive": 262,
|
||||
"wilsonPrecisionLower95": 0.9293
|
||||
},
|
||||
"threshold": 0.745
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.8637,
|
||||
"falseNegative": 153,
|
||||
"falsePositive": 26,
|
||||
"precision": 0.9562,
|
||||
"predictedPositive": 593,
|
||||
"recall": 0.7875,
|
||||
"total": 2970,
|
||||
"trueNegative": 2224,
|
||||
"truePositive": 567,
|
||||
"wilsonPrecisionLower95": 0.9365
|
||||
},
|
||||
"threshold": 0.755
|
||||
}
|
||||
}
|
||||
}
|
||||
+14831
File diff suppressed because it is too large
Load Diff
+8422
File diff suppressed because it is too large
Load Diff
+629
@@ -0,0 +1,629 @@
|
||||
{
|
||||
"automaticPromotionAllowed": false,
|
||||
"baseCorpus": "ModelTraining/ClipboardSemantics/clipboard_semantic_corpus.jsonl",
|
||||
"configuration": {
|
||||
"alpha": 2e-05,
|
||||
"augmentation": "none",
|
||||
"char_max": 5,
|
||||
"char_min": 2,
|
||||
"external_weight": 0.5700000000000001,
|
||||
"hard_example_weight": 1.0,
|
||||
"l1_ratio": 0.0,
|
||||
"max_char_features": 45000,
|
||||
"max_word_features": 22000,
|
||||
"min_df": 2,
|
||||
"pseudo_weight": 0.16499999999999998,
|
||||
"round": 1,
|
||||
"word_max": 2
|
||||
},
|
||||
"configurationsByIntent": {
|
||||
"blessing": {
|
||||
"alpha": 1.2e-06,
|
||||
"augmentation": "numeric",
|
||||
"char_max": 8,
|
||||
"char_min": 3,
|
||||
"external_weight": 0.93,
|
||||
"hard_example_weight": 2.7,
|
||||
"l1_ratio": 0.25,
|
||||
"max_char_features": 95000,
|
||||
"max_word_features": 45000,
|
||||
"min_df": 1,
|
||||
"pseudo_weight": 0.43499999999999994,
|
||||
"round": 19,
|
||||
"word_max": 3
|
||||
},
|
||||
"complaint": {
|
||||
"alpha": 2e-05,
|
||||
"augmentation": "none",
|
||||
"char_max": 5,
|
||||
"char_min": 2,
|
||||
"external_weight": 0.5700000000000001,
|
||||
"hard_example_weight": 1.0,
|
||||
"l1_ratio": 0.0,
|
||||
"max_char_features": 45000,
|
||||
"max_word_features": 22000,
|
||||
"min_df": 2,
|
||||
"pseudo_weight": 0.16499999999999998,
|
||||
"round": 1,
|
||||
"word_max": 2
|
||||
},
|
||||
"confirmationDecision": {
|
||||
"alpha": 1e-05,
|
||||
"augmentation": "none",
|
||||
"char_max": 6,
|
||||
"char_min": 2,
|
||||
"external_weight": 0.6100000000000001,
|
||||
"hard_example_weight": 1.2,
|
||||
"l1_ratio": 0.05,
|
||||
"max_char_features": 55000,
|
||||
"max_word_features": 25000,
|
||||
"min_df": 2,
|
||||
"pseudo_weight": 0.195,
|
||||
"round": 3,
|
||||
"word_max": 2
|
||||
},
|
||||
"followUpReminder": {
|
||||
"alpha": 2e-05,
|
||||
"augmentation": "none",
|
||||
"char_max": 5,
|
||||
"char_min": 2,
|
||||
"external_weight": 0.5700000000000001,
|
||||
"hard_example_weight": 1.0,
|
||||
"l1_ratio": 0.0,
|
||||
"max_char_features": 45000,
|
||||
"max_word_features": 22000,
|
||||
"min_df": 2,
|
||||
"pseudo_weight": 0.16499999999999998,
|
||||
"round": 1,
|
||||
"word_max": 2
|
||||
},
|
||||
"invitation": {
|
||||
"alpha": 2e-05,
|
||||
"augmentation": "none",
|
||||
"char_max": 5,
|
||||
"char_min": 2,
|
||||
"external_weight": 0.5700000000000001,
|
||||
"hard_example_weight": 1.0,
|
||||
"l1_ratio": 0.0,
|
||||
"max_char_features": 45000,
|
||||
"max_word_features": 22000,
|
||||
"min_df": 2,
|
||||
"pseudo_weight": 0.16499999999999998,
|
||||
"round": 1,
|
||||
"word_max": 2
|
||||
},
|
||||
"question": {
|
||||
"alpha": 2e-05,
|
||||
"augmentation": "none",
|
||||
"char_max": 5,
|
||||
"char_min": 2,
|
||||
"external_weight": 0.5700000000000001,
|
||||
"hard_example_weight": 1.0,
|
||||
"l1_ratio": 0.0,
|
||||
"max_char_features": 45000,
|
||||
"max_word_features": 22000,
|
||||
"min_df": 2,
|
||||
"pseudo_weight": 0.16499999999999998,
|
||||
"round": 1,
|
||||
"word_max": 2
|
||||
},
|
||||
"replyableMessage": {
|
||||
"alpha": 2e-05,
|
||||
"augmentation": "none",
|
||||
"char_max": 5,
|
||||
"char_min": 2,
|
||||
"external_weight": 0.5700000000000001,
|
||||
"hard_example_weight": 1.0,
|
||||
"l1_ratio": 0.0,
|
||||
"max_char_features": 45000,
|
||||
"max_word_features": 22000,
|
||||
"min_df": 2,
|
||||
"pseudo_weight": 0.16499999999999998,
|
||||
"round": 1,
|
||||
"word_max": 2
|
||||
},
|
||||
"scheduleNegotiation": {
|
||||
"alpha": 6e-06,
|
||||
"augmentation": "punctuation",
|
||||
"char_max": 6,
|
||||
"char_min": 3,
|
||||
"external_weight": 0.6900000000000001,
|
||||
"hard_example_weight": 1.5,
|
||||
"l1_ratio": 0.1,
|
||||
"max_char_features": 65000,
|
||||
"max_word_features": 32000,
|
||||
"min_df": 2,
|
||||
"pseudo_weight": 0.255,
|
||||
"round": 7,
|
||||
"word_max": 3
|
||||
},
|
||||
"task": {
|
||||
"alpha": 3.5e-06,
|
||||
"augmentation": "prefix",
|
||||
"char_max": 6,
|
||||
"char_min": 2,
|
||||
"external_weight": 0.8500000000000001,
|
||||
"hard_example_weight": 2.2,
|
||||
"l1_ratio": 0.2,
|
||||
"max_char_features": 85000,
|
||||
"max_word_features": 42000,
|
||||
"min_df": 1,
|
||||
"pseudo_weight": 0.375,
|
||||
"round": 15,
|
||||
"word_max": 3
|
||||
}
|
||||
},
|
||||
"openCorpus": "ModelTraining/ClipboardSemantics/open-training-corpus.jsonl",
|
||||
"reason": "Surrogate feature weights are not deployable NLModel assets and cannot bypass the existing macOS acceptance policy.",
|
||||
"requiredCommands": [
|
||||
"python3 Scripts/clipboard_semantics/generate_open_training_corpus.py",
|
||||
"xcrun swift Scripts/clipboard_semantics/train_models.swift --algorithms maxEnt --corpus ModelTraining/ClipboardSemantics/combined-training-corpus.jsonl",
|
||||
"xcrun swift Scripts/clipboard_semantics/evaluate_random_holdout.swift --corpus ModelTraining/ClipboardSemantics/random-holdout-corpus.jsonl"
|
||||
],
|
||||
"schemaVersion": 1,
|
||||
"seed": 20260827,
|
||||
"selectedRound": 1,
|
||||
"selectedRoundsByIntent": {
|
||||
"blessing": 19,
|
||||
"complaint": 1,
|
||||
"confirmationDecision": 3,
|
||||
"followUpReminder": 1,
|
||||
"invitation": 1,
|
||||
"question": 1,
|
||||
"replyableMessage": 1,
|
||||
"scheduleNegotiation": 7,
|
||||
"task": 15
|
||||
},
|
||||
"thresholds": {
|
||||
"blessing": {
|
||||
"abstained": true,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": true,
|
||||
"metrics": {
|
||||
"f1": 0.0,
|
||||
"falseNegative": 0,
|
||||
"falsePositive": 0,
|
||||
"precision": 0.0,
|
||||
"predictedPositive": 0,
|
||||
"recall": 0.0,
|
||||
"total": 0,
|
||||
"trueNegative": 0,
|
||||
"truePositive": 0,
|
||||
"wilsonPrecisionLower95": 0
|
||||
},
|
||||
"threshold": 1.0
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": true,
|
||||
"metrics": {
|
||||
"f1": 0.0,
|
||||
"falseNegative": 0,
|
||||
"falsePositive": 0,
|
||||
"precision": 0.0,
|
||||
"predictedPositive": 0,
|
||||
"recall": 0.0,
|
||||
"total": 0,
|
||||
"trueNegative": 0,
|
||||
"truePositive": 0,
|
||||
"wilsonPrecisionLower95": 0
|
||||
},
|
||||
"threshold": 1.0
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.0,
|
||||
"falseNegative": 0,
|
||||
"falsePositive": 0,
|
||||
"precision": 0.0,
|
||||
"predictedPositive": 0,
|
||||
"recall": 0.0,
|
||||
"total": 0,
|
||||
"trueNegative": 0,
|
||||
"truePositive": 0,
|
||||
"wilsonPrecisionLower95": 0
|
||||
},
|
||||
"threshold": 1.0
|
||||
},
|
||||
"complaint": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 1.0,
|
||||
"falseNegative": 0,
|
||||
"falsePositive": 0,
|
||||
"precision": 1.0,
|
||||
"predictedPositive": 180,
|
||||
"recall": 1.0,
|
||||
"total": 1485,
|
||||
"trueNegative": 1305,
|
||||
"truePositive": 180,
|
||||
"wilsonPrecisionLower95": 0.9791
|
||||
},
|
||||
"threshold": 0.5
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.9526,
|
||||
"falseNegative": 9,
|
||||
"falsePositive": 8,
|
||||
"precision": 0.9553,
|
||||
"predictedPositive": 179,
|
||||
"recall": 0.95,
|
||||
"total": 1485,
|
||||
"trueNegative": 1297,
|
||||
"truePositive": 171,
|
||||
"wilsonPrecisionLower95": 0.9143
|
||||
},
|
||||
"threshold": 0.54
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.9764,
|
||||
"falseNegative": 9,
|
||||
"falsePositive": 8,
|
||||
"precision": 0.9777,
|
||||
"predictedPositive": 359,
|
||||
"recall": 0.975,
|
||||
"total": 2970,
|
||||
"trueNegative": 2602,
|
||||
"truePositive": 351,
|
||||
"wilsonPrecisionLower95": 0.9567
|
||||
},
|
||||
"threshold": 0.54
|
||||
},
|
||||
"confirmationDecision": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.9727,
|
||||
"falseNegative": 1,
|
||||
"falsePositive": 4,
|
||||
"precision": 0.957,
|
||||
"predictedPositive": 93,
|
||||
"recall": 0.9889,
|
||||
"total": 1485,
|
||||
"trueNegative": 1391,
|
||||
"truePositive": 89,
|
||||
"wilsonPrecisionLower95": 0.8946
|
||||
},
|
||||
"threshold": 0.585
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.7815,
|
||||
"falseNegative": 31,
|
||||
"falsePositive": 2,
|
||||
"precision": 0.9672,
|
||||
"predictedPositive": 61,
|
||||
"recall": 0.6556,
|
||||
"total": 1485,
|
||||
"trueNegative": 1393,
|
||||
"truePositive": 59,
|
||||
"wilsonPrecisionLower95": 0.8881
|
||||
},
|
||||
"threshold": 0.5
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.8701,
|
||||
"falseNegative": 36,
|
||||
"falsePositive": 7,
|
||||
"precision": 0.9536,
|
||||
"predictedPositive": 151,
|
||||
"recall": 0.8,
|
||||
"total": 2970,
|
||||
"trueNegative": 2783,
|
||||
"truePositive": 144,
|
||||
"wilsonPrecisionLower95": 0.9074
|
||||
},
|
||||
"threshold": 0.555
|
||||
},
|
||||
"followUpReminder": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.8554,
|
||||
"falseNegative": 41,
|
||||
"falsePositive": 6,
|
||||
"precision": 0.9586,
|
||||
"predictedPositive": 145,
|
||||
"recall": 0.7722,
|
||||
"total": 1485,
|
||||
"trueNegative": 1299,
|
||||
"truePositive": 139,
|
||||
"wilsonPrecisionLower95": 0.9127
|
||||
},
|
||||
"threshold": 0.83
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.8354,
|
||||
"falseNegative": 48,
|
||||
"falsePositive": 4,
|
||||
"precision": 0.9706,
|
||||
"predictedPositive": 136,
|
||||
"recall": 0.7333,
|
||||
"total": 1485,
|
||||
"trueNegative": 1301,
|
||||
"truePositive": 132,
|
||||
"wilsonPrecisionLower95": 0.9268
|
||||
},
|
||||
"threshold": 0.505
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.7994,
|
||||
"falseNegative": 113,
|
||||
"falsePositive": 11,
|
||||
"precision": 0.9574,
|
||||
"predictedPositive": 258,
|
||||
"recall": 0.6861,
|
||||
"total": 2970,
|
||||
"trueNegative": 2599,
|
||||
"truePositive": 247,
|
||||
"wilsonPrecisionLower95": 0.9253
|
||||
},
|
||||
"threshold": 0.815
|
||||
},
|
||||
"invitation": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.8302,
|
||||
"falseNegative": 24,
|
||||
"falsePositive": 3,
|
||||
"precision": 0.9565,
|
||||
"predictedPositive": 69,
|
||||
"recall": 0.7333,
|
||||
"total": 1485,
|
||||
"trueNegative": 1392,
|
||||
"truePositive": 66,
|
||||
"wilsonPrecisionLower95": 0.8798
|
||||
},
|
||||
"threshold": 0.935
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.6765,
|
||||
"falseNegative": 44,
|
||||
"falsePositive": 0,
|
||||
"precision": 1.0,
|
||||
"predictedPositive": 46,
|
||||
"recall": 0.5111,
|
||||
"total": 1485,
|
||||
"trueNegative": 1395,
|
||||
"truePositive": 46,
|
||||
"wilsonPrecisionLower95": 0.9229
|
||||
},
|
||||
"threshold": 0.505
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.5476,
|
||||
"falseNegative": 111,
|
||||
"falsePositive": 3,
|
||||
"precision": 0.9583,
|
||||
"predictedPositive": 72,
|
||||
"recall": 0.3833,
|
||||
"total": 2970,
|
||||
"trueNegative": 2787,
|
||||
"truePositive": 69,
|
||||
"wilsonPrecisionLower95": 0.8845
|
||||
},
|
||||
"threshold": 0.935
|
||||
},
|
||||
"question": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.8445,
|
||||
"falseNegative": 76,
|
||||
"falsePositive": 12,
|
||||
"precision": 0.9522,
|
||||
"predictedPositive": 251,
|
||||
"recall": 0.7587,
|
||||
"total": 1485,
|
||||
"trueNegative": 1158,
|
||||
"truePositive": 239,
|
||||
"wilsonPrecisionLower95": 0.9183
|
||||
},
|
||||
"threshold": 0.725
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.8536,
|
||||
"falseNegative": 76,
|
||||
"falsePositive": 6,
|
||||
"precision": 0.9755,
|
||||
"predictedPositive": 245,
|
||||
"recall": 0.7587,
|
||||
"total": 1485,
|
||||
"trueNegative": 1164,
|
||||
"truePositive": 239,
|
||||
"wilsonPrecisionLower95": 0.9476
|
||||
},
|
||||
"threshold": 0.5
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.8729,
|
||||
"falseNegative": 122,
|
||||
"falsePositive": 26,
|
||||
"precision": 0.9513,
|
||||
"predictedPositive": 534,
|
||||
"recall": 0.8063,
|
||||
"total": 2970,
|
||||
"trueNegative": 2314,
|
||||
"truePositive": 508,
|
||||
"wilsonPrecisionLower95": 0.9296
|
||||
},
|
||||
"threshold": 0.515
|
||||
},
|
||||
"replyableMessage": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.948,
|
||||
"falseNegative": 52,
|
||||
"falsePositive": 46,
|
||||
"precision": 0.951,
|
||||
"predictedPositive": 939,
|
||||
"recall": 0.945,
|
||||
"total": 1485,
|
||||
"trueNegative": 494,
|
||||
"truePositive": 893,
|
||||
"wilsonPrecisionLower95": 0.9353
|
||||
},
|
||||
"threshold": 0.7
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.9424,
|
||||
"falseNegative": 62,
|
||||
"falsePositive": 46,
|
||||
"precision": 0.9505,
|
||||
"predictedPositive": 929,
|
||||
"recall": 0.9344,
|
||||
"total": 1485,
|
||||
"trueNegative": 494,
|
||||
"truePositive": 883,
|
||||
"wilsonPrecisionLower95": 0.9346
|
||||
},
|
||||
"threshold": 0.695
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.9454,
|
||||
"falseNegative": 115,
|
||||
"falsePositive": 90,
|
||||
"precision": 0.9517,
|
||||
"predictedPositive": 1865,
|
||||
"recall": 0.9392,
|
||||
"total": 2970,
|
||||
"trueNegative": 990,
|
||||
"truePositive": 1775,
|
||||
"wilsonPrecisionLower95": 0.9411
|
||||
},
|
||||
"threshold": 0.7
|
||||
},
|
||||
"scheduleNegotiation": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 1.0,
|
||||
"falseNegative": 0,
|
||||
"falsePositive": 0,
|
||||
"precision": 1.0,
|
||||
"predictedPositive": 45,
|
||||
"recall": 1.0,
|
||||
"total": 1485,
|
||||
"trueNegative": 1440,
|
||||
"truePositive": 45,
|
||||
"wilsonPrecisionLower95": 0.9213
|
||||
},
|
||||
"threshold": 0.66
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 1.0,
|
||||
"falseNegative": 0,
|
||||
"falsePositive": 0,
|
||||
"precision": 1.0,
|
||||
"predictedPositive": 45,
|
||||
"recall": 1.0,
|
||||
"total": 1485,
|
||||
"trueNegative": 1440,
|
||||
"truePositive": 45,
|
||||
"wilsonPrecisionLower95": 0.9213
|
||||
},
|
||||
"threshold": 0.55
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 1.0,
|
||||
"falseNegative": 0,
|
||||
"falsePositive": 0,
|
||||
"precision": 1.0,
|
||||
"predictedPositive": 90,
|
||||
"recall": 1.0,
|
||||
"total": 2970,
|
||||
"trueNegative": 2880,
|
||||
"truePositive": 90,
|
||||
"wilsonPrecisionLower95": 0.9591
|
||||
},
|
||||
"threshold": 0.66
|
||||
},
|
||||
"task": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.8593,
|
||||
"falseNegative": 79,
|
||||
"falsePositive": 13,
|
||||
"precision": 0.9558,
|
||||
"predictedPositive": 294,
|
||||
"recall": 0.7806,
|
||||
"total": 1485,
|
||||
"trueNegative": 1112,
|
||||
"truePositive": 281,
|
||||
"wilsonPrecisionLower95": 0.9258
|
||||
},
|
||||
"threshold": 0.98
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.8166,
|
||||
"falseNegative": 104,
|
||||
"falsePositive": 11,
|
||||
"precision": 0.9588,
|
||||
"predictedPositive": 267,
|
||||
"recall": 0.7111,
|
||||
"total": 1485,
|
||||
"trueNegative": 1114,
|
||||
"truePositive": 256,
|
||||
"wilsonPrecisionLower95": 0.9277
|
||||
},
|
||||
"threshold": 0.96
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.8434,
|
||||
"falseNegative": 176,
|
||||
"falsePositive": 26,
|
||||
"precision": 0.9544,
|
||||
"predictedPositive": 570,
|
||||
"recall": 0.7556,
|
||||
"total": 2970,
|
||||
"trueNegative": 2224,
|
||||
"truePositive": 544,
|
||||
"wilsonPrecisionLower95": 0.934
|
||||
},
|
||||
"threshold": 0.965
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,481 @@
|
||||
{
|
||||
"automaticPromotionAllowed": false,
|
||||
"baseCorpus": "ModelTraining/ClipboardSemantics/clipboard_semantic_corpus.jsonl",
|
||||
"configuration": {
|
||||
"alpha": 2e-05,
|
||||
"augmentation": "none",
|
||||
"char_max": 5,
|
||||
"char_min": 2,
|
||||
"external_weight": 0.5700000000000001,
|
||||
"hard_example_weight": 1.0,
|
||||
"l1_ratio": 0.0,
|
||||
"max_char_features": 45000,
|
||||
"max_word_features": 22000,
|
||||
"min_df": 2,
|
||||
"pseudo_weight": 0.16499999999999998,
|
||||
"round": 1,
|
||||
"word_max": 2
|
||||
},
|
||||
"openCorpus": "ModelTraining/ClipboardSemantics/open-training-corpus.jsonl",
|
||||
"reason": "Surrogate feature weights are not deployable NLModel assets and cannot bypass the existing macOS acceptance policy.",
|
||||
"requiredCommands": [
|
||||
"python3 Scripts/clipboard_semantics/generate_open_training_corpus.py",
|
||||
"xcrun swift Scripts/clipboard_semantics/train_models.swift --algorithms maxEnt --corpus ModelTraining/ClipboardSemantics/combined-training-corpus.jsonl",
|
||||
"xcrun swift Scripts/clipboard_semantics/evaluate_random_holdout.swift --corpus ModelTraining/ClipboardSemantics/random-holdout-corpus.jsonl"
|
||||
],
|
||||
"schemaVersion": 1,
|
||||
"seed": 20260827,
|
||||
"selectedRound": 1,
|
||||
"thresholds": {
|
||||
"blessing": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.9318,
|
||||
"falseNegative": 4,
|
||||
"falsePositive": 2,
|
||||
"precision": 0.9535,
|
||||
"predictedPositive": 43,
|
||||
"recall": 0.9111,
|
||||
"total": 1485,
|
||||
"trueNegative": 1438,
|
||||
"truePositive": 41,
|
||||
"wilsonPrecisionLower95": 0.8454
|
||||
},
|
||||
"threshold": 0.915
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.7568,
|
||||
"falseNegative": 17,
|
||||
"falsePositive": 1,
|
||||
"precision": 0.9655,
|
||||
"predictedPositive": 29,
|
||||
"recall": 0.6222,
|
||||
"total": 1485,
|
||||
"trueNegative": 1439,
|
||||
"truePositive": 28,
|
||||
"wilsonPrecisionLower95": 0.8282
|
||||
},
|
||||
"threshold": 0.9
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.8375,
|
||||
"falseNegative": 23,
|
||||
"falsePositive": 3,
|
||||
"precision": 0.9571,
|
||||
"predictedPositive": 70,
|
||||
"recall": 0.7444,
|
||||
"total": 2970,
|
||||
"trueNegative": 2877,
|
||||
"truePositive": 67,
|
||||
"wilsonPrecisionLower95": 0.8814
|
||||
},
|
||||
"threshold": 0.915
|
||||
},
|
||||
"complaint": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 1.0,
|
||||
"falseNegative": 0,
|
||||
"falsePositive": 0,
|
||||
"precision": 1.0,
|
||||
"predictedPositive": 180,
|
||||
"recall": 1.0,
|
||||
"total": 1485,
|
||||
"trueNegative": 1305,
|
||||
"truePositive": 180,
|
||||
"wilsonPrecisionLower95": 0.9791
|
||||
},
|
||||
"threshold": 0.5
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.9526,
|
||||
"falseNegative": 9,
|
||||
"falsePositive": 8,
|
||||
"precision": 0.9553,
|
||||
"predictedPositive": 179,
|
||||
"recall": 0.95,
|
||||
"total": 1485,
|
||||
"trueNegative": 1297,
|
||||
"truePositive": 171,
|
||||
"wilsonPrecisionLower95": 0.9143
|
||||
},
|
||||
"threshold": 0.54
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.9764,
|
||||
"falseNegative": 9,
|
||||
"falsePositive": 8,
|
||||
"precision": 0.9777,
|
||||
"predictedPositive": 359,
|
||||
"recall": 0.975,
|
||||
"total": 2970,
|
||||
"trueNegative": 2602,
|
||||
"truePositive": 351,
|
||||
"wilsonPrecisionLower95": 0.9567
|
||||
},
|
||||
"threshold": 0.54
|
||||
},
|
||||
"confirmationDecision": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.8848,
|
||||
"falseNegative": 17,
|
||||
"falsePositive": 2,
|
||||
"precision": 0.9733,
|
||||
"predictedPositive": 75,
|
||||
"recall": 0.8111,
|
||||
"total": 1485,
|
||||
"trueNegative": 1393,
|
||||
"truePositive": 73,
|
||||
"wilsonPrecisionLower95": 0.9079
|
||||
},
|
||||
"threshold": 0.5
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.7947,
|
||||
"falseNegative": 30,
|
||||
"falsePositive": 1,
|
||||
"precision": 0.9836,
|
||||
"predictedPositive": 61,
|
||||
"recall": 0.6667,
|
||||
"total": 1485,
|
||||
"trueNegative": 1394,
|
||||
"truePositive": 60,
|
||||
"wilsonPrecisionLower95": 0.9128
|
||||
},
|
||||
"threshold": 0.5
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.8418,
|
||||
"falseNegative": 47,
|
||||
"falsePositive": 3,
|
||||
"precision": 0.9779,
|
||||
"predictedPositive": 136,
|
||||
"recall": 0.7389,
|
||||
"total": 2970,
|
||||
"trueNegative": 2787,
|
||||
"truePositive": 133,
|
||||
"wilsonPrecisionLower95": 0.9372
|
||||
},
|
||||
"threshold": 0.5
|
||||
},
|
||||
"followUpReminder": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.8554,
|
||||
"falseNegative": 41,
|
||||
"falsePositive": 6,
|
||||
"precision": 0.9586,
|
||||
"predictedPositive": 145,
|
||||
"recall": 0.7722,
|
||||
"total": 1485,
|
||||
"trueNegative": 1299,
|
||||
"truePositive": 139,
|
||||
"wilsonPrecisionLower95": 0.9127
|
||||
},
|
||||
"threshold": 0.83
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.8354,
|
||||
"falseNegative": 48,
|
||||
"falsePositive": 4,
|
||||
"precision": 0.9706,
|
||||
"predictedPositive": 136,
|
||||
"recall": 0.7333,
|
||||
"total": 1485,
|
||||
"trueNegative": 1301,
|
||||
"truePositive": 132,
|
||||
"wilsonPrecisionLower95": 0.9268
|
||||
},
|
||||
"threshold": 0.505
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.7994,
|
||||
"falseNegative": 113,
|
||||
"falsePositive": 11,
|
||||
"precision": 0.9574,
|
||||
"predictedPositive": 258,
|
||||
"recall": 0.6861,
|
||||
"total": 2970,
|
||||
"trueNegative": 2599,
|
||||
"truePositive": 247,
|
||||
"wilsonPrecisionLower95": 0.9253
|
||||
},
|
||||
"threshold": 0.815
|
||||
},
|
||||
"invitation": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.8302,
|
||||
"falseNegative": 24,
|
||||
"falsePositive": 3,
|
||||
"precision": 0.9565,
|
||||
"predictedPositive": 69,
|
||||
"recall": 0.7333,
|
||||
"total": 1485,
|
||||
"trueNegative": 1392,
|
||||
"truePositive": 66,
|
||||
"wilsonPrecisionLower95": 0.8798
|
||||
},
|
||||
"threshold": 0.935
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.6765,
|
||||
"falseNegative": 44,
|
||||
"falsePositive": 0,
|
||||
"precision": 1.0,
|
||||
"predictedPositive": 46,
|
||||
"recall": 0.5111,
|
||||
"total": 1485,
|
||||
"trueNegative": 1395,
|
||||
"truePositive": 46,
|
||||
"wilsonPrecisionLower95": 0.9229
|
||||
},
|
||||
"threshold": 0.505
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.5476,
|
||||
"falseNegative": 111,
|
||||
"falsePositive": 3,
|
||||
"precision": 0.9583,
|
||||
"predictedPositive": 72,
|
||||
"recall": 0.3833,
|
||||
"total": 2970,
|
||||
"trueNegative": 2787,
|
||||
"truePositive": 69,
|
||||
"wilsonPrecisionLower95": 0.8845
|
||||
},
|
||||
"threshold": 0.935
|
||||
},
|
||||
"question": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.8445,
|
||||
"falseNegative": 76,
|
||||
"falsePositive": 12,
|
||||
"precision": 0.9522,
|
||||
"predictedPositive": 251,
|
||||
"recall": 0.7587,
|
||||
"total": 1485,
|
||||
"trueNegative": 1158,
|
||||
"truePositive": 239,
|
||||
"wilsonPrecisionLower95": 0.9183
|
||||
},
|
||||
"threshold": 0.725
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.8536,
|
||||
"falseNegative": 76,
|
||||
"falsePositive": 6,
|
||||
"precision": 0.9755,
|
||||
"predictedPositive": 245,
|
||||
"recall": 0.7587,
|
||||
"total": 1485,
|
||||
"trueNegative": 1164,
|
||||
"truePositive": 239,
|
||||
"wilsonPrecisionLower95": 0.9476
|
||||
},
|
||||
"threshold": 0.5
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.8729,
|
||||
"falseNegative": 122,
|
||||
"falsePositive": 26,
|
||||
"precision": 0.9513,
|
||||
"predictedPositive": 534,
|
||||
"recall": 0.8063,
|
||||
"total": 2970,
|
||||
"trueNegative": 2314,
|
||||
"truePositive": 508,
|
||||
"wilsonPrecisionLower95": 0.9296
|
||||
},
|
||||
"threshold": 0.515
|
||||
},
|
||||
"replyableMessage": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.948,
|
||||
"falseNegative": 52,
|
||||
"falsePositive": 46,
|
||||
"precision": 0.951,
|
||||
"predictedPositive": 939,
|
||||
"recall": 0.945,
|
||||
"total": 1485,
|
||||
"trueNegative": 494,
|
||||
"truePositive": 893,
|
||||
"wilsonPrecisionLower95": 0.9353
|
||||
},
|
||||
"threshold": 0.7
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.9424,
|
||||
"falseNegative": 62,
|
||||
"falsePositive": 46,
|
||||
"precision": 0.9505,
|
||||
"predictedPositive": 929,
|
||||
"recall": 0.9344,
|
||||
"total": 1485,
|
||||
"trueNegative": 494,
|
||||
"truePositive": 883,
|
||||
"wilsonPrecisionLower95": 0.9346
|
||||
},
|
||||
"threshold": 0.695
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.9454,
|
||||
"falseNegative": 115,
|
||||
"falsePositive": 90,
|
||||
"precision": 0.9517,
|
||||
"predictedPositive": 1865,
|
||||
"recall": 0.9392,
|
||||
"total": 2970,
|
||||
"trueNegative": 990,
|
||||
"truePositive": 1775,
|
||||
"wilsonPrecisionLower95": 0.9411
|
||||
},
|
||||
"threshold": 0.7
|
||||
},
|
||||
"scheduleNegotiation": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 1.0,
|
||||
"falseNegative": 0,
|
||||
"falsePositive": 0,
|
||||
"precision": 1.0,
|
||||
"predictedPositive": 45,
|
||||
"recall": 1.0,
|
||||
"total": 1485,
|
||||
"trueNegative": 1440,
|
||||
"truePositive": 45,
|
||||
"wilsonPrecisionLower95": 0.9213
|
||||
},
|
||||
"threshold": 0.5
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.967,
|
||||
"falseNegative": 1,
|
||||
"falsePositive": 2,
|
||||
"precision": 0.9565,
|
||||
"predictedPositive": 46,
|
||||
"recall": 0.9778,
|
||||
"total": 1485,
|
||||
"trueNegative": 1438,
|
||||
"truePositive": 44,
|
||||
"wilsonPrecisionLower95": 0.8547
|
||||
},
|
||||
"threshold": 0.93
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.978,
|
||||
"falseNegative": 1,
|
||||
"falsePositive": 3,
|
||||
"precision": 0.9674,
|
||||
"predictedPositive": 92,
|
||||
"recall": 0.9889,
|
||||
"total": 2970,
|
||||
"trueNegative": 2877,
|
||||
"truePositive": 89,
|
||||
"wilsonPrecisionLower95": 0.9085
|
||||
},
|
||||
"threshold": 0.91
|
||||
},
|
||||
"task": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.8403,
|
||||
"falseNegative": 89,
|
||||
"falsePositive": 14,
|
||||
"precision": 0.9509,
|
||||
"predictedPositive": 285,
|
||||
"recall": 0.7528,
|
||||
"total": 1485,
|
||||
"trueNegative": 1111,
|
||||
"truePositive": 271,
|
||||
"wilsonPrecisionLower95": 0.9192
|
||||
},
|
||||
"threshold": 0.835
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.8278,
|
||||
"falseNegative": 98,
|
||||
"falsePositive": 11,
|
||||
"precision": 0.9597,
|
||||
"predictedPositive": 273,
|
||||
"recall": 0.7278,
|
||||
"total": 1485,
|
||||
"trueNegative": 1114,
|
||||
"truePositive": 262,
|
||||
"wilsonPrecisionLower95": 0.9293
|
||||
},
|
||||
"threshold": 0.745
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.8637,
|
||||
"falseNegative": 153,
|
||||
"falsePositive": 26,
|
||||
"precision": 0.9562,
|
||||
"predictedPositive": 593,
|
||||
"recall": 0.7875,
|
||||
"total": 2970,
|
||||
"trueNegative": 2224,
|
||||
"truePositive": 567,
|
||||
"wilsonPrecisionLower95": 0.9365
|
||||
},
|
||||
"threshold": 0.755
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"completedResearchRounds": 60,
|
||||
"requestedCoreRounds": 20,
|
||||
"seed": 20260827,
|
||||
"data": {
|
||||
"baseTrainingRecords": 11500,
|
||||
"reachableLicensedOpenRecords": 3845,
|
||||
"holdoutOverlapCounts": {
|
||||
"comprehensiveOnline": 0,
|
||||
"random": 0,
|
||||
"targetedRelease": 0
|
||||
},
|
||||
"unavailableSources": [
|
||||
"MASSIVE",
|
||||
"CLINC150",
|
||||
"CFPB"
|
||||
]
|
||||
},
|
||||
"phases": [
|
||||
{
|
||||
"name": "shared-configuration-search",
|
||||
"rounds": 20,
|
||||
"selectedRound": 1,
|
||||
"golden": {
|
||||
"macroPrecision": 0.9366,
|
||||
"macroRecall": 0.6801,
|
||||
"macroF1": 0.7711
|
||||
},
|
||||
"randomHoldout": {
|
||||
"macroPrecision": 0.8569,
|
||||
"macroRecall": 0.4603,
|
||||
"macroF1": 0.5709
|
||||
},
|
||||
"researchOnlyComprehensive": {
|
||||
"macroPrecision": 0.4398,
|
||||
"macroRecall": 0.2161,
|
||||
"macroF1": 0.2383
|
||||
},
|
||||
"targetedRelease": {
|
||||
"macroPrecision": 0.5489,
|
||||
"macroRecall": 0.3139,
|
||||
"macroF1": 0.3899
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "per-intent-fine-tuning",
|
||||
"rounds": 20,
|
||||
"selectedRoundsByIntent": {
|
||||
"blessing": 19,
|
||||
"complaint": 1,
|
||||
"confirmationDecision": 3,
|
||||
"followUpReminder": 1,
|
||||
"invitation": 1,
|
||||
"question": 1,
|
||||
"replyableMessage": 1,
|
||||
"scheduleNegotiation": 7,
|
||||
"task": 15
|
||||
},
|
||||
"golden": {
|
||||
"macroPrecision": 0.9377,
|
||||
"macroRecall": 0.6819,
|
||||
"macroF1": 0.7666
|
||||
},
|
||||
"randomHoldout": {
|
||||
"macroPrecision": 0.8701,
|
||||
"macroRecall": 0.4705,
|
||||
"macroF1": 0.5770
|
||||
},
|
||||
"researchOnlyComprehensive": {
|
||||
"macroPrecision": 0.4292,
|
||||
"macroRecall": 0.2250,
|
||||
"macroF1": 0.2645
|
||||
},
|
||||
"targetedRelease": {
|
||||
"macroPrecision": 0.5467,
|
||||
"macroRecall": 0.3375,
|
||||
"macroF1": 0.4095
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "explicit-evidence-fine-tuning",
|
||||
"rounds": 20,
|
||||
"selectedRound": 1,
|
||||
"golden": {
|
||||
"macroPrecision": 0.9877,
|
||||
"macroRecall": 0.4354,
|
||||
"macroF1": 0.5476
|
||||
},
|
||||
"randomHoldout": {
|
||||
"macroPrecision": 0.7731,
|
||||
"macroRecall": 0.3029,
|
||||
"macroF1": 0.3905
|
||||
},
|
||||
"researchOnlyComprehensive": {
|
||||
"macroPrecision": 0.6097,
|
||||
"macroRecall": 0.1647,
|
||||
"macroF1": 0.2325
|
||||
},
|
||||
"targetedRelease": {
|
||||
"macroPrecision": 0.5489,
|
||||
"macroRecall": 0.2537,
|
||||
"macroF1": 0.3209
|
||||
}
|
||||
}
|
||||
],
|
||||
"currentDeployedReference": {
|
||||
"randomHoldout": {
|
||||
"macroPrecision": 0.9832,
|
||||
"macroRecall": 0.6557,
|
||||
"macroF1": 0.7669
|
||||
},
|
||||
"researchOnlyComprehensive": {
|
||||
"macroPrecision": 0.2948,
|
||||
"macroRecall": 0.3320,
|
||||
"macroF1": 0.2783
|
||||
}
|
||||
},
|
||||
"findings": [
|
||||
"The first low-complexity round generalized better than later self-training rounds.",
|
||||
"Three-round high-confidence pseudo-labeling amplified source and template bias instead of improving frozen regressions.",
|
||||
"Per-intent configuration selection improved the research-only comprehensive F1 from 0.2383 to 0.2645 but remained below the deployed reference F1 of 0.2783.",
|
||||
"Explicit evidence gates raised golden precision to 0.9877 and research-only precision to 0.6097, but recall collapsed and the release gate still failed.",
|
||||
"No surrogate candidate is eligible to replace or augment the deployed Apple models."
|
||||
],
|
||||
"releaseGatePassed": false,
|
||||
"deploymentDecision": "retain-current-deployed-models",
|
||||
"platformLimitation": "Linux cannot train, compile, run, or benchmark the deployable Apple Create ML models. These rounds are surrogate research and produce a macOS replay specification, not app-ready model assets."
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -122,6 +122,48 @@ Without human-reviewed gold labels, these results measure agreement with the
|
||||
model committee, not production truth. A passing verifier may enter shadow
|
||||
deployment, but cannot be described as having 95% real-user precision.
|
||||
|
||||
## Iterative weakly supervised research
|
||||
|
||||
The portable research harness runs a fixed 20-round matrix over bilingual
|
||||
character/word n-grams, sparse logistic classifiers, class balancing,
|
||||
deterministic label-preserving augmentation, hard-example weighting, and
|
||||
three-round high-confidence self-training consensus:
|
||||
|
||||
```bash
|
||||
python3 -m pip install -r \
|
||||
Scripts/clipboard_semantics/requirements-research.txt
|
||||
python3 Scripts/clipboard_semantics/generate_open_training_corpus.py \
|
||||
--allow-unavailable-sources
|
||||
python3 Scripts/clipboard_semantics/run_iterative_retraining.py
|
||||
```
|
||||
|
||||
The harness fits thresholds only on the generated validation split. Synthetic
|
||||
test, golden, random, targeted-release, and comprehensive online corpora never
|
||||
enter training or threshold calibration. Exact normalized overlap with every
|
||||
frozen holdout is a fatal error.
|
||||
|
||||
The 2026-08-27 study completed the requested 20 rounds and two additional
|
||||
20-round fine-tuning phases after the first phase missed its release target:
|
||||
|
||||
- The shared-configuration phase selected round 1. Random-holdout macro F1 was
|
||||
`0.5709`; research-only comprehensive macro F1 was `0.2383`.
|
||||
- Per-intent selection improved those values to `0.5770` and `0.2645`.
|
||||
- Explicit evidence gates raised golden macro precision to `0.9877` and
|
||||
research-only precision to `0.6097`, but reduced recall too severely.
|
||||
- The current deployed reference remains stronger: random-holdout macro F1
|
||||
`0.7669` and research-only comprehensive macro F1 `0.2783`.
|
||||
|
||||
All three phases failed the release gate, so no model was promoted. Detailed
|
||||
rounds, final evaluations, source breakdowns, and macOS replay instructions are
|
||||
stored under `IterativeResearch/`.
|
||||
|
||||
This harness is deliberately a Linux surrogate. It cannot emit the
|
||||
`NLModel`-compatible Create ML artifacts used by the keyboard extension.
|
||||
Deployable training, Core ML compilation, simulator regression, latency, and
|
||||
memory checks still require macOS with Xcode 26+. A surrogate result can
|
||||
nominate a data/threshold policy for macOS replay, but cannot authorize
|
||||
automatic deployment.
|
||||
|
||||
## Deployment decision
|
||||
|
||||
Only maxEnt models are trained and deployed because they are self-contained in
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
{
|
||||
"baseCorpusRecords": 17692,
|
||||
"combinedCorpusSHA256": "70d1547b41b08e4417cfa86ad93dd5637b557d174c6503575c44fbf152cc3ee5",
|
||||
"combinedRecords": 23918,
|
||||
"combinedCorpusSHA256": "f111bc7425912213082e01038b52dd3d25fe21418d156f532da8f01f27fa16e4",
|
||||
"combinedRecords": 21537,
|
||||
"excludedBaseOverlap": 6,
|
||||
"excludedHoldoutOverlap": 151,
|
||||
"excludedHoldoutOverlap": 72,
|
||||
"excludedSources": [
|
||||
{
|
||||
"dataset": "MIDAS",
|
||||
@@ -38,72 +38,11 @@
|
||||
"reason": "No clean official train split independent from the frozen holdout."
|
||||
}
|
||||
],
|
||||
"openTrainingRecords": 6226,
|
||||
"openTrainingRecords": 3845,
|
||||
"policy": "Licensed official training splits only. Per-record knownLabels prevent unannotated intents from becoming false negatives. Exact normalized text overlap with every local *holdout-corpus.jsonl file is excluded.",
|
||||
"schemaVersion": 1,
|
||||
"seed": 20260827,
|
||||
"sources": [
|
||||
{
|
||||
"dataset": "AmazonScience/MASSIVE en-US",
|
||||
"knownLabelCounts": {
|
||||
"followUpReminder": 7,
|
||||
"question": 223,
|
||||
"sentiment": 4,
|
||||
"task": 143
|
||||
},
|
||||
"languages": {
|
||||
"en": 498
|
||||
},
|
||||
"license": "CC-BY-4.0",
|
||||
"records": 498,
|
||||
"revision": "1.1",
|
||||
"url": "https://huggingface.co/datasets/AmazonScience/massive"
|
||||
},
|
||||
{
|
||||
"dataset": "AmazonScience/MASSIVE zh-CN",
|
||||
"knownLabelCounts": {
|
||||
"followUpReminder": 3,
|
||||
"question": 212,
|
||||
"sentiment": 6,
|
||||
"task": 165
|
||||
},
|
||||
"languages": {
|
||||
"zh-Hans": 487
|
||||
},
|
||||
"license": "CC-BY-4.0",
|
||||
"records": 487,
|
||||
"revision": "1.1",
|
||||
"url": "https://huggingface.co/datasets/AmazonScience/massive"
|
||||
},
|
||||
{
|
||||
"dataset": "CFPB via hpe-ai/customer-complaints",
|
||||
"knownLabelCounts": {
|
||||
"complaint": 898,
|
||||
"sentiment": 898
|
||||
},
|
||||
"languages": {
|
||||
"en": 898
|
||||
},
|
||||
"license": "CC0-1.0 source / Apache-2.0 mirror",
|
||||
"records": 898,
|
||||
"revision": "e5fec64e1f0688e47699b9cf8c26fe4ed350123a",
|
||||
"url": "https://www.consumerfinance.gov/data-research/consumer-complaints/search/detail/4696537"
|
||||
},
|
||||
{
|
||||
"dataset": "CLINC150 UCI",
|
||||
"knownLabelCounts": {
|
||||
"followUpReminder": 5,
|
||||
"question": 308,
|
||||
"task": 143
|
||||
},
|
||||
"languages": {
|
||||
"en": 498
|
||||
},
|
||||
"license": "CC-BY-3.0",
|
||||
"records": 498,
|
||||
"revision": "UCI-570-2020-05-07",
|
||||
"url": "https://archive.ics.uci.edu/dataset/570/clinc150"
|
||||
},
|
||||
{
|
||||
"dataset": "CrossWOZ",
|
||||
"knownLabelCounts": {
|
||||
@@ -121,13 +60,13 @@
|
||||
{
|
||||
"dataset": "Google Research GoEmotions",
|
||||
"knownLabelCounts": {
|
||||
"sentiment": 799
|
||||
"sentiment": 798
|
||||
},
|
||||
"languages": {
|
||||
"en": 1199
|
||||
"en": 1198
|
||||
},
|
||||
"license": "Apache-2.0",
|
||||
"records": 1199,
|
||||
"records": 1198,
|
||||
"revision": "5d8f4ac97c873bde3a792ba4628f00bb9103d3e6",
|
||||
"url": "https://github.com/google-research/google-research/blob/5d8f4ac97c873bde3a792ba4628f00bb9103d3e6/goemotions/data/train.tsv"
|
||||
},
|
||||
@@ -164,17 +103,31 @@
|
||||
{
|
||||
"dataset": "MultiDoGO",
|
||||
"knownLabelCounts": {
|
||||
"confirmationDecision": 28,
|
||||
"confirmationDecision": 27,
|
||||
"question": 83,
|
||||
"task": 222
|
||||
},
|
||||
"languages": {
|
||||
"en": 464
|
||||
"en": 465
|
||||
},
|
||||
"license": "CDLA-Permissive-1.0",
|
||||
"records": 464,
|
||||
"records": 465,
|
||||
"revision": "baa30639c4b271f394b81443c842193407cdf26d",
|
||||
"url": "https://github.com/awslabs/multi-domain-goal-oriented-dialogues-dataset/blob/baa30639c4b271f394b81443c842193407cdf26d/data/paper_splits/splits_annotated_at_turn_level/insurance/train.tsv"
|
||||
"url": "https://github.com/awslabs/multi-domain-goal-oriented-dialogues-dataset/blob/baa30639c4b271f394b81443c842193407cdf26d/data/paper_splits/splits_annotated_at_turn_level/software/train.tsv"
|
||||
}
|
||||
],
|
||||
"unavailableSources": [
|
||||
{
|
||||
"dataset": "MASSIVE",
|
||||
"reason": "URLError: <urlopen error [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1000)>"
|
||||
},
|
||||
{
|
||||
"dataset": "CLINC150",
|
||||
"reason": "URLError: <urlopen error [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1000)>"
|
||||
},
|
||||
{
|
||||
"dataset": "CFPB",
|
||||
"reason": "URLError: <urlopen error [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1000)>"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user