docs(semantics): record iterative retraining study
Co-authored-by: Rocky <hkgood@users.noreply.github.com>
This commit is contained in:
Cursor Agent
+8422
File diff suppressed because it is too large
Load Diff
+629
@@ -0,0 +1,629 @@
|
||||
{
|
||||
"automaticPromotionAllowed": false,
|
||||
"baseCorpus": "ModelTraining/ClipboardSemantics/clipboard_semantic_corpus.jsonl",
|
||||
"configuration": {
|
||||
"alpha": 2e-05,
|
||||
"augmentation": "none",
|
||||
"char_max": 5,
|
||||
"char_min": 2,
|
||||
"external_weight": 0.5700000000000001,
|
||||
"hard_example_weight": 1.0,
|
||||
"l1_ratio": 0.0,
|
||||
"max_char_features": 45000,
|
||||
"max_word_features": 22000,
|
||||
"min_df": 2,
|
||||
"pseudo_weight": 0.16499999999999998,
|
||||
"round": 1,
|
||||
"word_max": 2
|
||||
},
|
||||
"configurationsByIntent": {
|
||||
"blessing": {
|
||||
"alpha": 1.2e-06,
|
||||
"augmentation": "numeric",
|
||||
"char_max": 8,
|
||||
"char_min": 3,
|
||||
"external_weight": 0.93,
|
||||
"hard_example_weight": 2.7,
|
||||
"l1_ratio": 0.25,
|
||||
"max_char_features": 95000,
|
||||
"max_word_features": 45000,
|
||||
"min_df": 1,
|
||||
"pseudo_weight": 0.43499999999999994,
|
||||
"round": 19,
|
||||
"word_max": 3
|
||||
},
|
||||
"complaint": {
|
||||
"alpha": 2e-05,
|
||||
"augmentation": "none",
|
||||
"char_max": 5,
|
||||
"char_min": 2,
|
||||
"external_weight": 0.5700000000000001,
|
||||
"hard_example_weight": 1.0,
|
||||
"l1_ratio": 0.0,
|
||||
"max_char_features": 45000,
|
||||
"max_word_features": 22000,
|
||||
"min_df": 2,
|
||||
"pseudo_weight": 0.16499999999999998,
|
||||
"round": 1,
|
||||
"word_max": 2
|
||||
},
|
||||
"confirmationDecision": {
|
||||
"alpha": 5e-06,
|
||||
"augmentation": "prefix",
|
||||
"char_max": 6,
|
||||
"char_min": 2,
|
||||
"external_weight": 0.77,
|
||||
"hard_example_weight": 1.8,
|
||||
"l1_ratio": 0.1,
|
||||
"max_char_features": 70000,
|
||||
"max_word_features": 30000,
|
||||
"min_df": 2,
|
||||
"pseudo_weight": 0.31499999999999995,
|
||||
"round": 11,
|
||||
"word_max": 2
|
||||
},
|
||||
"followUpReminder": {
|
||||
"alpha": 2.5e-06,
|
||||
"augmentation": "prefix",
|
||||
"char_max": 7,
|
||||
"char_min": 3,
|
||||
"external_weight": 0.8300000000000001,
|
||||
"hard_example_weight": 2.2,
|
||||
"l1_ratio": 0.2,
|
||||
"max_char_features": 80000,
|
||||
"max_word_features": 40000,
|
||||
"min_df": 1,
|
||||
"pseudo_weight": 0.36,
|
||||
"round": 14,
|
||||
"word_max": 3
|
||||
},
|
||||
"invitation": {
|
||||
"alpha": 2e-05,
|
||||
"augmentation": "none",
|
||||
"char_max": 5,
|
||||
"char_min": 2,
|
||||
"external_weight": 0.5700000000000001,
|
||||
"hard_example_weight": 1.0,
|
||||
"l1_ratio": 0.0,
|
||||
"max_char_features": 45000,
|
||||
"max_word_features": 22000,
|
||||
"min_df": 2,
|
||||
"pseudo_weight": 0.16499999999999998,
|
||||
"round": 1,
|
||||
"word_max": 2
|
||||
},
|
||||
"question": {
|
||||
"alpha": 2e-05,
|
||||
"augmentation": "none",
|
||||
"char_max": 5,
|
||||
"char_min": 2,
|
||||
"external_weight": 0.5700000000000001,
|
||||
"hard_example_weight": 1.0,
|
||||
"l1_ratio": 0.0,
|
||||
"max_char_features": 45000,
|
||||
"max_word_features": 22000,
|
||||
"min_df": 2,
|
||||
"pseudo_weight": 0.16499999999999998,
|
||||
"round": 1,
|
||||
"word_max": 2
|
||||
},
|
||||
"replyableMessage": {
|
||||
"alpha": 2e-05,
|
||||
"augmentation": "none",
|
||||
"char_max": 5,
|
||||
"char_min": 2,
|
||||
"external_weight": 0.5700000000000001,
|
||||
"hard_example_weight": 1.0,
|
||||
"l1_ratio": 0.0,
|
||||
"max_char_features": 45000,
|
||||
"max_word_features": 22000,
|
||||
"min_df": 2,
|
||||
"pseudo_weight": 0.16499999999999998,
|
||||
"round": 1,
|
||||
"word_max": 2
|
||||
},
|
||||
"scheduleNegotiation": {
|
||||
"alpha": 6e-06,
|
||||
"augmentation": "punctuation",
|
||||
"char_max": 6,
|
||||
"char_min": 3,
|
||||
"external_weight": 0.6900000000000001,
|
||||
"hard_example_weight": 1.5,
|
||||
"l1_ratio": 0.1,
|
||||
"max_char_features": 65000,
|
||||
"max_word_features": 32000,
|
||||
"min_df": 2,
|
||||
"pseudo_weight": 0.255,
|
||||
"round": 7,
|
||||
"word_max": 3
|
||||
},
|
||||
"task": {
|
||||
"alpha": 2e-05,
|
||||
"augmentation": "none",
|
||||
"char_max": 5,
|
||||
"char_min": 2,
|
||||
"external_weight": 0.5700000000000001,
|
||||
"hard_example_weight": 1.0,
|
||||
"l1_ratio": 0.0,
|
||||
"max_char_features": 45000,
|
||||
"max_word_features": 22000,
|
||||
"min_df": 2,
|
||||
"pseudo_weight": 0.16499999999999998,
|
||||
"round": 1,
|
||||
"word_max": 2
|
||||
}
|
||||
},
|
||||
"openCorpus": "ModelTraining/ClipboardSemantics/open-training-corpus.jsonl",
|
||||
"reason": "Surrogate feature weights are not deployable NLModel assets and cannot bypass the existing macOS acceptance policy.",
|
||||
"requiredCommands": [
|
||||
"python3 Scripts/clipboard_semantics/generate_open_training_corpus.py",
|
||||
"xcrun swift Scripts/clipboard_semantics/train_models.swift --algorithms maxEnt --corpus ModelTraining/ClipboardSemantics/combined-training-corpus.jsonl",
|
||||
"xcrun swift Scripts/clipboard_semantics/evaluate_random_holdout.swift --corpus ModelTraining/ClipboardSemantics/random-holdout-corpus.jsonl"
|
||||
],
|
||||
"schemaVersion": 1,
|
||||
"seed": 20260827,
|
||||
"selectedRound": 1,
|
||||
"selectedRoundsByIntent": {
|
||||
"blessing": 19,
|
||||
"complaint": 1,
|
||||
"confirmationDecision": 11,
|
||||
"followUpReminder": 14,
|
||||
"invitation": 1,
|
||||
"question": 1,
|
||||
"replyableMessage": 1,
|
||||
"scheduleNegotiation": 7,
|
||||
"task": 1
|
||||
},
|
||||
"thresholds": {
|
||||
"blessing": {
|
||||
"abstained": true,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": true,
|
||||
"metrics": {
|
||||
"f1": 0.0,
|
||||
"falseNegative": 0,
|
||||
"falsePositive": 0,
|
||||
"precision": 0.0,
|
||||
"predictedPositive": 0,
|
||||
"recall": 0.0,
|
||||
"total": 0,
|
||||
"trueNegative": 0,
|
||||
"truePositive": 0,
|
||||
"wilsonPrecisionLower95": 0
|
||||
},
|
||||
"threshold": 1.0
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": true,
|
||||
"metrics": {
|
||||
"f1": 0.0,
|
||||
"falseNegative": 0,
|
||||
"falsePositive": 0,
|
||||
"precision": 0.0,
|
||||
"predictedPositive": 0,
|
||||
"recall": 0.0,
|
||||
"total": 0,
|
||||
"trueNegative": 0,
|
||||
"truePositive": 0,
|
||||
"wilsonPrecisionLower95": 0
|
||||
},
|
||||
"threshold": 1.0
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.0,
|
||||
"falseNegative": 0,
|
||||
"falsePositive": 0,
|
||||
"precision": 0.0,
|
||||
"predictedPositive": 0,
|
||||
"recall": 0.0,
|
||||
"total": 0,
|
||||
"trueNegative": 0,
|
||||
"truePositive": 0,
|
||||
"wilsonPrecisionLower95": 0
|
||||
},
|
||||
"threshold": 1.0
|
||||
},
|
||||
"complaint": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 1.0,
|
||||
"falseNegative": 0,
|
||||
"falsePositive": 0,
|
||||
"precision": 1.0,
|
||||
"predictedPositive": 180,
|
||||
"recall": 1.0,
|
||||
"total": 1485,
|
||||
"trueNegative": 1305,
|
||||
"truePositive": 180,
|
||||
"wilsonPrecisionLower95": 0.9791
|
||||
},
|
||||
"threshold": 0.5
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.9526,
|
||||
"falseNegative": 9,
|
||||
"falsePositive": 8,
|
||||
"precision": 0.9553,
|
||||
"predictedPositive": 179,
|
||||
"recall": 0.95,
|
||||
"total": 1485,
|
||||
"trueNegative": 1297,
|
||||
"truePositive": 171,
|
||||
"wilsonPrecisionLower95": 0.9143
|
||||
},
|
||||
"threshold": 0.54
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.9764,
|
||||
"falseNegative": 9,
|
||||
"falsePositive": 8,
|
||||
"precision": 0.9777,
|
||||
"predictedPositive": 359,
|
||||
"recall": 0.975,
|
||||
"total": 2970,
|
||||
"trueNegative": 2602,
|
||||
"truePositive": 351,
|
||||
"wilsonPrecisionLower95": 0.9567
|
||||
},
|
||||
"threshold": 0.54
|
||||
},
|
||||
"confirmationDecision": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.931,
|
||||
"falseNegative": 9,
|
||||
"falsePositive": 3,
|
||||
"precision": 0.9643,
|
||||
"predictedPositive": 84,
|
||||
"recall": 0.9,
|
||||
"total": 1485,
|
||||
"trueNegative": 1392,
|
||||
"truePositive": 81,
|
||||
"wilsonPrecisionLower95": 0.9002
|
||||
},
|
||||
"threshold": 0.97
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.7222,
|
||||
"falseNegative": 38,
|
||||
"falsePositive": 2,
|
||||
"precision": 0.963,
|
||||
"predictedPositive": 54,
|
||||
"recall": 0.5778,
|
||||
"total": 1485,
|
||||
"trueNegative": 1393,
|
||||
"truePositive": 52,
|
||||
"wilsonPrecisionLower95": 0.8746
|
||||
},
|
||||
"threshold": 0.86
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.8077,
|
||||
"falseNegative": 54,
|
||||
"falsePositive": 6,
|
||||
"precision": 0.9545,
|
||||
"predictedPositive": 132,
|
||||
"recall": 0.7,
|
||||
"total": 2970,
|
||||
"trueNegative": 2784,
|
||||
"truePositive": 126,
|
||||
"wilsonPrecisionLower95": 0.9044
|
||||
},
|
||||
"threshold": 0.95
|
||||
},
|
||||
"followUpReminder": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.8788,
|
||||
"falseNegative": 35,
|
||||
"falsePositive": 5,
|
||||
"precision": 0.9667,
|
||||
"predictedPositive": 150,
|
||||
"recall": 0.8056,
|
||||
"total": 1485,
|
||||
"trueNegative": 1300,
|
||||
"truePositive": 145,
|
||||
"wilsonPrecisionLower95": 0.9243
|
||||
},
|
||||
"threshold": 0.845
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.6113,
|
||||
"falseNegative": 99,
|
||||
"falsePositive": 4,
|
||||
"precision": 0.9529,
|
||||
"predictedPositive": 85,
|
||||
"recall": 0.45,
|
||||
"total": 1485,
|
||||
"trueNegative": 1301,
|
||||
"truePositive": 81,
|
||||
"wilsonPrecisionLower95": 0.8852
|
||||
},
|
||||
"threshold": 0.97
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.7682,
|
||||
"falseNegative": 128,
|
||||
"falsePositive": 12,
|
||||
"precision": 0.9508,
|
||||
"predictedPositive": 244,
|
||||
"recall": 0.6444,
|
||||
"total": 2970,
|
||||
"trueNegative": 2598,
|
||||
"truePositive": 232,
|
||||
"wilsonPrecisionLower95": 0.916
|
||||
},
|
||||
"threshold": 0.91
|
||||
},
|
||||
"invitation": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.8302,
|
||||
"falseNegative": 24,
|
||||
"falsePositive": 3,
|
||||
"precision": 0.9565,
|
||||
"predictedPositive": 69,
|
||||
"recall": 0.7333,
|
||||
"total": 1485,
|
||||
"trueNegative": 1392,
|
||||
"truePositive": 66,
|
||||
"wilsonPrecisionLower95": 0.8798
|
||||
},
|
||||
"threshold": 0.935
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.6765,
|
||||
"falseNegative": 44,
|
||||
"falsePositive": 0,
|
||||
"precision": 1.0,
|
||||
"predictedPositive": 46,
|
||||
"recall": 0.5111,
|
||||
"total": 1485,
|
||||
"trueNegative": 1395,
|
||||
"truePositive": 46,
|
||||
"wilsonPrecisionLower95": 0.9229
|
||||
},
|
||||
"threshold": 0.505
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.5476,
|
||||
"falseNegative": 111,
|
||||
"falsePositive": 3,
|
||||
"precision": 0.9583,
|
||||
"predictedPositive": 72,
|
||||
"recall": 0.3833,
|
||||
"total": 2970,
|
||||
"trueNegative": 2787,
|
||||
"truePositive": 69,
|
||||
"wilsonPrecisionLower95": 0.8845
|
||||
},
|
||||
"threshold": 0.935
|
||||
},
|
||||
"question": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.8445,
|
||||
"falseNegative": 76,
|
||||
"falsePositive": 12,
|
||||
"precision": 0.9522,
|
||||
"predictedPositive": 251,
|
||||
"recall": 0.7587,
|
||||
"total": 1485,
|
||||
"trueNegative": 1158,
|
||||
"truePositive": 239,
|
||||
"wilsonPrecisionLower95": 0.9183
|
||||
},
|
||||
"threshold": 0.725
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.8536,
|
||||
"falseNegative": 76,
|
||||
"falsePositive": 6,
|
||||
"precision": 0.9755,
|
||||
"predictedPositive": 245,
|
||||
"recall": 0.7587,
|
||||
"total": 1485,
|
||||
"trueNegative": 1164,
|
||||
"truePositive": 239,
|
||||
"wilsonPrecisionLower95": 0.9476
|
||||
},
|
||||
"threshold": 0.5
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.8729,
|
||||
"falseNegative": 122,
|
||||
"falsePositive": 26,
|
||||
"precision": 0.9513,
|
||||
"predictedPositive": 534,
|
||||
"recall": 0.8063,
|
||||
"total": 2970,
|
||||
"trueNegative": 2314,
|
||||
"truePositive": 508,
|
||||
"wilsonPrecisionLower95": 0.9296
|
||||
},
|
||||
"threshold": 0.515
|
||||
},
|
||||
"replyableMessage": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.948,
|
||||
"falseNegative": 52,
|
||||
"falsePositive": 46,
|
||||
"precision": 0.951,
|
||||
"predictedPositive": 939,
|
||||
"recall": 0.945,
|
||||
"total": 1485,
|
||||
"trueNegative": 494,
|
||||
"truePositive": 893,
|
||||
"wilsonPrecisionLower95": 0.9353
|
||||
},
|
||||
"threshold": 0.7
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.9424,
|
||||
"falseNegative": 62,
|
||||
"falsePositive": 46,
|
||||
"precision": 0.9505,
|
||||
"predictedPositive": 929,
|
||||
"recall": 0.9344,
|
||||
"total": 1485,
|
||||
"trueNegative": 494,
|
||||
"truePositive": 883,
|
||||
"wilsonPrecisionLower95": 0.9346
|
||||
},
|
||||
"threshold": 0.695
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.9454,
|
||||
"falseNegative": 115,
|
||||
"falsePositive": 90,
|
||||
"precision": 0.9517,
|
||||
"predictedPositive": 1865,
|
||||
"recall": 0.9392,
|
||||
"total": 2970,
|
||||
"trueNegative": 990,
|
||||
"truePositive": 1775,
|
||||
"wilsonPrecisionLower95": 0.9411
|
||||
},
|
||||
"threshold": 0.7
|
||||
},
|
||||
"scheduleNegotiation": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 1.0,
|
||||
"falseNegative": 0,
|
||||
"falsePositive": 0,
|
||||
"precision": 1.0,
|
||||
"predictedPositive": 45,
|
||||
"recall": 1.0,
|
||||
"total": 1485,
|
||||
"trueNegative": 1440,
|
||||
"truePositive": 45,
|
||||
"wilsonPrecisionLower95": 0.9213
|
||||
},
|
||||
"threshold": 0.66
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 1.0,
|
||||
"falseNegative": 0,
|
||||
"falsePositive": 0,
|
||||
"precision": 1.0,
|
||||
"predictedPositive": 45,
|
||||
"recall": 1.0,
|
||||
"total": 1485,
|
||||
"trueNegative": 1440,
|
||||
"truePositive": 45,
|
||||
"wilsonPrecisionLower95": 0.9213
|
||||
},
|
||||
"threshold": 0.55
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 1.0,
|
||||
"falseNegative": 0,
|
||||
"falsePositive": 0,
|
||||
"precision": 1.0,
|
||||
"predictedPositive": 90,
|
||||
"recall": 1.0,
|
||||
"total": 2970,
|
||||
"trueNegative": 2880,
|
||||
"truePositive": 90,
|
||||
"wilsonPrecisionLower95": 0.9591
|
||||
},
|
||||
"threshold": 0.66
|
||||
},
|
||||
"task": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.8403,
|
||||
"falseNegative": 89,
|
||||
"falsePositive": 14,
|
||||
"precision": 0.9509,
|
||||
"predictedPositive": 285,
|
||||
"recall": 0.7528,
|
||||
"total": 1485,
|
||||
"trueNegative": 1111,
|
||||
"truePositive": 271,
|
||||
"wilsonPrecisionLower95": 0.9192
|
||||
},
|
||||
"threshold": 0.835
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.8278,
|
||||
"falseNegative": 98,
|
||||
"falsePositive": 11,
|
||||
"precision": 0.9597,
|
||||
"predictedPositive": 273,
|
||||
"recall": 0.7278,
|
||||
"total": 1485,
|
||||
"trueNegative": 1114,
|
||||
"truePositive": 262,
|
||||
"wilsonPrecisionLower95": 0.9293
|
||||
},
|
||||
"threshold": 0.745
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.8637,
|
||||
"falseNegative": 153,
|
||||
"falsePositive": 26,
|
||||
"precision": 0.9562,
|
||||
"predictedPositive": 593,
|
||||
"recall": 0.7875,
|
||||
"total": 2970,
|
||||
"trueNegative": 2224,
|
||||
"truePositive": 567,
|
||||
"wilsonPrecisionLower95": 0.9365
|
||||
},
|
||||
"threshold": 0.755
|
||||
}
|
||||
}
|
||||
}
|
||||
+14831
File diff suppressed because it is too large
Load Diff
+8422
File diff suppressed because it is too large
Load Diff
+629
@@ -0,0 +1,629 @@
|
||||
{
|
||||
"automaticPromotionAllowed": false,
|
||||
"baseCorpus": "ModelTraining/ClipboardSemantics/clipboard_semantic_corpus.jsonl",
|
||||
"configuration": {
|
||||
"alpha": 2e-05,
|
||||
"augmentation": "none",
|
||||
"char_max": 5,
|
||||
"char_min": 2,
|
||||
"external_weight": 0.5700000000000001,
|
||||
"hard_example_weight": 1.0,
|
||||
"l1_ratio": 0.0,
|
||||
"max_char_features": 45000,
|
||||
"max_word_features": 22000,
|
||||
"min_df": 2,
|
||||
"pseudo_weight": 0.16499999999999998,
|
||||
"round": 1,
|
||||
"word_max": 2
|
||||
},
|
||||
"configurationsByIntent": {
|
||||
"blessing": {
|
||||
"alpha": 1.2e-06,
|
||||
"augmentation": "numeric",
|
||||
"char_max": 8,
|
||||
"char_min": 3,
|
||||
"external_weight": 0.93,
|
||||
"hard_example_weight": 2.7,
|
||||
"l1_ratio": 0.25,
|
||||
"max_char_features": 95000,
|
||||
"max_word_features": 45000,
|
||||
"min_df": 1,
|
||||
"pseudo_weight": 0.43499999999999994,
|
||||
"round": 19,
|
||||
"word_max": 3
|
||||
},
|
||||
"complaint": {
|
||||
"alpha": 2e-05,
|
||||
"augmentation": "none",
|
||||
"char_max": 5,
|
||||
"char_min": 2,
|
||||
"external_weight": 0.5700000000000001,
|
||||
"hard_example_weight": 1.0,
|
||||
"l1_ratio": 0.0,
|
||||
"max_char_features": 45000,
|
||||
"max_word_features": 22000,
|
||||
"min_df": 2,
|
||||
"pseudo_weight": 0.16499999999999998,
|
||||
"round": 1,
|
||||
"word_max": 2
|
||||
},
|
||||
"confirmationDecision": {
|
||||
"alpha": 1e-05,
|
||||
"augmentation": "none",
|
||||
"char_max": 6,
|
||||
"char_min": 2,
|
||||
"external_weight": 0.6100000000000001,
|
||||
"hard_example_weight": 1.2,
|
||||
"l1_ratio": 0.05,
|
||||
"max_char_features": 55000,
|
||||
"max_word_features": 25000,
|
||||
"min_df": 2,
|
||||
"pseudo_weight": 0.195,
|
||||
"round": 3,
|
||||
"word_max": 2
|
||||
},
|
||||
"followUpReminder": {
|
||||
"alpha": 2e-05,
|
||||
"augmentation": "none",
|
||||
"char_max": 5,
|
||||
"char_min": 2,
|
||||
"external_weight": 0.5700000000000001,
|
||||
"hard_example_weight": 1.0,
|
||||
"l1_ratio": 0.0,
|
||||
"max_char_features": 45000,
|
||||
"max_word_features": 22000,
|
||||
"min_df": 2,
|
||||
"pseudo_weight": 0.16499999999999998,
|
||||
"round": 1,
|
||||
"word_max": 2
|
||||
},
|
||||
"invitation": {
|
||||
"alpha": 2e-05,
|
||||
"augmentation": "none",
|
||||
"char_max": 5,
|
||||
"char_min": 2,
|
||||
"external_weight": 0.5700000000000001,
|
||||
"hard_example_weight": 1.0,
|
||||
"l1_ratio": 0.0,
|
||||
"max_char_features": 45000,
|
||||
"max_word_features": 22000,
|
||||
"min_df": 2,
|
||||
"pseudo_weight": 0.16499999999999998,
|
||||
"round": 1,
|
||||
"word_max": 2
|
||||
},
|
||||
"question": {
|
||||
"alpha": 2e-05,
|
||||
"augmentation": "none",
|
||||
"char_max": 5,
|
||||
"char_min": 2,
|
||||
"external_weight": 0.5700000000000001,
|
||||
"hard_example_weight": 1.0,
|
||||
"l1_ratio": 0.0,
|
||||
"max_char_features": 45000,
|
||||
"max_word_features": 22000,
|
||||
"min_df": 2,
|
||||
"pseudo_weight": 0.16499999999999998,
|
||||
"round": 1,
|
||||
"word_max": 2
|
||||
},
|
||||
"replyableMessage": {
|
||||
"alpha": 2e-05,
|
||||
"augmentation": "none",
|
||||
"char_max": 5,
|
||||
"char_min": 2,
|
||||
"external_weight": 0.5700000000000001,
|
||||
"hard_example_weight": 1.0,
|
||||
"l1_ratio": 0.0,
|
||||
"max_char_features": 45000,
|
||||
"max_word_features": 22000,
|
||||
"min_df": 2,
|
||||
"pseudo_weight": 0.16499999999999998,
|
||||
"round": 1,
|
||||
"word_max": 2
|
||||
},
|
||||
"scheduleNegotiation": {
|
||||
"alpha": 6e-06,
|
||||
"augmentation": "punctuation",
|
||||
"char_max": 6,
|
||||
"char_min": 3,
|
||||
"external_weight": 0.6900000000000001,
|
||||
"hard_example_weight": 1.5,
|
||||
"l1_ratio": 0.1,
|
||||
"max_char_features": 65000,
|
||||
"max_word_features": 32000,
|
||||
"min_df": 2,
|
||||
"pseudo_weight": 0.255,
|
||||
"round": 7,
|
||||
"word_max": 3
|
||||
},
|
||||
"task": {
|
||||
"alpha": 3.5e-06,
|
||||
"augmentation": "prefix",
|
||||
"char_max": 6,
|
||||
"char_min": 2,
|
||||
"external_weight": 0.8500000000000001,
|
||||
"hard_example_weight": 2.2,
|
||||
"l1_ratio": 0.2,
|
||||
"max_char_features": 85000,
|
||||
"max_word_features": 42000,
|
||||
"min_df": 1,
|
||||
"pseudo_weight": 0.375,
|
||||
"round": 15,
|
||||
"word_max": 3
|
||||
}
|
||||
},
|
||||
"openCorpus": "ModelTraining/ClipboardSemantics/open-training-corpus.jsonl",
|
||||
"reason": "Surrogate feature weights are not deployable NLModel assets and cannot bypass the existing macOS acceptance policy.",
|
||||
"requiredCommands": [
|
||||
"python3 Scripts/clipboard_semantics/generate_open_training_corpus.py",
|
||||
"xcrun swift Scripts/clipboard_semantics/train_models.swift --algorithms maxEnt --corpus ModelTraining/ClipboardSemantics/combined-training-corpus.jsonl",
|
||||
"xcrun swift Scripts/clipboard_semantics/evaluate_random_holdout.swift --corpus ModelTraining/ClipboardSemantics/random-holdout-corpus.jsonl"
|
||||
],
|
||||
"schemaVersion": 1,
|
||||
"seed": 20260827,
|
||||
"selectedRound": 1,
|
||||
"selectedRoundsByIntent": {
|
||||
"blessing": 19,
|
||||
"complaint": 1,
|
||||
"confirmationDecision": 3,
|
||||
"followUpReminder": 1,
|
||||
"invitation": 1,
|
||||
"question": 1,
|
||||
"replyableMessage": 1,
|
||||
"scheduleNegotiation": 7,
|
||||
"task": 15
|
||||
},
|
||||
"thresholds": {
|
||||
"blessing": {
|
||||
"abstained": true,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": true,
|
||||
"metrics": {
|
||||
"f1": 0.0,
|
||||
"falseNegative": 0,
|
||||
"falsePositive": 0,
|
||||
"precision": 0.0,
|
||||
"predictedPositive": 0,
|
||||
"recall": 0.0,
|
||||
"total": 0,
|
||||
"trueNegative": 0,
|
||||
"truePositive": 0,
|
||||
"wilsonPrecisionLower95": 0
|
||||
},
|
||||
"threshold": 1.0
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": true,
|
||||
"metrics": {
|
||||
"f1": 0.0,
|
||||
"falseNegative": 0,
|
||||
"falsePositive": 0,
|
||||
"precision": 0.0,
|
||||
"predictedPositive": 0,
|
||||
"recall": 0.0,
|
||||
"total": 0,
|
||||
"trueNegative": 0,
|
||||
"truePositive": 0,
|
||||
"wilsonPrecisionLower95": 0
|
||||
},
|
||||
"threshold": 1.0
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.0,
|
||||
"falseNegative": 0,
|
||||
"falsePositive": 0,
|
||||
"precision": 0.0,
|
||||
"predictedPositive": 0,
|
||||
"recall": 0.0,
|
||||
"total": 0,
|
||||
"trueNegative": 0,
|
||||
"truePositive": 0,
|
||||
"wilsonPrecisionLower95": 0
|
||||
},
|
||||
"threshold": 1.0
|
||||
},
|
||||
"complaint": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 1.0,
|
||||
"falseNegative": 0,
|
||||
"falsePositive": 0,
|
||||
"precision": 1.0,
|
||||
"predictedPositive": 180,
|
||||
"recall": 1.0,
|
||||
"total": 1485,
|
||||
"trueNegative": 1305,
|
||||
"truePositive": 180,
|
||||
"wilsonPrecisionLower95": 0.9791
|
||||
},
|
||||
"threshold": 0.5
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.9526,
|
||||
"falseNegative": 9,
|
||||
"falsePositive": 8,
|
||||
"precision": 0.9553,
|
||||
"predictedPositive": 179,
|
||||
"recall": 0.95,
|
||||
"total": 1485,
|
||||
"trueNegative": 1297,
|
||||
"truePositive": 171,
|
||||
"wilsonPrecisionLower95": 0.9143
|
||||
},
|
||||
"threshold": 0.54
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.9764,
|
||||
"falseNegative": 9,
|
||||
"falsePositive": 8,
|
||||
"precision": 0.9777,
|
||||
"predictedPositive": 359,
|
||||
"recall": 0.975,
|
||||
"total": 2970,
|
||||
"trueNegative": 2602,
|
||||
"truePositive": 351,
|
||||
"wilsonPrecisionLower95": 0.9567
|
||||
},
|
||||
"threshold": 0.54
|
||||
},
|
||||
"confirmationDecision": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.9727,
|
||||
"falseNegative": 1,
|
||||
"falsePositive": 4,
|
||||
"precision": 0.957,
|
||||
"predictedPositive": 93,
|
||||
"recall": 0.9889,
|
||||
"total": 1485,
|
||||
"trueNegative": 1391,
|
||||
"truePositive": 89,
|
||||
"wilsonPrecisionLower95": 0.8946
|
||||
},
|
||||
"threshold": 0.585
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.7815,
|
||||
"falseNegative": 31,
|
||||
"falsePositive": 2,
|
||||
"precision": 0.9672,
|
||||
"predictedPositive": 61,
|
||||
"recall": 0.6556,
|
||||
"total": 1485,
|
||||
"trueNegative": 1393,
|
||||
"truePositive": 59,
|
||||
"wilsonPrecisionLower95": 0.8881
|
||||
},
|
||||
"threshold": 0.5
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.8701,
|
||||
"falseNegative": 36,
|
||||
"falsePositive": 7,
|
||||
"precision": 0.9536,
|
||||
"predictedPositive": 151,
|
||||
"recall": 0.8,
|
||||
"total": 2970,
|
||||
"trueNegative": 2783,
|
||||
"truePositive": 144,
|
||||
"wilsonPrecisionLower95": 0.9074
|
||||
},
|
||||
"threshold": 0.555
|
||||
},
|
||||
"followUpReminder": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.8554,
|
||||
"falseNegative": 41,
|
||||
"falsePositive": 6,
|
||||
"precision": 0.9586,
|
||||
"predictedPositive": 145,
|
||||
"recall": 0.7722,
|
||||
"total": 1485,
|
||||
"trueNegative": 1299,
|
||||
"truePositive": 139,
|
||||
"wilsonPrecisionLower95": 0.9127
|
||||
},
|
||||
"threshold": 0.83
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.8354,
|
||||
"falseNegative": 48,
|
||||
"falsePositive": 4,
|
||||
"precision": 0.9706,
|
||||
"predictedPositive": 136,
|
||||
"recall": 0.7333,
|
||||
"total": 1485,
|
||||
"trueNegative": 1301,
|
||||
"truePositive": 132,
|
||||
"wilsonPrecisionLower95": 0.9268
|
||||
},
|
||||
"threshold": 0.505
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.7994,
|
||||
"falseNegative": 113,
|
||||
"falsePositive": 11,
|
||||
"precision": 0.9574,
|
||||
"predictedPositive": 258,
|
||||
"recall": 0.6861,
|
||||
"total": 2970,
|
||||
"trueNegative": 2599,
|
||||
"truePositive": 247,
|
||||
"wilsonPrecisionLower95": 0.9253
|
||||
},
|
||||
"threshold": 0.815
|
||||
},
|
||||
"invitation": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.8302,
|
||||
"falseNegative": 24,
|
||||
"falsePositive": 3,
|
||||
"precision": 0.9565,
|
||||
"predictedPositive": 69,
|
||||
"recall": 0.7333,
|
||||
"total": 1485,
|
||||
"trueNegative": 1392,
|
||||
"truePositive": 66,
|
||||
"wilsonPrecisionLower95": 0.8798
|
||||
},
|
||||
"threshold": 0.935
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.6765,
|
||||
"falseNegative": 44,
|
||||
"falsePositive": 0,
|
||||
"precision": 1.0,
|
||||
"predictedPositive": 46,
|
||||
"recall": 0.5111,
|
||||
"total": 1485,
|
||||
"trueNegative": 1395,
|
||||
"truePositive": 46,
|
||||
"wilsonPrecisionLower95": 0.9229
|
||||
},
|
||||
"threshold": 0.505
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.5476,
|
||||
"falseNegative": 111,
|
||||
"falsePositive": 3,
|
||||
"precision": 0.9583,
|
||||
"predictedPositive": 72,
|
||||
"recall": 0.3833,
|
||||
"total": 2970,
|
||||
"trueNegative": 2787,
|
||||
"truePositive": 69,
|
||||
"wilsonPrecisionLower95": 0.8845
|
||||
},
|
||||
"threshold": 0.935
|
||||
},
|
||||
"question": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.8445,
|
||||
"falseNegative": 76,
|
||||
"falsePositive": 12,
|
||||
"precision": 0.9522,
|
||||
"predictedPositive": 251,
|
||||
"recall": 0.7587,
|
||||
"total": 1485,
|
||||
"trueNegative": 1158,
|
||||
"truePositive": 239,
|
||||
"wilsonPrecisionLower95": 0.9183
|
||||
},
|
||||
"threshold": 0.725
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.8536,
|
||||
"falseNegative": 76,
|
||||
"falsePositive": 6,
|
||||
"precision": 0.9755,
|
||||
"predictedPositive": 245,
|
||||
"recall": 0.7587,
|
||||
"total": 1485,
|
||||
"trueNegative": 1164,
|
||||
"truePositive": 239,
|
||||
"wilsonPrecisionLower95": 0.9476
|
||||
},
|
||||
"threshold": 0.5
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.8729,
|
||||
"falseNegative": 122,
|
||||
"falsePositive": 26,
|
||||
"precision": 0.9513,
|
||||
"predictedPositive": 534,
|
||||
"recall": 0.8063,
|
||||
"total": 2970,
|
||||
"trueNegative": 2314,
|
||||
"truePositive": 508,
|
||||
"wilsonPrecisionLower95": 0.9296
|
||||
},
|
||||
"threshold": 0.515
|
||||
},
|
||||
"replyableMessage": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.948,
|
||||
"falseNegative": 52,
|
||||
"falsePositive": 46,
|
||||
"precision": 0.951,
|
||||
"predictedPositive": 939,
|
||||
"recall": 0.945,
|
||||
"total": 1485,
|
||||
"trueNegative": 494,
|
||||
"truePositive": 893,
|
||||
"wilsonPrecisionLower95": 0.9353
|
||||
},
|
||||
"threshold": 0.7
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.9424,
|
||||
"falseNegative": 62,
|
||||
"falsePositive": 46,
|
||||
"precision": 0.9505,
|
||||
"predictedPositive": 929,
|
||||
"recall": 0.9344,
|
||||
"total": 1485,
|
||||
"trueNegative": 494,
|
||||
"truePositive": 883,
|
||||
"wilsonPrecisionLower95": 0.9346
|
||||
},
|
||||
"threshold": 0.695
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.9454,
|
||||
"falseNegative": 115,
|
||||
"falsePositive": 90,
|
||||
"precision": 0.9517,
|
||||
"predictedPositive": 1865,
|
||||
"recall": 0.9392,
|
||||
"total": 2970,
|
||||
"trueNegative": 990,
|
||||
"truePositive": 1775,
|
||||
"wilsonPrecisionLower95": 0.9411
|
||||
},
|
||||
"threshold": 0.7
|
||||
},
|
||||
"scheduleNegotiation": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 1.0,
|
||||
"falseNegative": 0,
|
||||
"falsePositive": 0,
|
||||
"precision": 1.0,
|
||||
"predictedPositive": 45,
|
||||
"recall": 1.0,
|
||||
"total": 1485,
|
||||
"trueNegative": 1440,
|
||||
"truePositive": 45,
|
||||
"wilsonPrecisionLower95": 0.9213
|
||||
},
|
||||
"threshold": 0.66
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 1.0,
|
||||
"falseNegative": 0,
|
||||
"falsePositive": 0,
|
||||
"precision": 1.0,
|
||||
"predictedPositive": 45,
|
||||
"recall": 1.0,
|
||||
"total": 1485,
|
||||
"trueNegative": 1440,
|
||||
"truePositive": 45,
|
||||
"wilsonPrecisionLower95": 0.9213
|
||||
},
|
||||
"threshold": 0.55
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 1.0,
|
||||
"falseNegative": 0,
|
||||
"falsePositive": 0,
|
||||
"precision": 1.0,
|
||||
"predictedPositive": 90,
|
||||
"recall": 1.0,
|
||||
"total": 2970,
|
||||
"trueNegative": 2880,
|
||||
"truePositive": 90,
|
||||
"wilsonPrecisionLower95": 0.9591
|
||||
},
|
||||
"threshold": 0.66
|
||||
},
|
||||
"task": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.8593,
|
||||
"falseNegative": 79,
|
||||
"falsePositive": 13,
|
||||
"precision": 0.9558,
|
||||
"predictedPositive": 294,
|
||||
"recall": 0.7806,
|
||||
"total": 1485,
|
||||
"trueNegative": 1112,
|
||||
"truePositive": 281,
|
||||
"wilsonPrecisionLower95": 0.9258
|
||||
},
|
||||
"threshold": 0.98
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.8166,
|
||||
"falseNegative": 104,
|
||||
"falsePositive": 11,
|
||||
"precision": 0.9588,
|
||||
"predictedPositive": 267,
|
||||
"recall": 0.7111,
|
||||
"total": 1485,
|
||||
"trueNegative": 1114,
|
||||
"truePositive": 256,
|
||||
"wilsonPrecisionLower95": 0.9277
|
||||
},
|
||||
"threshold": 0.96
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.8434,
|
||||
"falseNegative": 176,
|
||||
"falsePositive": 26,
|
||||
"precision": 0.9544,
|
||||
"predictedPositive": 570,
|
||||
"recall": 0.7556,
|
||||
"total": 2970,
|
||||
"trueNegative": 2224,
|
||||
"truePositive": 544,
|
||||
"wilsonPrecisionLower95": 0.934
|
||||
},
|
||||
"threshold": 0.965
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,481 @@
|
||||
{
|
||||
"automaticPromotionAllowed": false,
|
||||
"baseCorpus": "ModelTraining/ClipboardSemantics/clipboard_semantic_corpus.jsonl",
|
||||
"configuration": {
|
||||
"alpha": 2e-05,
|
||||
"augmentation": "none",
|
||||
"char_max": 5,
|
||||
"char_min": 2,
|
||||
"external_weight": 0.5700000000000001,
|
||||
"hard_example_weight": 1.0,
|
||||
"l1_ratio": 0.0,
|
||||
"max_char_features": 45000,
|
||||
"max_word_features": 22000,
|
||||
"min_df": 2,
|
||||
"pseudo_weight": 0.16499999999999998,
|
||||
"round": 1,
|
||||
"word_max": 2
|
||||
},
|
||||
"openCorpus": "ModelTraining/ClipboardSemantics/open-training-corpus.jsonl",
|
||||
"reason": "Surrogate feature weights are not deployable NLModel assets and cannot bypass the existing macOS acceptance policy.",
|
||||
"requiredCommands": [
|
||||
"python3 Scripts/clipboard_semantics/generate_open_training_corpus.py",
|
||||
"xcrun swift Scripts/clipboard_semantics/train_models.swift --algorithms maxEnt --corpus ModelTraining/ClipboardSemantics/combined-training-corpus.jsonl",
|
||||
"xcrun swift Scripts/clipboard_semantics/evaluate_random_holdout.swift --corpus ModelTraining/ClipboardSemantics/random-holdout-corpus.jsonl"
|
||||
],
|
||||
"schemaVersion": 1,
|
||||
"seed": 20260827,
|
||||
"selectedRound": 1,
|
||||
"thresholds": {
|
||||
"blessing": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.9318,
|
||||
"falseNegative": 4,
|
||||
"falsePositive": 2,
|
||||
"precision": 0.9535,
|
||||
"predictedPositive": 43,
|
||||
"recall": 0.9111,
|
||||
"total": 1485,
|
||||
"trueNegative": 1438,
|
||||
"truePositive": 41,
|
||||
"wilsonPrecisionLower95": 0.8454
|
||||
},
|
||||
"threshold": 0.915
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.7568,
|
||||
"falseNegative": 17,
|
||||
"falsePositive": 1,
|
||||
"precision": 0.9655,
|
||||
"predictedPositive": 29,
|
||||
"recall": 0.6222,
|
||||
"total": 1485,
|
||||
"trueNegative": 1439,
|
||||
"truePositive": 28,
|
||||
"wilsonPrecisionLower95": 0.8282
|
||||
},
|
||||
"threshold": 0.9
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.8375,
|
||||
"falseNegative": 23,
|
||||
"falsePositive": 3,
|
||||
"precision": 0.9571,
|
||||
"predictedPositive": 70,
|
||||
"recall": 0.7444,
|
||||
"total": 2970,
|
||||
"trueNegative": 2877,
|
||||
"truePositive": 67,
|
||||
"wilsonPrecisionLower95": 0.8814
|
||||
},
|
||||
"threshold": 0.915
|
||||
},
|
||||
"complaint": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 1.0,
|
||||
"falseNegative": 0,
|
||||
"falsePositive": 0,
|
||||
"precision": 1.0,
|
||||
"predictedPositive": 180,
|
||||
"recall": 1.0,
|
||||
"total": 1485,
|
||||
"trueNegative": 1305,
|
||||
"truePositive": 180,
|
||||
"wilsonPrecisionLower95": 0.9791
|
||||
},
|
||||
"threshold": 0.5
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.9526,
|
||||
"falseNegative": 9,
|
||||
"falsePositive": 8,
|
||||
"precision": 0.9553,
|
||||
"predictedPositive": 179,
|
||||
"recall": 0.95,
|
||||
"total": 1485,
|
||||
"trueNegative": 1297,
|
||||
"truePositive": 171,
|
||||
"wilsonPrecisionLower95": 0.9143
|
||||
},
|
||||
"threshold": 0.54
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.9764,
|
||||
"falseNegative": 9,
|
||||
"falsePositive": 8,
|
||||
"precision": 0.9777,
|
||||
"predictedPositive": 359,
|
||||
"recall": 0.975,
|
||||
"total": 2970,
|
||||
"trueNegative": 2602,
|
||||
"truePositive": 351,
|
||||
"wilsonPrecisionLower95": 0.9567
|
||||
},
|
||||
"threshold": 0.54
|
||||
},
|
||||
"confirmationDecision": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.8848,
|
||||
"falseNegative": 17,
|
||||
"falsePositive": 2,
|
||||
"precision": 0.9733,
|
||||
"predictedPositive": 75,
|
||||
"recall": 0.8111,
|
||||
"total": 1485,
|
||||
"trueNegative": 1393,
|
||||
"truePositive": 73,
|
||||
"wilsonPrecisionLower95": 0.9079
|
||||
},
|
||||
"threshold": 0.5
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.7947,
|
||||
"falseNegative": 30,
|
||||
"falsePositive": 1,
|
||||
"precision": 0.9836,
|
||||
"predictedPositive": 61,
|
||||
"recall": 0.6667,
|
||||
"total": 1485,
|
||||
"trueNegative": 1394,
|
||||
"truePositive": 60,
|
||||
"wilsonPrecisionLower95": 0.9128
|
||||
},
|
||||
"threshold": 0.5
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.8418,
|
||||
"falseNegative": 47,
|
||||
"falsePositive": 3,
|
||||
"precision": 0.9779,
|
||||
"predictedPositive": 136,
|
||||
"recall": 0.7389,
|
||||
"total": 2970,
|
||||
"trueNegative": 2787,
|
||||
"truePositive": 133,
|
||||
"wilsonPrecisionLower95": 0.9372
|
||||
},
|
||||
"threshold": 0.5
|
||||
},
|
||||
"followUpReminder": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.8554,
|
||||
"falseNegative": 41,
|
||||
"falsePositive": 6,
|
||||
"precision": 0.9586,
|
||||
"predictedPositive": 145,
|
||||
"recall": 0.7722,
|
||||
"total": 1485,
|
||||
"trueNegative": 1299,
|
||||
"truePositive": 139,
|
||||
"wilsonPrecisionLower95": 0.9127
|
||||
},
|
||||
"threshold": 0.83
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.8354,
|
||||
"falseNegative": 48,
|
||||
"falsePositive": 4,
|
||||
"precision": 0.9706,
|
||||
"predictedPositive": 136,
|
||||
"recall": 0.7333,
|
||||
"total": 1485,
|
||||
"trueNegative": 1301,
|
||||
"truePositive": 132,
|
||||
"wilsonPrecisionLower95": 0.9268
|
||||
},
|
||||
"threshold": 0.505
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.7994,
|
||||
"falseNegative": 113,
|
||||
"falsePositive": 11,
|
||||
"precision": 0.9574,
|
||||
"predictedPositive": 258,
|
||||
"recall": 0.6861,
|
||||
"total": 2970,
|
||||
"trueNegative": 2599,
|
||||
"truePositive": 247,
|
||||
"wilsonPrecisionLower95": 0.9253
|
||||
},
|
||||
"threshold": 0.815
|
||||
},
|
||||
"invitation": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.8302,
|
||||
"falseNegative": 24,
|
||||
"falsePositive": 3,
|
||||
"precision": 0.9565,
|
||||
"predictedPositive": 69,
|
||||
"recall": 0.7333,
|
||||
"total": 1485,
|
||||
"trueNegative": 1392,
|
||||
"truePositive": 66,
|
||||
"wilsonPrecisionLower95": 0.8798
|
||||
},
|
||||
"threshold": 0.935
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.6765,
|
||||
"falseNegative": 44,
|
||||
"falsePositive": 0,
|
||||
"precision": 1.0,
|
||||
"predictedPositive": 46,
|
||||
"recall": 0.5111,
|
||||
"total": 1485,
|
||||
"trueNegative": 1395,
|
||||
"truePositive": 46,
|
||||
"wilsonPrecisionLower95": 0.9229
|
||||
},
|
||||
"threshold": 0.505
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.5476,
|
||||
"falseNegative": 111,
|
||||
"falsePositive": 3,
|
||||
"precision": 0.9583,
|
||||
"predictedPositive": 72,
|
||||
"recall": 0.3833,
|
||||
"total": 2970,
|
||||
"trueNegative": 2787,
|
||||
"truePositive": 69,
|
||||
"wilsonPrecisionLower95": 0.8845
|
||||
},
|
||||
"threshold": 0.935
|
||||
},
|
||||
"question": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.8445,
|
||||
"falseNegative": 76,
|
||||
"falsePositive": 12,
|
||||
"precision": 0.9522,
|
||||
"predictedPositive": 251,
|
||||
"recall": 0.7587,
|
||||
"total": 1485,
|
||||
"trueNegative": 1158,
|
||||
"truePositive": 239,
|
||||
"wilsonPrecisionLower95": 0.9183
|
||||
},
|
||||
"threshold": 0.725
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.8536,
|
||||
"falseNegative": 76,
|
||||
"falsePositive": 6,
|
||||
"precision": 0.9755,
|
||||
"predictedPositive": 245,
|
||||
"recall": 0.7587,
|
||||
"total": 1485,
|
||||
"trueNegative": 1164,
|
||||
"truePositive": 239,
|
||||
"wilsonPrecisionLower95": 0.9476
|
||||
},
|
||||
"threshold": 0.5
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.8729,
|
||||
"falseNegative": 122,
|
||||
"falsePositive": 26,
|
||||
"precision": 0.9513,
|
||||
"predictedPositive": 534,
|
||||
"recall": 0.8063,
|
||||
"total": 2970,
|
||||
"trueNegative": 2314,
|
||||
"truePositive": 508,
|
||||
"wilsonPrecisionLower95": 0.9296
|
||||
},
|
||||
"threshold": 0.515
|
||||
},
|
||||
"replyableMessage": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.948,
|
||||
"falseNegative": 52,
|
||||
"falsePositive": 46,
|
||||
"precision": 0.951,
|
||||
"predictedPositive": 939,
|
||||
"recall": 0.945,
|
||||
"total": 1485,
|
||||
"trueNegative": 494,
|
||||
"truePositive": 893,
|
||||
"wilsonPrecisionLower95": 0.9353
|
||||
},
|
||||
"threshold": 0.7
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.9424,
|
||||
"falseNegative": 62,
|
||||
"falsePositive": 46,
|
||||
"precision": 0.9505,
|
||||
"predictedPositive": 929,
|
||||
"recall": 0.9344,
|
||||
"total": 1485,
|
||||
"trueNegative": 494,
|
||||
"truePositive": 883,
|
||||
"wilsonPrecisionLower95": 0.9346
|
||||
},
|
||||
"threshold": 0.695
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.9454,
|
||||
"falseNegative": 115,
|
||||
"falsePositive": 90,
|
||||
"precision": 0.9517,
|
||||
"predictedPositive": 1865,
|
||||
"recall": 0.9392,
|
||||
"total": 2970,
|
||||
"trueNegative": 990,
|
||||
"truePositive": 1775,
|
||||
"wilsonPrecisionLower95": 0.9411
|
||||
},
|
||||
"threshold": 0.7
|
||||
},
|
||||
"scheduleNegotiation": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 1.0,
|
||||
"falseNegative": 0,
|
||||
"falsePositive": 0,
|
||||
"precision": 1.0,
|
||||
"predictedPositive": 45,
|
||||
"recall": 1.0,
|
||||
"total": 1485,
|
||||
"trueNegative": 1440,
|
||||
"truePositive": 45,
|
||||
"wilsonPrecisionLower95": 0.9213
|
||||
},
|
||||
"threshold": 0.5
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.967,
|
||||
"falseNegative": 1,
|
||||
"falsePositive": 2,
|
||||
"precision": 0.9565,
|
||||
"predictedPositive": 46,
|
||||
"recall": 0.9778,
|
||||
"total": 1485,
|
||||
"trueNegative": 1438,
|
||||
"truePositive": 44,
|
||||
"wilsonPrecisionLower95": 0.8547
|
||||
},
|
||||
"threshold": 0.93
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.978,
|
||||
"falseNegative": 1,
|
||||
"falsePositive": 3,
|
||||
"precision": 0.9674,
|
||||
"predictedPositive": 92,
|
||||
"recall": 0.9889,
|
||||
"total": 2970,
|
||||
"trueNegative": 2877,
|
||||
"truePositive": 89,
|
||||
"wilsonPrecisionLower95": 0.9085
|
||||
},
|
||||
"threshold": 0.91
|
||||
},
|
||||
"task": {
|
||||
"abstained": false,
|
||||
"byLanguage": {
|
||||
"en": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.8403,
|
||||
"falseNegative": 89,
|
||||
"falsePositive": 14,
|
||||
"precision": 0.9509,
|
||||
"predictedPositive": 285,
|
||||
"recall": 0.7528,
|
||||
"total": 1485,
|
||||
"trueNegative": 1111,
|
||||
"truePositive": 271,
|
||||
"wilsonPrecisionLower95": 0.9192
|
||||
},
|
||||
"threshold": 0.835
|
||||
},
|
||||
"zh-Hans": {
|
||||
"abstained": false,
|
||||
"metrics": {
|
||||
"f1": 0.8278,
|
||||
"falseNegative": 98,
|
||||
"falsePositive": 11,
|
||||
"precision": 0.9597,
|
||||
"predictedPositive": 273,
|
||||
"recall": 0.7278,
|
||||
"total": 1485,
|
||||
"trueNegative": 1114,
|
||||
"truePositive": 262,
|
||||
"wilsonPrecisionLower95": 0.9293
|
||||
},
|
||||
"threshold": 0.745
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"f1": 0.8637,
|
||||
"falseNegative": 153,
|
||||
"falsePositive": 26,
|
||||
"precision": 0.9562,
|
||||
"predictedPositive": 593,
|
||||
"recall": 0.7875,
|
||||
"total": 2970,
|
||||
"trueNegative": 2224,
|
||||
"truePositive": 567,
|
||||
"wilsonPrecisionLower95": 0.9365
|
||||
},
|
||||
"threshold": 0.755
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"completedResearchRounds": 60,
|
||||
"requestedCoreRounds": 20,
|
||||
"seed": 20260827,
|
||||
"data": {
|
||||
"baseTrainingRecords": 11500,
|
||||
"reachableLicensedOpenRecords": 3845,
|
||||
"holdoutOverlapCounts": {
|
||||
"comprehensiveOnline": 0,
|
||||
"random": 0,
|
||||
"targetedRelease": 0
|
||||
},
|
||||
"unavailableSources": [
|
||||
"MASSIVE",
|
||||
"CLINC150",
|
||||
"CFPB"
|
||||
]
|
||||
},
|
||||
"phases": [
|
||||
{
|
||||
"name": "shared-configuration-search",
|
||||
"rounds": 20,
|
||||
"selectedRound": 1,
|
||||
"golden": {
|
||||
"macroPrecision": 0.9366,
|
||||
"macroRecall": 0.6801,
|
||||
"macroF1": 0.7711
|
||||
},
|
||||
"randomHoldout": {
|
||||
"macroPrecision": 0.8569,
|
||||
"macroRecall": 0.4603,
|
||||
"macroF1": 0.5709
|
||||
},
|
||||
"researchOnlyComprehensive": {
|
||||
"macroPrecision": 0.4398,
|
||||
"macroRecall": 0.2161,
|
||||
"macroF1": 0.2383
|
||||
},
|
||||
"targetedRelease": {
|
||||
"macroPrecision": 0.5489,
|
||||
"macroRecall": 0.3139,
|
||||
"macroF1": 0.3899
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "per-intent-fine-tuning",
|
||||
"rounds": 20,
|
||||
"selectedRoundsByIntent": {
|
||||
"blessing": 19,
|
||||
"complaint": 1,
|
||||
"confirmationDecision": 3,
|
||||
"followUpReminder": 1,
|
||||
"invitation": 1,
|
||||
"question": 1,
|
||||
"replyableMessage": 1,
|
||||
"scheduleNegotiation": 7,
|
||||
"task": 15
|
||||
},
|
||||
"golden": {
|
||||
"macroPrecision": 0.9377,
|
||||
"macroRecall": 0.6819,
|
||||
"macroF1": 0.7666
|
||||
},
|
||||
"randomHoldout": {
|
||||
"macroPrecision": 0.8701,
|
||||
"macroRecall": 0.4705,
|
||||
"macroF1": 0.5770
|
||||
},
|
||||
"researchOnlyComprehensive": {
|
||||
"macroPrecision": 0.4292,
|
||||
"macroRecall": 0.2250,
|
||||
"macroF1": 0.2645
|
||||
},
|
||||
"targetedRelease": {
|
||||
"macroPrecision": 0.5467,
|
||||
"macroRecall": 0.3375,
|
||||
"macroF1": 0.4095
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "explicit-evidence-fine-tuning",
|
||||
"rounds": 20,
|
||||
"selectedRound": 1,
|
||||
"golden": {
|
||||
"macroPrecision": 0.9877,
|
||||
"macroRecall": 0.4354,
|
||||
"macroF1": 0.5476
|
||||
},
|
||||
"randomHoldout": {
|
||||
"macroPrecision": 0.7731,
|
||||
"macroRecall": 0.3029,
|
||||
"macroF1": 0.3905
|
||||
},
|
||||
"researchOnlyComprehensive": {
|
||||
"macroPrecision": 0.6097,
|
||||
"macroRecall": 0.1647,
|
||||
"macroF1": 0.2325
|
||||
},
|
||||
"targetedRelease": {
|
||||
"macroPrecision": 0.5489,
|
||||
"macroRecall": 0.2537,
|
||||
"macroF1": 0.3209
|
||||
}
|
||||
}
|
||||
],
|
||||
"currentDeployedReference": {
|
||||
"randomHoldout": {
|
||||
"macroPrecision": 0.9832,
|
||||
"macroRecall": 0.6557,
|
||||
"macroF1": 0.7669
|
||||
},
|
||||
"researchOnlyComprehensive": {
|
||||
"macroPrecision": 0.2948,
|
||||
"macroRecall": 0.3320,
|
||||
"macroF1": 0.2783
|
||||
}
|
||||
},
|
||||
"findings": [
|
||||
"The first low-complexity round generalized better than later self-training rounds.",
|
||||
"Three-round high-confidence pseudo-labeling amplified source and template bias instead of improving frozen regressions.",
|
||||
"Per-intent configuration selection improved the research-only comprehensive F1 from 0.2383 to 0.2645 but remained below the deployed reference F1 of 0.2783.",
|
||||
"Explicit evidence gates raised golden precision to 0.9877 and research-only precision to 0.6097, but recall collapsed and the release gate still failed.",
|
||||
"No surrogate candidate is eligible to replace or augment the deployed Apple models."
|
||||
],
|
||||
"releaseGatePassed": false,
|
||||
"deploymentDecision": "retain-current-deployed-models",
|
||||
"platformLimitation": "Linux cannot train, compile, run, or benchmark the deployable Apple Create ML models. These rounds are surrogate research and produce a macOS replay specification, not app-ready model assets."
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user