rocky/OSGKeyboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

docs(semantics): record iterative retraining study

Browse Source 
Co-authored-by: Rocky <hkgood@users.noreply.github.com>
This commit is contained in:
Cursor Agent
2026-08-27 10:23:44 +00:00
parent 00b90d9d28
commit de9cd332a5
13 changed files with 71684 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ModelTraining/ClipboardSemantics/IterativeResearch/macos-replay.json
+481
View File
@@ -0,0 +1,481 @@
{
"automaticPromotionAllowed": false,
"baseCorpus": "ModelTraining/ClipboardSemantics/clipboard_semantic_corpus.jsonl",
"configuration": {
"alpha": 2e-05,
"augmentation": "none",
"char_max": 5,
"char_min": 2,
"external_weight": 0.5700000000000001,
"hard_example_weight": 1.0,
"l1_ratio": 0.0,
"max_char_features": 45000,
"max_word_features": 22000,
"min_df": 2,
"pseudo_weight": 0.16499999999999998,
"round": 1,
"word_max": 2
},
"openCorpus": "ModelTraining/ClipboardSemantics/open-training-corpus.jsonl",
"reason": "Surrogate feature weights are not deployable NLModel assets and cannot bypass the existing macOS acceptance policy.",
"requiredCommands": [
"python3 Scripts/clipboard_semantics/generate_open_training_corpus.py",
"xcrun swift Scripts/clipboard_semantics/train_models.swift --algorithms maxEnt --corpus ModelTraining/ClipboardSemantics/combined-training-corpus.jsonl",
"xcrun swift Scripts/clipboard_semantics/evaluate_random_holdout.swift --corpus ModelTraining/ClipboardSemantics/random-holdout-corpus.jsonl"
],
"schemaVersion": 1,
"seed": 20260827,
"selectedRound": 1,
"thresholds": {
"blessing": {
"abstained": false,
"byLanguage": {
"en": {
"abstained": false,
"metrics": {
"f1": 0.9318,
"falseNegative": 4,
"falsePositive": 2,
"precision": 0.9535,
"predictedPositive": 43,
"recall": 0.9111,
"total": 1485,
"trueNegative": 1438,
"truePositive": 41,
"wilsonPrecisionLower95": 0.8454
},
"threshold": 0.915
},
"zh-Hans": {
"abstained": false,
"metrics": {
"f1": 0.7568,
"falseNegative": 17,
"falsePositive": 1,
"precision": 0.9655,
"predictedPositive": 29,
"recall": 0.6222,
"total": 1485,
"trueNegative": 1439,
"truePositive": 28,
"wilsonPrecisionLower95": 0.8282
},
"threshold": 0.9
}
},
"metrics": {
"f1": 0.8375,
"falseNegative": 23,
"falsePositive": 3,
"precision": 0.9571,
"predictedPositive": 70,
"recall": 0.7444,
"total": 2970,
"trueNegative": 2877,
"truePositive": 67,
"wilsonPrecisionLower95": 0.8814
},
"threshold": 0.915
},
"complaint": {
"abstained": false,
"byLanguage": {
"en": {
"abstained": false,
"metrics": {
"f1": 1.0,
"falseNegative": 0,
"falsePositive": 0,
"precision": 1.0,
"predictedPositive": 180,
"recall": 1.0,
"total": 1485,
"trueNegative": 1305,
"truePositive": 180,
"wilsonPrecisionLower95": 0.9791
},
"threshold": 0.5
},
"zh-Hans": {
"abstained": false,
"metrics": {
"f1": 0.9526,
"falseNegative": 9,
"falsePositive": 8,
"precision": 0.9553,
"predictedPositive": 179,
"recall": 0.95,
"total": 1485,
"trueNegative": 1297,
"truePositive": 171,
"wilsonPrecisionLower95": 0.9143
},
"threshold": 0.54
}
},
"metrics": {
"f1": 0.9764,
"falseNegative": 9,
"falsePositive": 8,
"precision": 0.9777,
"predictedPositive": 359,
"recall": 0.975,
"total": 2970,
"trueNegative": 2602,
"truePositive": 351,
"wilsonPrecisionLower95": 0.9567
},
"threshold": 0.54
},
"confirmationDecision": {
"abstained": false,
"byLanguage": {
"en": {
"abstained": false,
"metrics": {
"f1": 0.8848,
"falseNegative": 17,
"falsePositive": 2,
"precision": 0.9733,
"predictedPositive": 75,
"recall": 0.8111,
"total": 1485,
"trueNegative": 1393,
"truePositive": 73,
"wilsonPrecisionLower95": 0.9079
},
"threshold": 0.5
},
"zh-Hans": {
"abstained": false,
"metrics": {
"f1": 0.7947,
"falseNegative": 30,
"falsePositive": 1,
"precision": 0.9836,
"predictedPositive": 61,
"recall": 0.6667,
"total": 1485,
"trueNegative": 1394,
"truePositive": 60,
"wilsonPrecisionLower95": 0.9128
},
"threshold": 0.5
}
},
"metrics": {
"f1": 0.8418,
"falseNegative": 47,
"falsePositive": 3,
"precision": 0.9779,
"predictedPositive": 136,
"recall": 0.7389,
"total": 2970,
"trueNegative": 2787,
"truePositive": 133,
"wilsonPrecisionLower95": 0.9372
},
"threshold": 0.5
},
"followUpReminder": {
"abstained": false,
"byLanguage": {
"en": {
"abstained": false,
"metrics": {
"f1": 0.8554,
"falseNegative": 41,
"falsePositive": 6,
"precision": 0.9586,
"predictedPositive": 145,
"recall": 0.7722,
"total": 1485,
"trueNegative": 1299,
"truePositive": 139,
"wilsonPrecisionLower95": 0.9127
},
"threshold": 0.83
},
"zh-Hans": {
"abstained": false,
"metrics": {
"f1": 0.8354,
"falseNegative": 48,
"falsePositive": 4,
"precision": 0.9706,
"predictedPositive": 136,
"recall": 0.7333,
"total": 1485,
"trueNegative": 1301,
"truePositive": 132,
"wilsonPrecisionLower95": 0.9268
},
"threshold": 0.505
}
},
"metrics": {
"f1": 0.7994,
"falseNegative": 113,
"falsePositive": 11,
"precision": 0.9574,
"predictedPositive": 258,
"recall": 0.6861,
"total": 2970,
"trueNegative": 2599,
"truePositive": 247,
"wilsonPrecisionLower95": 0.9253
},
"threshold": 0.815
},
"invitation": {
"abstained": false,
"byLanguage": {
"en": {
"abstained": false,
"metrics": {
"f1": 0.8302,
"falseNegative": 24,
"falsePositive": 3,
"precision": 0.9565,
"predictedPositive": 69,
"recall": 0.7333,
"total": 1485,
"trueNegative": 1392,
"truePositive": 66,
"wilsonPrecisionLower95": 0.8798
},
"threshold": 0.935
},
"zh-Hans": {
"abstained": false,
"metrics": {
"f1": 0.6765,
"falseNegative": 44,
"falsePositive": 0,
"precision": 1.0,
"predictedPositive": 46,
"recall": 0.5111,
"total": 1485,
"trueNegative": 1395,
"truePositive": 46,
"wilsonPrecisionLower95": 0.9229
},
"threshold": 0.505
}
},
"metrics": {
"f1": 0.5476,
"falseNegative": 111,
"falsePositive": 3,
"precision": 0.9583,
"predictedPositive": 72,
"recall": 0.3833,
"total": 2970,
"trueNegative": 2787,
"truePositive": 69,
"wilsonPrecisionLower95": 0.8845
},
"threshold": 0.935
},
"question": {
"abstained": false,
"byLanguage": {
"en": {
"abstained": false,
"metrics": {
"f1": 0.8445,
"falseNegative": 76,
"falsePositive": 12,
"precision": 0.9522,
"predictedPositive": 251,
"recall": 0.7587,
"total": 1485,
"trueNegative": 1158,
"truePositive": 239,
"wilsonPrecisionLower95": 0.9183
},
"threshold": 0.725
},
"zh-Hans": {
"abstained": false,
"metrics": {
"f1": 0.8536,
"falseNegative": 76,
"falsePositive": 6,
"precision": 0.9755,
"predictedPositive": 245,
"recall": 0.7587,
"total": 1485,
"trueNegative": 1164,
"truePositive": 239,
"wilsonPrecisionLower95": 0.9476
},
"threshold": 0.5
}
},
"metrics": {
"f1": 0.8729,
"falseNegative": 122,
"falsePositive": 26,
"precision": 0.9513,
"predictedPositive": 534,
"recall": 0.8063,
"total": 2970,
"trueNegative": 2314,
"truePositive": 508,
"wilsonPrecisionLower95": 0.9296
},
"threshold": 0.515
},
"replyableMessage": {
"abstained": false,
"byLanguage": {
"en": {
"abstained": false,
"metrics": {
"f1": 0.948,
"falseNegative": 52,
"falsePositive": 46,
"precision": 0.951,
"predictedPositive": 939,
"recall": 0.945,
"total": 1485,
"trueNegative": 494,
"truePositive": 893,
"wilsonPrecisionLower95": 0.9353
},
"threshold": 0.7
},
"zh-Hans": {
"abstained": false,
"metrics": {
"f1": 0.9424,
"falseNegative": 62,
"falsePositive": 46,
"precision": 0.9505,
"predictedPositive": 929,
"recall": 0.9344,
"total": 1485,
"trueNegative": 494,
"truePositive": 883,
"wilsonPrecisionLower95": 0.9346
},
"threshold": 0.695
}
},
"metrics": {
"f1": 0.9454,
"falseNegative": 115,
"falsePositive": 90,
"precision": 0.9517,
"predictedPositive": 1865,
"recall": 0.9392,
"total": 2970,
"trueNegative": 990,
"truePositive": 1775,
"wilsonPrecisionLower95": 0.9411
},
"threshold": 0.7
},
"scheduleNegotiation": {
"abstained": false,
"byLanguage": {
"en": {
"abstained": false,
"metrics": {
"f1": 1.0,
"falseNegative": 0,
"falsePositive": 0,
"precision": 1.0,
"predictedPositive": 45,
"recall": 1.0,
"total": 1485,
"trueNegative": 1440,
"truePositive": 45,
"wilsonPrecisionLower95": 0.9213
},
"threshold": 0.5
},
"zh-Hans": {
"abstained": false,
"metrics": {
"f1": 0.967,
"falseNegative": 1,
"falsePositive": 2,
"precision": 0.9565,
"predictedPositive": 46,
"recall": 0.9778,
"total": 1485,
"trueNegative": 1438,
"truePositive": 44,
"wilsonPrecisionLower95": 0.8547
},
"threshold": 0.93
}
},
"metrics": {
"f1": 0.978,
"falseNegative": 1,
"falsePositive": 3,
"precision": 0.9674,
"predictedPositive": 92,
"recall": 0.9889,
"total": 2970,
"trueNegative": 2877,
"truePositive": 89,
"wilsonPrecisionLower95": 0.9085
},
"threshold": 0.91
},
"task": {
"abstained": false,
"byLanguage": {
"en": {
"abstained": false,
"metrics": {
"f1": 0.8403,
"falseNegative": 89,
"falsePositive": 14,
"precision": 0.9509,
"predictedPositive": 285,
"recall": 0.7528,
"total": 1485,
"trueNegative": 1111,
"truePositive": 271,
"wilsonPrecisionLower95": 0.9192
},
"threshold": 0.835
},
"zh-Hans": {
"abstained": false,
"metrics": {
"f1": 0.8278,
"falseNegative": 98,
"falsePositive": 11,
"precision": 0.9597,
"predictedPositive": 273,
"recall": 0.7278,
"total": 1485,
"trueNegative": 1114,
"truePositive": 262,
"wilsonPrecisionLower95": 0.9293
},
"threshold": 0.745
}
},
"metrics": {
"f1": 0.8637,
"falseNegative": 153,
"falsePositive": 26,
"precision": 0.9562,
"predictedPositive": 593,
"recall": 0.7875,
"total": 2970,
"trueNegative": 2224,
"truePositive": 567,
"wilsonPrecisionLower95": 0.9365
},
"threshold": 0.755
}
}
}