docs(semantics): record iterative retraining study
Co-authored-by: Rocky <hkgood@users.noreply.github.com>
This commit is contained in:
Cursor Agent
@@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|||||||
### Changed
|
### Changed
|
||||||
- **Clipboard intent precision**: specialize the local task and complaint models, add conservative language thresholds, and suppress complaint-only text from task routing unless it contains an explicit assignment. / **剪贴板意图精度**：专项优化本地任务与投诉模型，加入保守的分语言阈值，并在没有明确委派时阻止纯投诉文本进入任务路由。
|
- **Clipboard intent precision**: specialize the local task and complaint models, add conservative language thresholds, and suppress complaint-only text from task routing unless it contains an explicit assignment. / **剪贴板意图精度**：专项优化本地任务与投诉模型，加入保守的分语言阈值，并在没有明确委派时阻止纯投诉文本进入任务路由。
|
||||||
- **Precision-first semantic routing**: add traceable multi-model consensus labels, local joint intent verifiers, confidence-margin abstention, layered source-isolated evaluation, and privacy-safe shadow disagreement counters while retaining current models until every acceptance gate passes. / **精度优先语义路由**：新增可追溯多模型共识银标、本地联合意图验证器、置信度与差值拒识、按来源隔离的分层评估及隐私安全的影子分歧计数；所有验收门槛通过前继续保留当前模型。
|
- **Precision-first semantic routing**: add traceable multi-model consensus labels, local joint intent verifiers, confidence-margin abstention, layered source-isolated evaluation, and privacy-safe shadow disagreement counters while retaining current models until every acceptance gate passes. / **精度优先语义路由**：新增可追溯多模型共识银标、本地联合意图验证器、置信度与差值拒识、按来源隔离的分层评估及隐私安全的影子分歧计数；所有验收门槛通过前继续保留当前模型。
|
||||||
|
- **Iterative semantic research**: add reproducible twenty-round weakly supervised model selection, high-confidence self-training consensus, per-intent fine-tuning, explicit-evidence experiments, frozen-overlap gates, and macOS replay manifests; retain deployed models when every candidate underperforms release baselines. / **迭代语义研究**：新增可复现的二十轮弱监督模型选择、高置信自训练共识、分意图微调、显式证据实验、冻结集重叠门禁及 macOS 重放清单；所有候选低于发布基线时继续保留现有部署模型。
|
||||||
- **Card depth**: add a faint directional long shadow to card surfaces throughout the iOS app in both light and dark appearances. / **卡片层次**：为 iOS App 内的卡片界面统一加入淡雅的定向长投影，并适配浅色与深色外观。
|
- **Card depth**: add a faint directional long shadow to card surfaces throughout the iOS app in both light and dark appearances. / **卡片层次**：为 iOS App 内的卡片界面统一加入淡雅的定向长投影，并适配浅色与深色外观。
|
||||||
- **Neutral dialog actions**: use adaptive black or white styling for system alerts, confirmation dialogs, and modal actions instead of inheriting the green brand accent. / **中性弹窗操作色**：系统警告、确认弹窗及模态操作改用自适应黑色或白色，不再继承绿色品牌强调色。
|
- **Neutral dialog actions**: use adaptive black or white styling for system alerts, confirmation dialogs, and modal actions instead of inheriting the green brand accent. / **中性弹窗操作色**：系统警告、确认弹窗及模态操作改用自适应黑色或白色，不再继承绿色品牌强调色。
|
||||||
- **Monthly usage calendar**: replace the seven-day Home and Mac Dashboard chart with a full current-month calendar whose date-circle opacity reflects each day's dictation characters, placing the Home calendar between its four metrics and Personal Dictionary. / **月度用量日历**：以完整的当月日历替换首页与 Mac 总览的近七天图表，并通过日期圆形背景透明度表示每天的听写字数；首页日历位于四项统计指标与个性词库之间。
|
- **Monthly usage calendar**: replace the seven-day Home and Mac Dashboard chart with a full current-month calendar whose date-circle opacity reflects each day's dictation characters, placing the Home calendar between its four metrics and Personal Dictionary. / **月度用量日历**：以完整的当月日历替换首页与 Mac 总览的近七天图表，并通过日期圆形背景透明度表示每天的听写字数；首页日历位于四项统计指标与个性词库之间。
|
||||||
|
|||||||
+8422
File diff suppressed because it is too large
Load Diff
+629
@@ -0,0 +1,629 @@
|
|||||||
|
{
|
||||||
|
"automaticPromotionAllowed": false,
|
||||||
|
"baseCorpus": "ModelTraining/ClipboardSemantics/clipboard_semantic_corpus.jsonl",
|
||||||
|
"configuration": {
|
||||||
|
"alpha": 2e-05,
|
||||||
|
"augmentation": "none",
|
||||||
|
"char_max": 5,
|
||||||
|
"char_min": 2,
|
||||||
|
"external_weight": 0.5700000000000001,
|
||||||
|
"hard_example_weight": 1.0,
|
||||||
|
"l1_ratio": 0.0,
|
||||||
|
"max_char_features": 45000,
|
||||||
|
"max_word_features": 22000,
|
||||||
|
"min_df": 2,
|
||||||
|
"pseudo_weight": 0.16499999999999998,
|
||||||
|
"round": 1,
|
||||||
|
"word_max": 2
|
||||||
|
},
|
||||||
|
"configurationsByIntent": {
|
||||||
|
"blessing": {
|
||||||
|
"alpha": 1.2e-06,
|
||||||
|
"augmentation": "numeric",
|
||||||
|
"char_max": 8,
|
||||||
|
"char_min": 3,
|
||||||
|
"external_weight": 0.93,
|
||||||
|
"hard_example_weight": 2.7,
|
||||||
|
"l1_ratio": 0.25,
|
||||||
|
"max_char_features": 95000,
|
||||||
|
"max_word_features": 45000,
|
||||||
|
"min_df": 1,
|
||||||
|
"pseudo_weight": 0.43499999999999994,
|
||||||
|
"round": 19,
|
||||||
|
"word_max": 3
|
||||||
|
},
|
||||||
|
"complaint": {
|
||||||
|
"alpha": 2e-05,
|
||||||
|
"augmentation": "none",
|
||||||
|
"char_max": 5,
|
||||||
|
"char_min": 2,
|
||||||
|
"external_weight": 0.5700000000000001,
|
||||||
|
"hard_example_weight": 1.0,
|
||||||
|
"l1_ratio": 0.0,
|
||||||
|
"max_char_features": 45000,
|
||||||
|
"max_word_features": 22000,
|
||||||
|
"min_df": 2,
|
||||||
|
"pseudo_weight": 0.16499999999999998,
|
||||||
|
"round": 1,
|
||||||
|
"word_max": 2
|
||||||
|
},
|
||||||
|
"confirmationDecision": {
|
||||||
|
"alpha": 5e-06,
|
||||||
|
"augmentation": "prefix",
|
||||||
|
"char_max": 6,
|
||||||
|
"char_min": 2,
|
||||||
|
"external_weight": 0.77,
|
||||||
|
"hard_example_weight": 1.8,
|
||||||
|
"l1_ratio": 0.1,
|
||||||
|
"max_char_features": 70000,
|
||||||
|
"max_word_features": 30000,
|
||||||
|
"min_df": 2,
|
||||||
|
"pseudo_weight": 0.31499999999999995,
|
||||||
|
"round": 11,
|
||||||
|
"word_max": 2
|
||||||
|
},
|
||||||
|
"followUpReminder": {
|
||||||
|
"alpha": 2.5e-06,
|
||||||
|
"augmentation": "prefix",
|
||||||
|
"char_max": 7,
|
||||||
|
"char_min": 3,
|
||||||
|
"external_weight": 0.8300000000000001,
|
||||||
|
"hard_example_weight": 2.2,
|
||||||
|
"l1_ratio": 0.2,
|
||||||
|
"max_char_features": 80000,
|
||||||
|
"max_word_features": 40000,
|
||||||
|
"min_df": 1,
|
||||||
|
"pseudo_weight": 0.36,
|
||||||
|
"round": 14,
|
||||||
|
"word_max": 3
|
||||||
|
},
|
||||||
|
"invitation": {
|
||||||
|
"alpha": 2e-05,
|
||||||
|
"augmentation": "none",
|
||||||
|
"char_max": 5,
|
||||||
|
"char_min": 2,
|
||||||
|
"external_weight": 0.5700000000000001,
|
||||||
|
"hard_example_weight": 1.0,
|
||||||
|
"l1_ratio": 0.0,
|
||||||
|
"max_char_features": 45000,
|
||||||
|
"max_word_features": 22000,
|
||||||
|
"min_df": 2,
|
||||||
|
"pseudo_weight": 0.16499999999999998,
|
||||||
|
"round": 1,
|
||||||
|
"word_max": 2
|
||||||
|
},
|
||||||
|
"question": {
|
||||||
|
"alpha": 2e-05,
|
||||||
|
"augmentation": "none",
|
||||||
|
"char_max": 5,
|
||||||
|
"char_min": 2,
|
||||||
|
"external_weight": 0.5700000000000001,
|
||||||
|
"hard_example_weight": 1.0,
|
||||||
|
"l1_ratio": 0.0,
|
||||||
|
"max_char_features": 45000,
|
||||||
|
"max_word_features": 22000,
|
||||||
|
"min_df": 2,
|
||||||
|
"pseudo_weight": 0.16499999999999998,
|
||||||
|
"round": 1,
|
||||||
|
"word_max": 2
|
||||||
|
},
|
||||||
|
"replyableMessage": {
|
||||||
|
"alpha": 2e-05,
|
||||||
|
"augmentation": "none",
|
||||||
|
"char_max": 5,
|
||||||
|
"char_min": 2,
|
||||||
|
"external_weight": 0.5700000000000001,
|
||||||
|
"hard_example_weight": 1.0,
|
||||||
|
"l1_ratio": 0.0,
|
||||||
|
"max_char_features": 45000,
|
||||||
|
"max_word_features": 22000,
|
||||||
|
"min_df": 2,
|
||||||
|
"pseudo_weight": 0.16499999999999998,
|
||||||
|
"round": 1,
|
||||||
|
"word_max": 2
|
||||||
|
},
|
||||||
|
"scheduleNegotiation": {
|
||||||
|
"alpha": 6e-06,
|
||||||
|
"augmentation": "punctuation",
|
||||||
|
"char_max": 6,
|
||||||
|
"char_min": 3,
|
||||||
|
"external_weight": 0.6900000000000001,
|
||||||
|
"hard_example_weight": 1.5,
|
||||||
|
"l1_ratio": 0.1,
|
||||||
|
"max_char_features": 65000,
|
||||||
|
"max_word_features": 32000,
|
||||||
|
"min_df": 2,
|
||||||
|
"pseudo_weight": 0.255,
|
||||||
|
"round": 7,
|
||||||
|
"word_max": 3
|
||||||
|
},
|
||||||
|
"task": {
|
||||||
|
"alpha": 2e-05,
|
||||||
|
"augmentation": "none",
|
||||||
|
"char_max": 5,
|
||||||
|
"char_min": 2,
|
||||||
|
"external_weight": 0.5700000000000001,
|
||||||
|
"hard_example_weight": 1.0,
|
||||||
|
"l1_ratio": 0.0,
|
||||||
|
"max_char_features": 45000,
|
||||||
|
"max_word_features": 22000,
|
||||||
|
"min_df": 2,
|
||||||
|
"pseudo_weight": 0.16499999999999998,
|
||||||
|
"round": 1,
|
||||||
|
"word_max": 2
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"openCorpus": "ModelTraining/ClipboardSemantics/open-training-corpus.jsonl",
|
||||||
|
"reason": "Surrogate feature weights are not deployable NLModel assets and cannot bypass the existing macOS acceptance policy.",
|
||||||
|
"requiredCommands": [
|
||||||
|
"python3 Scripts/clipboard_semantics/generate_open_training_corpus.py",
|
||||||
|
"xcrun swift Scripts/clipboard_semantics/train_models.swift --algorithms maxEnt --corpus ModelTraining/ClipboardSemantics/combined-training-corpus.jsonl",
|
||||||
|
"xcrun swift Scripts/clipboard_semantics/evaluate_random_holdout.swift --corpus ModelTraining/ClipboardSemantics/random-holdout-corpus.jsonl"
|
||||||
|
],
|
||||||
|
"schemaVersion": 1,
|
||||||
|
"seed": 20260827,
|
||||||
|
"selectedRound": 1,
|
||||||
|
"selectedRoundsByIntent": {
|
||||||
|
"blessing": 19,
|
||||||
|
"complaint": 1,
|
||||||
|
"confirmationDecision": 11,
|
||||||
|
"followUpReminder": 14,
|
||||||
|
"invitation": 1,
|
||||||
|
"question": 1,
|
||||||
|
"replyableMessage": 1,
|
||||||
|
"scheduleNegotiation": 7,
|
||||||
|
"task": 1
|
||||||
|
},
|
||||||
|
"thresholds": {
|
||||||
|
"blessing": {
|
||||||
|
"abstained": true,
|
||||||
|
"byLanguage": {
|
||||||
|
"en": {
|
||||||
|
"abstained": true,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.0,
|
||||||
|
"falseNegative": 0,
|
||||||
|
"falsePositive": 0,
|
||||||
|
"precision": 0.0,
|
||||||
|
"predictedPositive": 0,
|
||||||
|
"recall": 0.0,
|
||||||
|
"total": 0,
|
||||||
|
"trueNegative": 0,
|
||||||
|
"truePositive": 0,
|
||||||
|
"wilsonPrecisionLower95": 0
|
||||||
|
},
|
||||||
|
"threshold": 1.0
|
||||||
|
},
|
||||||
|
"zh-Hans": {
|
||||||
|
"abstained": true,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.0,
|
||||||
|
"falseNegative": 0,
|
||||||
|
"falsePositive": 0,
|
||||||
|
"precision": 0.0,
|
||||||
|
"predictedPositive": 0,
|
||||||
|
"recall": 0.0,
|
||||||
|
"total": 0,
|
||||||
|
"trueNegative": 0,
|
||||||
|
"truePositive": 0,
|
||||||
|
"wilsonPrecisionLower95": 0
|
||||||
|
},
|
||||||
|
"threshold": 1.0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.0,
|
||||||
|
"falseNegative": 0,
|
||||||
|
"falsePositive": 0,
|
||||||
|
"precision": 0.0,
|
||||||
|
"predictedPositive": 0,
|
||||||
|
"recall": 0.0,
|
||||||
|
"total": 0,
|
||||||
|
"trueNegative": 0,
|
||||||
|
"truePositive": 0,
|
||||||
|
"wilsonPrecisionLower95": 0
|
||||||
|
},
|
||||||
|
"threshold": 1.0
|
||||||
|
},
|
||||||
|
"complaint": {
|
||||||
|
"abstained": false,
|
||||||
|
"byLanguage": {
|
||||||
|
"en": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 1.0,
|
||||||
|
"falseNegative": 0,
|
||||||
|
"falsePositive": 0,
|
||||||
|
"precision": 1.0,
|
||||||
|
"predictedPositive": 180,
|
||||||
|
"recall": 1.0,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 1305,
|
||||||
|
"truePositive": 180,
|
||||||
|
"wilsonPrecisionLower95": 0.9791
|
||||||
|
},
|
||||||
|
"threshold": 0.5
|
||||||
|
},
|
||||||
|
"zh-Hans": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.9526,
|
||||||
|
"falseNegative": 9,
|
||||||
|
"falsePositive": 8,
|
||||||
|
"precision": 0.9553,
|
||||||
|
"predictedPositive": 179,
|
||||||
|
"recall": 0.95,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 1297,
|
||||||
|
"truePositive": 171,
|
||||||
|
"wilsonPrecisionLower95": 0.9143
|
||||||
|
},
|
||||||
|
"threshold": 0.54
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.9764,
|
||||||
|
"falseNegative": 9,
|
||||||
|
"falsePositive": 8,
|
||||||
|
"precision": 0.9777,
|
||||||
|
"predictedPositive": 359,
|
||||||
|
"recall": 0.975,
|
||||||
|
"total": 2970,
|
||||||
|
"trueNegative": 2602,
|
||||||
|
"truePositive": 351,
|
||||||
|
"wilsonPrecisionLower95": 0.9567
|
||||||
|
},
|
||||||
|
"threshold": 0.54
|
||||||
|
},
|
||||||
|
"confirmationDecision": {
|
||||||
|
"abstained": false,
|
||||||
|
"byLanguage": {
|
||||||
|
"en": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.931,
|
||||||
|
"falseNegative": 9,
|
||||||
|
"falsePositive": 3,
|
||||||
|
"precision": 0.9643,
|
||||||
|
"predictedPositive": 84,
|
||||||
|
"recall": 0.9,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 1392,
|
||||||
|
"truePositive": 81,
|
||||||
|
"wilsonPrecisionLower95": 0.9002
|
||||||
|
},
|
||||||
|
"threshold": 0.97
|
||||||
|
},
|
||||||
|
"zh-Hans": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.7222,
|
||||||
|
"falseNegative": 38,
|
||||||
|
"falsePositive": 2,
|
||||||
|
"precision": 0.963,
|
||||||
|
"predictedPositive": 54,
|
||||||
|
"recall": 0.5778,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 1393,
|
||||||
|
"truePositive": 52,
|
||||||
|
"wilsonPrecisionLower95": 0.8746
|
||||||
|
},
|
||||||
|
"threshold": 0.86
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.8077,
|
||||||
|
"falseNegative": 54,
|
||||||
|
"falsePositive": 6,
|
||||||
|
"precision": 0.9545,
|
||||||
|
"predictedPositive": 132,
|
||||||
|
"recall": 0.7,
|
||||||
|
"total": 2970,
|
||||||
|
"trueNegative": 2784,
|
||||||
|
"truePositive": 126,
|
||||||
|
"wilsonPrecisionLower95": 0.9044
|
||||||
|
},
|
||||||
|
"threshold": 0.95
|
||||||
|
},
|
||||||
|
"followUpReminder": {
|
||||||
|
"abstained": false,
|
||||||
|
"byLanguage": {
|
||||||
|
"en": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.8788,
|
||||||
|
"falseNegative": 35,
|
||||||
|
"falsePositive": 5,
|
||||||
|
"precision": 0.9667,
|
||||||
|
"predictedPositive": 150,
|
||||||
|
"recall": 0.8056,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 1300,
|
||||||
|
"truePositive": 145,
|
||||||
|
"wilsonPrecisionLower95": 0.9243
|
||||||
|
},
|
||||||
|
"threshold": 0.845
|
||||||
|
},
|
||||||
|
"zh-Hans": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.6113,
|
||||||
|
"falseNegative": 99,
|
||||||
|
"falsePositive": 4,
|
||||||
|
"precision": 0.9529,
|
||||||
|
"predictedPositive": 85,
|
||||||
|
"recall": 0.45,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 1301,
|
||||||
|
"truePositive": 81,
|
||||||
|
"wilsonPrecisionLower95": 0.8852
|
||||||
|
},
|
||||||
|
"threshold": 0.97
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.7682,
|
||||||
|
"falseNegative": 128,
|
||||||
|
"falsePositive": 12,
|
||||||
|
"precision": 0.9508,
|
||||||
|
"predictedPositive": 244,
|
||||||
|
"recall": 0.6444,
|
||||||
|
"total": 2970,
|
||||||
|
"trueNegative": 2598,
|
||||||
|
"truePositive": 232,
|
||||||
|
"wilsonPrecisionLower95": 0.916
|
||||||
|
},
|
||||||
|
"threshold": 0.91
|
||||||
|
},
|
||||||
|
"invitation": {
|
||||||
|
"abstained": false,
|
||||||
|
"byLanguage": {
|
||||||
|
"en": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.8302,
|
||||||
|
"falseNegative": 24,
|
||||||
|
"falsePositive": 3,
|
||||||
|
"precision": 0.9565,
|
||||||
|
"predictedPositive": 69,
|
||||||
|
"recall": 0.7333,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 1392,
|
||||||
|
"truePositive": 66,
|
||||||
|
"wilsonPrecisionLower95": 0.8798
|
||||||
|
},
|
||||||
|
"threshold": 0.935
|
||||||
|
},
|
||||||
|
"zh-Hans": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.6765,
|
||||||
|
"falseNegative": 44,
|
||||||
|
"falsePositive": 0,
|
||||||
|
"precision": 1.0,
|
||||||
|
"predictedPositive": 46,
|
||||||
|
"recall": 0.5111,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 1395,
|
||||||
|
"truePositive": 46,
|
||||||
|
"wilsonPrecisionLower95": 0.9229
|
||||||
|
},
|
||||||
|
"threshold": 0.505
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.5476,
|
||||||
|
"falseNegative": 111,
|
||||||
|
"falsePositive": 3,
|
||||||
|
"precision": 0.9583,
|
||||||
|
"predictedPositive": 72,
|
||||||
|
"recall": 0.3833,
|
||||||
|
"total": 2970,
|
||||||
|
"trueNegative": 2787,
|
||||||
|
"truePositive": 69,
|
||||||
|
"wilsonPrecisionLower95": 0.8845
|
||||||
|
},
|
||||||
|
"threshold": 0.935
|
||||||
|
},
|
||||||
|
"question": {
|
||||||
|
"abstained": false,
|
||||||
|
"byLanguage": {
|
||||||
|
"en": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.8445,
|
||||||
|
"falseNegative": 76,
|
||||||
|
"falsePositive": 12,
|
||||||
|
"precision": 0.9522,
|
||||||
|
"predictedPositive": 251,
|
||||||
|
"recall": 0.7587,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 1158,
|
||||||
|
"truePositive": 239,
|
||||||
|
"wilsonPrecisionLower95": 0.9183
|
||||||
|
},
|
||||||
|
"threshold": 0.725
|
||||||
|
},
|
||||||
|
"zh-Hans": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.8536,
|
||||||
|
"falseNegative": 76,
|
||||||
|
"falsePositive": 6,
|
||||||
|
"precision": 0.9755,
|
||||||
|
"predictedPositive": 245,
|
||||||
|
"recall": 0.7587,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 1164,
|
||||||
|
"truePositive": 239,
|
||||||
|
"wilsonPrecisionLower95": 0.9476
|
||||||
|
},
|
||||||
|
"threshold": 0.5
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.8729,
|
||||||
|
"falseNegative": 122,
|
||||||
|
"falsePositive": 26,
|
||||||
|
"precision": 0.9513,
|
||||||
|
"predictedPositive": 534,
|
||||||
|
"recall": 0.8063,
|
||||||
|
"total": 2970,
|
||||||
|
"trueNegative": 2314,
|
||||||
|
"truePositive": 508,
|
||||||
|
"wilsonPrecisionLower95": 0.9296
|
||||||
|
},
|
||||||
|
"threshold": 0.515
|
||||||
|
},
|
||||||
|
"replyableMessage": {
|
||||||
|
"abstained": false,
|
||||||
|
"byLanguage": {
|
||||||
|
"en": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.948,
|
||||||
|
"falseNegative": 52,
|
||||||
|
"falsePositive": 46,
|
||||||
|
"precision": 0.951,
|
||||||
|
"predictedPositive": 939,
|
||||||
|
"recall": 0.945,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 494,
|
||||||
|
"truePositive": 893,
|
||||||
|
"wilsonPrecisionLower95": 0.9353
|
||||||
|
},
|
||||||
|
"threshold": 0.7
|
||||||
|
},
|
||||||
|
"zh-Hans": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.9424,
|
||||||
|
"falseNegative": 62,
|
||||||
|
"falsePositive": 46,
|
||||||
|
"precision": 0.9505,
|
||||||
|
"predictedPositive": 929,
|
||||||
|
"recall": 0.9344,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 494,
|
||||||
|
"truePositive": 883,
|
||||||
|
"wilsonPrecisionLower95": 0.9346
|
||||||
|
},
|
||||||
|
"threshold": 0.695
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.9454,
|
||||||
|
"falseNegative": 115,
|
||||||
|
"falsePositive": 90,
|
||||||
|
"precision": 0.9517,
|
||||||
|
"predictedPositive": 1865,
|
||||||
|
"recall": 0.9392,
|
||||||
|
"total": 2970,
|
||||||
|
"trueNegative": 990,
|
||||||
|
"truePositive": 1775,
|
||||||
|
"wilsonPrecisionLower95": 0.9411
|
||||||
|
},
|
||||||
|
"threshold": 0.7
|
||||||
|
},
|
||||||
|
"scheduleNegotiation": {
|
||||||
|
"abstained": false,
|
||||||
|
"byLanguage": {
|
||||||
|
"en": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 1.0,
|
||||||
|
"falseNegative": 0,
|
||||||
|
"falsePositive": 0,
|
||||||
|
"precision": 1.0,
|
||||||
|
"predictedPositive": 45,
|
||||||
|
"recall": 1.0,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 1440,
|
||||||
|
"truePositive": 45,
|
||||||
|
"wilsonPrecisionLower95": 0.9213
|
||||||
|
},
|
||||||
|
"threshold": 0.66
|
||||||
|
},
|
||||||
|
"zh-Hans": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 1.0,
|
||||||
|
"falseNegative": 0,
|
||||||
|
"falsePositive": 0,
|
||||||
|
"precision": 1.0,
|
||||||
|
"predictedPositive": 45,
|
||||||
|
"recall": 1.0,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 1440,
|
||||||
|
"truePositive": 45,
|
||||||
|
"wilsonPrecisionLower95": 0.9213
|
||||||
|
},
|
||||||
|
"threshold": 0.55
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"metrics": {
|
||||||
|
"f1": 1.0,
|
||||||
|
"falseNegative": 0,
|
||||||
|
"falsePositive": 0,
|
||||||
|
"precision": 1.0,
|
||||||
|
"predictedPositive": 90,
|
||||||
|
"recall": 1.0,
|
||||||
|
"total": 2970,
|
||||||
|
"trueNegative": 2880,
|
||||||
|
"truePositive": 90,
|
||||||
|
"wilsonPrecisionLower95": 0.9591
|
||||||
|
},
|
||||||
|
"threshold": 0.66
|
||||||
|
},
|
||||||
|
"task": {
|
||||||
|
"abstained": false,
|
||||||
|
"byLanguage": {
|
||||||
|
"en": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.8403,
|
||||||
|
"falseNegative": 89,
|
||||||
|
"falsePositive": 14,
|
||||||
|
"precision": 0.9509,
|
||||||
|
"predictedPositive": 285,
|
||||||
|
"recall": 0.7528,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 1111,
|
||||||
|
"truePositive": 271,
|
||||||
|
"wilsonPrecisionLower95": 0.9192
|
||||||
|
},
|
||||||
|
"threshold": 0.835
|
||||||
|
},
|
||||||
|
"zh-Hans": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.8278,
|
||||||
|
"falseNegative": 98,
|
||||||
|
"falsePositive": 11,
|
||||||
|
"precision": 0.9597,
|
||||||
|
"predictedPositive": 273,
|
||||||
|
"recall": 0.7278,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 1114,
|
||||||
|
"truePositive": 262,
|
||||||
|
"wilsonPrecisionLower95": 0.9293
|
||||||
|
},
|
||||||
|
"threshold": 0.745
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.8637,
|
||||||
|
"falseNegative": 153,
|
||||||
|
"falsePositive": 26,
|
||||||
|
"precision": 0.9562,
|
||||||
|
"predictedPositive": 593,
|
||||||
|
"recall": 0.7875,
|
||||||
|
"total": 2970,
|
||||||
|
"trueNegative": 2224,
|
||||||
|
"truePositive": 567,
|
||||||
|
"wilsonPrecisionLower95": 0.9365
|
||||||
|
},
|
||||||
|
"threshold": 0.755
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+14831
File diff suppressed because it is too large
Load Diff
+8422
File diff suppressed because it is too large
Load Diff
+629
@@ -0,0 +1,629 @@
|
|||||||
|
{
|
||||||
|
"automaticPromotionAllowed": false,
|
||||||
|
"baseCorpus": "ModelTraining/ClipboardSemantics/clipboard_semantic_corpus.jsonl",
|
||||||
|
"configuration": {
|
||||||
|
"alpha": 2e-05,
|
||||||
|
"augmentation": "none",
|
||||||
|
"char_max": 5,
|
||||||
|
"char_min": 2,
|
||||||
|
"external_weight": 0.5700000000000001,
|
||||||
|
"hard_example_weight": 1.0,
|
||||||
|
"l1_ratio": 0.0,
|
||||||
|
"max_char_features": 45000,
|
||||||
|
"max_word_features": 22000,
|
||||||
|
"min_df": 2,
|
||||||
|
"pseudo_weight": 0.16499999999999998,
|
||||||
|
"round": 1,
|
||||||
|
"word_max": 2
|
||||||
|
},
|
||||||
|
"configurationsByIntent": {
|
||||||
|
"blessing": {
|
||||||
|
"alpha": 1.2e-06,
|
||||||
|
"augmentation": "numeric",
|
||||||
|
"char_max": 8,
|
||||||
|
"char_min": 3,
|
||||||
|
"external_weight": 0.93,
|
||||||
|
"hard_example_weight": 2.7,
|
||||||
|
"l1_ratio": 0.25,
|
||||||
|
"max_char_features": 95000,
|
||||||
|
"max_word_features": 45000,
|
||||||
|
"min_df": 1,
|
||||||
|
"pseudo_weight": 0.43499999999999994,
|
||||||
|
"round": 19,
|
||||||
|
"word_max": 3
|
||||||
|
},
|
||||||
|
"complaint": {
|
||||||
|
"alpha": 2e-05,
|
||||||
|
"augmentation": "none",
|
||||||
|
"char_max": 5,
|
||||||
|
"char_min": 2,
|
||||||
|
"external_weight": 0.5700000000000001,
|
||||||
|
"hard_example_weight": 1.0,
|
||||||
|
"l1_ratio": 0.0,
|
||||||
|
"max_char_features": 45000,
|
||||||
|
"max_word_features": 22000,
|
||||||
|
"min_df": 2,
|
||||||
|
"pseudo_weight": 0.16499999999999998,
|
||||||
|
"round": 1,
|
||||||
|
"word_max": 2
|
||||||
|
},
|
||||||
|
"confirmationDecision": {
|
||||||
|
"alpha": 1e-05,
|
||||||
|
"augmentation": "none",
|
||||||
|
"char_max": 6,
|
||||||
|
"char_min": 2,
|
||||||
|
"external_weight": 0.6100000000000001,
|
||||||
|
"hard_example_weight": 1.2,
|
||||||
|
"l1_ratio": 0.05,
|
||||||
|
"max_char_features": 55000,
|
||||||
|
"max_word_features": 25000,
|
||||||
|
"min_df": 2,
|
||||||
|
"pseudo_weight": 0.195,
|
||||||
|
"round": 3,
|
||||||
|
"word_max": 2
|
||||||
|
},
|
||||||
|
"followUpReminder": {
|
||||||
|
"alpha": 2e-05,
|
||||||
|
"augmentation": "none",
|
||||||
|
"char_max": 5,
|
||||||
|
"char_min": 2,
|
||||||
|
"external_weight": 0.5700000000000001,
|
||||||
|
"hard_example_weight": 1.0,
|
||||||
|
"l1_ratio": 0.0,
|
||||||
|
"max_char_features": 45000,
|
||||||
|
"max_word_features": 22000,
|
||||||
|
"min_df": 2,
|
||||||
|
"pseudo_weight": 0.16499999999999998,
|
||||||
|
"round": 1,
|
||||||
|
"word_max": 2
|
||||||
|
},
|
||||||
|
"invitation": {
|
||||||
|
"alpha": 2e-05,
|
||||||
|
"augmentation": "none",
|
||||||
|
"char_max": 5,
|
||||||
|
"char_min": 2,
|
||||||
|
"external_weight": 0.5700000000000001,
|
||||||
|
"hard_example_weight": 1.0,
|
||||||
|
"l1_ratio": 0.0,
|
||||||
|
"max_char_features": 45000,
|
||||||
|
"max_word_features": 22000,
|
||||||
|
"min_df": 2,
|
||||||
|
"pseudo_weight": 0.16499999999999998,
|
||||||
|
"round": 1,
|
||||||
|
"word_max": 2
|
||||||
|
},
|
||||||
|
"question": {
|
||||||
|
"alpha": 2e-05,
|
||||||
|
"augmentation": "none",
|
||||||
|
"char_max": 5,
|
||||||
|
"char_min": 2,
|
||||||
|
"external_weight": 0.5700000000000001,
|
||||||
|
"hard_example_weight": 1.0,
|
||||||
|
"l1_ratio": 0.0,
|
||||||
|
"max_char_features": 45000,
|
||||||
|
"max_word_features": 22000,
|
||||||
|
"min_df": 2,
|
||||||
|
"pseudo_weight": 0.16499999999999998,
|
||||||
|
"round": 1,
|
||||||
|
"word_max": 2
|
||||||
|
},
|
||||||
|
"replyableMessage": {
|
||||||
|
"alpha": 2e-05,
|
||||||
|
"augmentation": "none",
|
||||||
|
"char_max": 5,
|
||||||
|
"char_min": 2,
|
||||||
|
"external_weight": 0.5700000000000001,
|
||||||
|
"hard_example_weight": 1.0,
|
||||||
|
"l1_ratio": 0.0,
|
||||||
|
"max_char_features": 45000,
|
||||||
|
"max_word_features": 22000,
|
||||||
|
"min_df": 2,
|
||||||
|
"pseudo_weight": 0.16499999999999998,
|
||||||
|
"round": 1,
|
||||||
|
"word_max": 2
|
||||||
|
},
|
||||||
|
"scheduleNegotiation": {
|
||||||
|
"alpha": 6e-06,
|
||||||
|
"augmentation": "punctuation",
|
||||||
|
"char_max": 6,
|
||||||
|
"char_min": 3,
|
||||||
|
"external_weight": 0.6900000000000001,
|
||||||
|
"hard_example_weight": 1.5,
|
||||||
|
"l1_ratio": 0.1,
|
||||||
|
"max_char_features": 65000,
|
||||||
|
"max_word_features": 32000,
|
||||||
|
"min_df": 2,
|
||||||
|
"pseudo_weight": 0.255,
|
||||||
|
"round": 7,
|
||||||
|
"word_max": 3
|
||||||
|
},
|
||||||
|
"task": {
|
||||||
|
"alpha": 3.5e-06,
|
||||||
|
"augmentation": "prefix",
|
||||||
|
"char_max": 6,
|
||||||
|
"char_min": 2,
|
||||||
|
"external_weight": 0.8500000000000001,
|
||||||
|
"hard_example_weight": 2.2,
|
||||||
|
"l1_ratio": 0.2,
|
||||||
|
"max_char_features": 85000,
|
||||||
|
"max_word_features": 42000,
|
||||||
|
"min_df": 1,
|
||||||
|
"pseudo_weight": 0.375,
|
||||||
|
"round": 15,
|
||||||
|
"word_max": 3
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"openCorpus": "ModelTraining/ClipboardSemantics/open-training-corpus.jsonl",
|
||||||
|
"reason": "Surrogate feature weights are not deployable NLModel assets and cannot bypass the existing macOS acceptance policy.",
|
||||||
|
"requiredCommands": [
|
||||||
|
"python3 Scripts/clipboard_semantics/generate_open_training_corpus.py",
|
||||||
|
"xcrun swift Scripts/clipboard_semantics/train_models.swift --algorithms maxEnt --corpus ModelTraining/ClipboardSemantics/combined-training-corpus.jsonl",
|
||||||
|
"xcrun swift Scripts/clipboard_semantics/evaluate_random_holdout.swift --corpus ModelTraining/ClipboardSemantics/random-holdout-corpus.jsonl"
|
||||||
|
],
|
||||||
|
"schemaVersion": 1,
|
||||||
|
"seed": 20260827,
|
||||||
|
"selectedRound": 1,
|
||||||
|
"selectedRoundsByIntent": {
|
||||||
|
"blessing": 19,
|
||||||
|
"complaint": 1,
|
||||||
|
"confirmationDecision": 3,
|
||||||
|
"followUpReminder": 1,
|
||||||
|
"invitation": 1,
|
||||||
|
"question": 1,
|
||||||
|
"replyableMessage": 1,
|
||||||
|
"scheduleNegotiation": 7,
|
||||||
|
"task": 15
|
||||||
|
},
|
||||||
|
"thresholds": {
|
||||||
|
"blessing": {
|
||||||
|
"abstained": true,
|
||||||
|
"byLanguage": {
|
||||||
|
"en": {
|
||||||
|
"abstained": true,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.0,
|
||||||
|
"falseNegative": 0,
|
||||||
|
"falsePositive": 0,
|
||||||
|
"precision": 0.0,
|
||||||
|
"predictedPositive": 0,
|
||||||
|
"recall": 0.0,
|
||||||
|
"total": 0,
|
||||||
|
"trueNegative": 0,
|
||||||
|
"truePositive": 0,
|
||||||
|
"wilsonPrecisionLower95": 0
|
||||||
|
},
|
||||||
|
"threshold": 1.0
|
||||||
|
},
|
||||||
|
"zh-Hans": {
|
||||||
|
"abstained": true,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.0,
|
||||||
|
"falseNegative": 0,
|
||||||
|
"falsePositive": 0,
|
||||||
|
"precision": 0.0,
|
||||||
|
"predictedPositive": 0,
|
||||||
|
"recall": 0.0,
|
||||||
|
"total": 0,
|
||||||
|
"trueNegative": 0,
|
||||||
|
"truePositive": 0,
|
||||||
|
"wilsonPrecisionLower95": 0
|
||||||
|
},
|
||||||
|
"threshold": 1.0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.0,
|
||||||
|
"falseNegative": 0,
|
||||||
|
"falsePositive": 0,
|
||||||
|
"precision": 0.0,
|
||||||
|
"predictedPositive": 0,
|
||||||
|
"recall": 0.0,
|
||||||
|
"total": 0,
|
||||||
|
"trueNegative": 0,
|
||||||
|
"truePositive": 0,
|
||||||
|
"wilsonPrecisionLower95": 0
|
||||||
|
},
|
||||||
|
"threshold": 1.0
|
||||||
|
},
|
||||||
|
"complaint": {
|
||||||
|
"abstained": false,
|
||||||
|
"byLanguage": {
|
||||||
|
"en": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 1.0,
|
||||||
|
"falseNegative": 0,
|
||||||
|
"falsePositive": 0,
|
||||||
|
"precision": 1.0,
|
||||||
|
"predictedPositive": 180,
|
||||||
|
"recall": 1.0,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 1305,
|
||||||
|
"truePositive": 180,
|
||||||
|
"wilsonPrecisionLower95": 0.9791
|
||||||
|
},
|
||||||
|
"threshold": 0.5
|
||||||
|
},
|
||||||
|
"zh-Hans": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.9526,
|
||||||
|
"falseNegative": 9,
|
||||||
|
"falsePositive": 8,
|
||||||
|
"precision": 0.9553,
|
||||||
|
"predictedPositive": 179,
|
||||||
|
"recall": 0.95,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 1297,
|
||||||
|
"truePositive": 171,
|
||||||
|
"wilsonPrecisionLower95": 0.9143
|
||||||
|
},
|
||||||
|
"threshold": 0.54
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.9764,
|
||||||
|
"falseNegative": 9,
|
||||||
|
"falsePositive": 8,
|
||||||
|
"precision": 0.9777,
|
||||||
|
"predictedPositive": 359,
|
||||||
|
"recall": 0.975,
|
||||||
|
"total": 2970,
|
||||||
|
"trueNegative": 2602,
|
||||||
|
"truePositive": 351,
|
||||||
|
"wilsonPrecisionLower95": 0.9567
|
||||||
|
},
|
||||||
|
"threshold": 0.54
|
||||||
|
},
|
||||||
|
"confirmationDecision": {
|
||||||
|
"abstained": false,
|
||||||
|
"byLanguage": {
|
||||||
|
"en": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.9727,
|
||||||
|
"falseNegative": 1,
|
||||||
|
"falsePositive": 4,
|
||||||
|
"precision": 0.957,
|
||||||
|
"predictedPositive": 93,
|
||||||
|
"recall": 0.9889,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 1391,
|
||||||
|
"truePositive": 89,
|
||||||
|
"wilsonPrecisionLower95": 0.8946
|
||||||
|
},
|
||||||
|
"threshold": 0.585
|
||||||
|
},
|
||||||
|
"zh-Hans": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.7815,
|
||||||
|
"falseNegative": 31,
|
||||||
|
"falsePositive": 2,
|
||||||
|
"precision": 0.9672,
|
||||||
|
"predictedPositive": 61,
|
||||||
|
"recall": 0.6556,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 1393,
|
||||||
|
"truePositive": 59,
|
||||||
|
"wilsonPrecisionLower95": 0.8881
|
||||||
|
},
|
||||||
|
"threshold": 0.5
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.8701,
|
||||||
|
"falseNegative": 36,
|
||||||
|
"falsePositive": 7,
|
||||||
|
"precision": 0.9536,
|
||||||
|
"predictedPositive": 151,
|
||||||
|
"recall": 0.8,
|
||||||
|
"total": 2970,
|
||||||
|
"trueNegative": 2783,
|
||||||
|
"truePositive": 144,
|
||||||
|
"wilsonPrecisionLower95": 0.9074
|
||||||
|
},
|
||||||
|
"threshold": 0.555
|
||||||
|
},
|
||||||
|
"followUpReminder": {
|
||||||
|
"abstained": false,
|
||||||
|
"byLanguage": {
|
||||||
|
"en": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.8554,
|
||||||
|
"falseNegative": 41,
|
||||||
|
"falsePositive": 6,
|
||||||
|
"precision": 0.9586,
|
||||||
|
"predictedPositive": 145,
|
||||||
|
"recall": 0.7722,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 1299,
|
||||||
|
"truePositive": 139,
|
||||||
|
"wilsonPrecisionLower95": 0.9127
|
||||||
|
},
|
||||||
|
"threshold": 0.83
|
||||||
|
},
|
||||||
|
"zh-Hans": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.8354,
|
||||||
|
"falseNegative": 48,
|
||||||
|
"falsePositive": 4,
|
||||||
|
"precision": 0.9706,
|
||||||
|
"predictedPositive": 136,
|
||||||
|
"recall": 0.7333,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 1301,
|
||||||
|
"truePositive": 132,
|
||||||
|
"wilsonPrecisionLower95": 0.9268
|
||||||
|
},
|
||||||
|
"threshold": 0.505
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.7994,
|
||||||
|
"falseNegative": 113,
|
||||||
|
"falsePositive": 11,
|
||||||
|
"precision": 0.9574,
|
||||||
|
"predictedPositive": 258,
|
||||||
|
"recall": 0.6861,
|
||||||
|
"total": 2970,
|
||||||
|
"trueNegative": 2599,
|
||||||
|
"truePositive": 247,
|
||||||
|
"wilsonPrecisionLower95": 0.9253
|
||||||
|
},
|
||||||
|
"threshold": 0.815
|
||||||
|
},
|
||||||
|
"invitation": {
|
||||||
|
"abstained": false,
|
||||||
|
"byLanguage": {
|
||||||
|
"en": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.8302,
|
||||||
|
"falseNegative": 24,
|
||||||
|
"falsePositive": 3,
|
||||||
|
"precision": 0.9565,
|
||||||
|
"predictedPositive": 69,
|
||||||
|
"recall": 0.7333,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 1392,
|
||||||
|
"truePositive": 66,
|
||||||
|
"wilsonPrecisionLower95": 0.8798
|
||||||
|
},
|
||||||
|
"threshold": 0.935
|
||||||
|
},
|
||||||
|
"zh-Hans": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.6765,
|
||||||
|
"falseNegative": 44,
|
||||||
|
"falsePositive": 0,
|
||||||
|
"precision": 1.0,
|
||||||
|
"predictedPositive": 46,
|
||||||
|
"recall": 0.5111,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 1395,
|
||||||
|
"truePositive": 46,
|
||||||
|
"wilsonPrecisionLower95": 0.9229
|
||||||
|
},
|
||||||
|
"threshold": 0.505
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.5476,
|
||||||
|
"falseNegative": 111,
|
||||||
|
"falsePositive": 3,
|
||||||
|
"precision": 0.9583,
|
||||||
|
"predictedPositive": 72,
|
||||||
|
"recall": 0.3833,
|
||||||
|
"total": 2970,
|
||||||
|
"trueNegative": 2787,
|
||||||
|
"truePositive": 69,
|
||||||
|
"wilsonPrecisionLower95": 0.8845
|
||||||
|
},
|
||||||
|
"threshold": 0.935
|
||||||
|
},
|
||||||
|
"question": {
|
||||||
|
"abstained": false,
|
||||||
|
"byLanguage": {
|
||||||
|
"en": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.8445,
|
||||||
|
"falseNegative": 76,
|
||||||
|
"falsePositive": 12,
|
||||||
|
"precision": 0.9522,
|
||||||
|
"predictedPositive": 251,
|
||||||
|
"recall": 0.7587,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 1158,
|
||||||
|
"truePositive": 239,
|
||||||
|
"wilsonPrecisionLower95": 0.9183
|
||||||
|
},
|
||||||
|
"threshold": 0.725
|
||||||
|
},
|
||||||
|
"zh-Hans": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.8536,
|
||||||
|
"falseNegative": 76,
|
||||||
|
"falsePositive": 6,
|
||||||
|
"precision": 0.9755,
|
||||||
|
"predictedPositive": 245,
|
||||||
|
"recall": 0.7587,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 1164,
|
||||||
|
"truePositive": 239,
|
||||||
|
"wilsonPrecisionLower95": 0.9476
|
||||||
|
},
|
||||||
|
"threshold": 0.5
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.8729,
|
||||||
|
"falseNegative": 122,
|
||||||
|
"falsePositive": 26,
|
||||||
|
"precision": 0.9513,
|
||||||
|
"predictedPositive": 534,
|
||||||
|
"recall": 0.8063,
|
||||||
|
"total": 2970,
|
||||||
|
"trueNegative": 2314,
|
||||||
|
"truePositive": 508,
|
||||||
|
"wilsonPrecisionLower95": 0.9296
|
||||||
|
},
|
||||||
|
"threshold": 0.515
|
||||||
|
},
|
||||||
|
"replyableMessage": {
|
||||||
|
"abstained": false,
|
||||||
|
"byLanguage": {
|
||||||
|
"en": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.948,
|
||||||
|
"falseNegative": 52,
|
||||||
|
"falsePositive": 46,
|
||||||
|
"precision": 0.951,
|
||||||
|
"predictedPositive": 939,
|
||||||
|
"recall": 0.945,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 494,
|
||||||
|
"truePositive": 893,
|
||||||
|
"wilsonPrecisionLower95": 0.9353
|
||||||
|
},
|
||||||
|
"threshold": 0.7
|
||||||
|
},
|
||||||
|
"zh-Hans": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.9424,
|
||||||
|
"falseNegative": 62,
|
||||||
|
"falsePositive": 46,
|
||||||
|
"precision": 0.9505,
|
||||||
|
"predictedPositive": 929,
|
||||||
|
"recall": 0.9344,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 494,
|
||||||
|
"truePositive": 883,
|
||||||
|
"wilsonPrecisionLower95": 0.9346
|
||||||
|
},
|
||||||
|
"threshold": 0.695
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.9454,
|
||||||
|
"falseNegative": 115,
|
||||||
|
"falsePositive": 90,
|
||||||
|
"precision": 0.9517,
|
||||||
|
"predictedPositive": 1865,
|
||||||
|
"recall": 0.9392,
|
||||||
|
"total": 2970,
|
||||||
|
"trueNegative": 990,
|
||||||
|
"truePositive": 1775,
|
||||||
|
"wilsonPrecisionLower95": 0.9411
|
||||||
|
},
|
||||||
|
"threshold": 0.7
|
||||||
|
},
|
||||||
|
"scheduleNegotiation": {
|
||||||
|
"abstained": false,
|
||||||
|
"byLanguage": {
|
||||||
|
"en": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 1.0,
|
||||||
|
"falseNegative": 0,
|
||||||
|
"falsePositive": 0,
|
||||||
|
"precision": 1.0,
|
||||||
|
"predictedPositive": 45,
|
||||||
|
"recall": 1.0,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 1440,
|
||||||
|
"truePositive": 45,
|
||||||
|
"wilsonPrecisionLower95": 0.9213
|
||||||
|
},
|
||||||
|
"threshold": 0.66
|
||||||
|
},
|
||||||
|
"zh-Hans": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 1.0,
|
||||||
|
"falseNegative": 0,
|
||||||
|
"falsePositive": 0,
|
||||||
|
"precision": 1.0,
|
||||||
|
"predictedPositive": 45,
|
||||||
|
"recall": 1.0,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 1440,
|
||||||
|
"truePositive": 45,
|
||||||
|
"wilsonPrecisionLower95": 0.9213
|
||||||
|
},
|
||||||
|
"threshold": 0.55
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"metrics": {
|
||||||
|
"f1": 1.0,
|
||||||
|
"falseNegative": 0,
|
||||||
|
"falsePositive": 0,
|
||||||
|
"precision": 1.0,
|
||||||
|
"predictedPositive": 90,
|
||||||
|
"recall": 1.0,
|
||||||
|
"total": 2970,
|
||||||
|
"trueNegative": 2880,
|
||||||
|
"truePositive": 90,
|
||||||
|
"wilsonPrecisionLower95": 0.9591
|
||||||
|
},
|
||||||
|
"threshold": 0.66
|
||||||
|
},
|
||||||
|
"task": {
|
||||||
|
"abstained": false,
|
||||||
|
"byLanguage": {
|
||||||
|
"en": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.8593,
|
||||||
|
"falseNegative": 79,
|
||||||
|
"falsePositive": 13,
|
||||||
|
"precision": 0.9558,
|
||||||
|
"predictedPositive": 294,
|
||||||
|
"recall": 0.7806,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 1112,
|
||||||
|
"truePositive": 281,
|
||||||
|
"wilsonPrecisionLower95": 0.9258
|
||||||
|
},
|
||||||
|
"threshold": 0.98
|
||||||
|
},
|
||||||
|
"zh-Hans": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.8166,
|
||||||
|
"falseNegative": 104,
|
||||||
|
"falsePositive": 11,
|
||||||
|
"precision": 0.9588,
|
||||||
|
"predictedPositive": 267,
|
||||||
|
"recall": 0.7111,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 1114,
|
||||||
|
"truePositive": 256,
|
||||||
|
"wilsonPrecisionLower95": 0.9277
|
||||||
|
},
|
||||||
|
"threshold": 0.96
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.8434,
|
||||||
|
"falseNegative": 176,
|
||||||
|
"falsePositive": 26,
|
||||||
|
"precision": 0.9544,
|
||||||
|
"predictedPositive": 570,
|
||||||
|
"recall": 0.7556,
|
||||||
|
"total": 2970,
|
||||||
|
"trueNegative": 2224,
|
||||||
|
"truePositive": 544,
|
||||||
|
"wilsonPrecisionLower95": 0.934
|
||||||
|
},
|
||||||
|
"threshold": 0.965
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,481 @@
|
|||||||
|
{
|
||||||
|
"automaticPromotionAllowed": false,
|
||||||
|
"baseCorpus": "ModelTraining/ClipboardSemantics/clipboard_semantic_corpus.jsonl",
|
||||||
|
"configuration": {
|
||||||
|
"alpha": 2e-05,
|
||||||
|
"augmentation": "none",
|
||||||
|
"char_max": 5,
|
||||||
|
"char_min": 2,
|
||||||
|
"external_weight": 0.5700000000000001,
|
||||||
|
"hard_example_weight": 1.0,
|
||||||
|
"l1_ratio": 0.0,
|
||||||
|
"max_char_features": 45000,
|
||||||
|
"max_word_features": 22000,
|
||||||
|
"min_df": 2,
|
||||||
|
"pseudo_weight": 0.16499999999999998,
|
||||||
|
"round": 1,
|
||||||
|
"word_max": 2
|
||||||
|
},
|
||||||
|
"openCorpus": "ModelTraining/ClipboardSemantics/open-training-corpus.jsonl",
|
||||||
|
"reason": "Surrogate feature weights are not deployable NLModel assets and cannot bypass the existing macOS acceptance policy.",
|
||||||
|
"requiredCommands": [
|
||||||
|
"python3 Scripts/clipboard_semantics/generate_open_training_corpus.py",
|
||||||
|
"xcrun swift Scripts/clipboard_semantics/train_models.swift --algorithms maxEnt --corpus ModelTraining/ClipboardSemantics/combined-training-corpus.jsonl",
|
||||||
|
"xcrun swift Scripts/clipboard_semantics/evaluate_random_holdout.swift --corpus ModelTraining/ClipboardSemantics/random-holdout-corpus.jsonl"
|
||||||
|
],
|
||||||
|
"schemaVersion": 1,
|
||||||
|
"seed": 20260827,
|
||||||
|
"selectedRound": 1,
|
||||||
|
"thresholds": {
|
||||||
|
"blessing": {
|
||||||
|
"abstained": false,
|
||||||
|
"byLanguage": {
|
||||||
|
"en": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.9318,
|
||||||
|
"falseNegative": 4,
|
||||||
|
"falsePositive": 2,
|
||||||
|
"precision": 0.9535,
|
||||||
|
"predictedPositive": 43,
|
||||||
|
"recall": 0.9111,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 1438,
|
||||||
|
"truePositive": 41,
|
||||||
|
"wilsonPrecisionLower95": 0.8454
|
||||||
|
},
|
||||||
|
"threshold": 0.915
|
||||||
|
},
|
||||||
|
"zh-Hans": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.7568,
|
||||||
|
"falseNegative": 17,
|
||||||
|
"falsePositive": 1,
|
||||||
|
"precision": 0.9655,
|
||||||
|
"predictedPositive": 29,
|
||||||
|
"recall": 0.6222,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 1439,
|
||||||
|
"truePositive": 28,
|
||||||
|
"wilsonPrecisionLower95": 0.8282
|
||||||
|
},
|
||||||
|
"threshold": 0.9
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.8375,
|
||||||
|
"falseNegative": 23,
|
||||||
|
"falsePositive": 3,
|
||||||
|
"precision": 0.9571,
|
||||||
|
"predictedPositive": 70,
|
||||||
|
"recall": 0.7444,
|
||||||
|
"total": 2970,
|
||||||
|
"trueNegative": 2877,
|
||||||
|
"truePositive": 67,
|
||||||
|
"wilsonPrecisionLower95": 0.8814
|
||||||
|
},
|
||||||
|
"threshold": 0.915
|
||||||
|
},
|
||||||
|
"complaint": {
|
||||||
|
"abstained": false,
|
||||||
|
"byLanguage": {
|
||||||
|
"en": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 1.0,
|
||||||
|
"falseNegative": 0,
|
||||||
|
"falsePositive": 0,
|
||||||
|
"precision": 1.0,
|
||||||
|
"predictedPositive": 180,
|
||||||
|
"recall": 1.0,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 1305,
|
||||||
|
"truePositive": 180,
|
||||||
|
"wilsonPrecisionLower95": 0.9791
|
||||||
|
},
|
||||||
|
"threshold": 0.5
|
||||||
|
},
|
||||||
|
"zh-Hans": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.9526,
|
||||||
|
"falseNegative": 9,
|
||||||
|
"falsePositive": 8,
|
||||||
|
"precision": 0.9553,
|
||||||
|
"predictedPositive": 179,
|
||||||
|
"recall": 0.95,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 1297,
|
||||||
|
"truePositive": 171,
|
||||||
|
"wilsonPrecisionLower95": 0.9143
|
||||||
|
},
|
||||||
|
"threshold": 0.54
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.9764,
|
||||||
|
"falseNegative": 9,
|
||||||
|
"falsePositive": 8,
|
||||||
|
"precision": 0.9777,
|
||||||
|
"predictedPositive": 359,
|
||||||
|
"recall": 0.975,
|
||||||
|
"total": 2970,
|
||||||
|
"trueNegative": 2602,
|
||||||
|
"truePositive": 351,
|
||||||
|
"wilsonPrecisionLower95": 0.9567
|
||||||
|
},
|
||||||
|
"threshold": 0.54
|
||||||
|
},
|
||||||
|
"confirmationDecision": {
|
||||||
|
"abstained": false,
|
||||||
|
"byLanguage": {
|
||||||
|
"en": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.8848,
|
||||||
|
"falseNegative": 17,
|
||||||
|
"falsePositive": 2,
|
||||||
|
"precision": 0.9733,
|
||||||
|
"predictedPositive": 75,
|
||||||
|
"recall": 0.8111,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 1393,
|
||||||
|
"truePositive": 73,
|
||||||
|
"wilsonPrecisionLower95": 0.9079
|
||||||
|
},
|
||||||
|
"threshold": 0.5
|
||||||
|
},
|
||||||
|
"zh-Hans": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.7947,
|
||||||
|
"falseNegative": 30,
|
||||||
|
"falsePositive": 1,
|
||||||
|
"precision": 0.9836,
|
||||||
|
"predictedPositive": 61,
|
||||||
|
"recall": 0.6667,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 1394,
|
||||||
|
"truePositive": 60,
|
||||||
|
"wilsonPrecisionLower95": 0.9128
|
||||||
|
},
|
||||||
|
"threshold": 0.5
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.8418,
|
||||||
|
"falseNegative": 47,
|
||||||
|
"falsePositive": 3,
|
||||||
|
"precision": 0.9779,
|
||||||
|
"predictedPositive": 136,
|
||||||
|
"recall": 0.7389,
|
||||||
|
"total": 2970,
|
||||||
|
"trueNegative": 2787,
|
||||||
|
"truePositive": 133,
|
||||||
|
"wilsonPrecisionLower95": 0.9372
|
||||||
|
},
|
||||||
|
"threshold": 0.5
|
||||||
|
},
|
||||||
|
"followUpReminder": {
|
||||||
|
"abstained": false,
|
||||||
|
"byLanguage": {
|
||||||
|
"en": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.8554,
|
||||||
|
"falseNegative": 41,
|
||||||
|
"falsePositive": 6,
|
||||||
|
"precision": 0.9586,
|
||||||
|
"predictedPositive": 145,
|
||||||
|
"recall": 0.7722,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 1299,
|
||||||
|
"truePositive": 139,
|
||||||
|
"wilsonPrecisionLower95": 0.9127
|
||||||
|
},
|
||||||
|
"threshold": 0.83
|
||||||
|
},
|
||||||
|
"zh-Hans": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.8354,
|
||||||
|
"falseNegative": 48,
|
||||||
|
"falsePositive": 4,
|
||||||
|
"precision": 0.9706,
|
||||||
|
"predictedPositive": 136,
|
||||||
|
"recall": 0.7333,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 1301,
|
||||||
|
"truePositive": 132,
|
||||||
|
"wilsonPrecisionLower95": 0.9268
|
||||||
|
},
|
||||||
|
"threshold": 0.505
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.7994,
|
||||||
|
"falseNegative": 113,
|
||||||
|
"falsePositive": 11,
|
||||||
|
"precision": 0.9574,
|
||||||
|
"predictedPositive": 258,
|
||||||
|
"recall": 0.6861,
|
||||||
|
"total": 2970,
|
||||||
|
"trueNegative": 2599,
|
||||||
|
"truePositive": 247,
|
||||||
|
"wilsonPrecisionLower95": 0.9253
|
||||||
|
},
|
||||||
|
"threshold": 0.815
|
||||||
|
},
|
||||||
|
"invitation": {
|
||||||
|
"abstained": false,
|
||||||
|
"byLanguage": {
|
||||||
|
"en": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.8302,
|
||||||
|
"falseNegative": 24,
|
||||||
|
"falsePositive": 3,
|
||||||
|
"precision": 0.9565,
|
||||||
|
"predictedPositive": 69,
|
||||||
|
"recall": 0.7333,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 1392,
|
||||||
|
"truePositive": 66,
|
||||||
|
"wilsonPrecisionLower95": 0.8798
|
||||||
|
},
|
||||||
|
"threshold": 0.935
|
||||||
|
},
|
||||||
|
"zh-Hans": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.6765,
|
||||||
|
"falseNegative": 44,
|
||||||
|
"falsePositive": 0,
|
||||||
|
"precision": 1.0,
|
||||||
|
"predictedPositive": 46,
|
||||||
|
"recall": 0.5111,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 1395,
|
||||||
|
"truePositive": 46,
|
||||||
|
"wilsonPrecisionLower95": 0.9229
|
||||||
|
},
|
||||||
|
"threshold": 0.505
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.5476,
|
||||||
|
"falseNegative": 111,
|
||||||
|
"falsePositive": 3,
|
||||||
|
"precision": 0.9583,
|
||||||
|
"predictedPositive": 72,
|
||||||
|
"recall": 0.3833,
|
||||||
|
"total": 2970,
|
||||||
|
"trueNegative": 2787,
|
||||||
|
"truePositive": 69,
|
||||||
|
"wilsonPrecisionLower95": 0.8845
|
||||||
|
},
|
||||||
|
"threshold": 0.935
|
||||||
|
},
|
||||||
|
"question": {
|
||||||
|
"abstained": false,
|
||||||
|
"byLanguage": {
|
||||||
|
"en": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.8445,
|
||||||
|
"falseNegative": 76,
|
||||||
|
"falsePositive": 12,
|
||||||
|
"precision": 0.9522,
|
||||||
|
"predictedPositive": 251,
|
||||||
|
"recall": 0.7587,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 1158,
|
||||||
|
"truePositive": 239,
|
||||||
|
"wilsonPrecisionLower95": 0.9183
|
||||||
|
},
|
||||||
|
"threshold": 0.725
|
||||||
|
},
|
||||||
|
"zh-Hans": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.8536,
|
||||||
|
"falseNegative": 76,
|
||||||
|
"falsePositive": 6,
|
||||||
|
"precision": 0.9755,
|
||||||
|
"predictedPositive": 245,
|
||||||
|
"recall": 0.7587,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 1164,
|
||||||
|
"truePositive": 239,
|
||||||
|
"wilsonPrecisionLower95": 0.9476
|
||||||
|
},
|
||||||
|
"threshold": 0.5
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.8729,
|
||||||
|
"falseNegative": 122,
|
||||||
|
"falsePositive": 26,
|
||||||
|
"precision": 0.9513,
|
||||||
|
"predictedPositive": 534,
|
||||||
|
"recall": 0.8063,
|
||||||
|
"total": 2970,
|
||||||
|
"trueNegative": 2314,
|
||||||
|
"truePositive": 508,
|
||||||
|
"wilsonPrecisionLower95": 0.9296
|
||||||
|
},
|
||||||
|
"threshold": 0.515
|
||||||
|
},
|
||||||
|
"replyableMessage": {
|
||||||
|
"abstained": false,
|
||||||
|
"byLanguage": {
|
||||||
|
"en": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.948,
|
||||||
|
"falseNegative": 52,
|
||||||
|
"falsePositive": 46,
|
||||||
|
"precision": 0.951,
|
||||||
|
"predictedPositive": 939,
|
||||||
|
"recall": 0.945,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 494,
|
||||||
|
"truePositive": 893,
|
||||||
|
"wilsonPrecisionLower95": 0.9353
|
||||||
|
},
|
||||||
|
"threshold": 0.7
|
||||||
|
},
|
||||||
|
"zh-Hans": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.9424,
|
||||||
|
"falseNegative": 62,
|
||||||
|
"falsePositive": 46,
|
||||||
|
"precision": 0.9505,
|
||||||
|
"predictedPositive": 929,
|
||||||
|
"recall": 0.9344,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 494,
|
||||||
|
"truePositive": 883,
|
||||||
|
"wilsonPrecisionLower95": 0.9346
|
||||||
|
},
|
||||||
|
"threshold": 0.695
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.9454,
|
||||||
|
"falseNegative": 115,
|
||||||
|
"falsePositive": 90,
|
||||||
|
"precision": 0.9517,
|
||||||
|
"predictedPositive": 1865,
|
||||||
|
"recall": 0.9392,
|
||||||
|
"total": 2970,
|
||||||
|
"trueNegative": 990,
|
||||||
|
"truePositive": 1775,
|
||||||
|
"wilsonPrecisionLower95": 0.9411
|
||||||
|
},
|
||||||
|
"threshold": 0.7
|
||||||
|
},
|
||||||
|
"scheduleNegotiation": {
|
||||||
|
"abstained": false,
|
||||||
|
"byLanguage": {
|
||||||
|
"en": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 1.0,
|
||||||
|
"falseNegative": 0,
|
||||||
|
"falsePositive": 0,
|
||||||
|
"precision": 1.0,
|
||||||
|
"predictedPositive": 45,
|
||||||
|
"recall": 1.0,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 1440,
|
||||||
|
"truePositive": 45,
|
||||||
|
"wilsonPrecisionLower95": 0.9213
|
||||||
|
},
|
||||||
|
"threshold": 0.5
|
||||||
|
},
|
||||||
|
"zh-Hans": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.967,
|
||||||
|
"falseNegative": 1,
|
||||||
|
"falsePositive": 2,
|
||||||
|
"precision": 0.9565,
|
||||||
|
"predictedPositive": 46,
|
||||||
|
"recall": 0.9778,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 1438,
|
||||||
|
"truePositive": 44,
|
||||||
|
"wilsonPrecisionLower95": 0.8547
|
||||||
|
},
|
||||||
|
"threshold": 0.93
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.978,
|
||||||
|
"falseNegative": 1,
|
||||||
|
"falsePositive": 3,
|
||||||
|
"precision": 0.9674,
|
||||||
|
"predictedPositive": 92,
|
||||||
|
"recall": 0.9889,
|
||||||
|
"total": 2970,
|
||||||
|
"trueNegative": 2877,
|
||||||
|
"truePositive": 89,
|
||||||
|
"wilsonPrecisionLower95": 0.9085
|
||||||
|
},
|
||||||
|
"threshold": 0.91
|
||||||
|
},
|
||||||
|
"task": {
|
||||||
|
"abstained": false,
|
||||||
|
"byLanguage": {
|
||||||
|
"en": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.8403,
|
||||||
|
"falseNegative": 89,
|
||||||
|
"falsePositive": 14,
|
||||||
|
"precision": 0.9509,
|
||||||
|
"predictedPositive": 285,
|
||||||
|
"recall": 0.7528,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 1111,
|
||||||
|
"truePositive": 271,
|
||||||
|
"wilsonPrecisionLower95": 0.9192
|
||||||
|
},
|
||||||
|
"threshold": 0.835
|
||||||
|
},
|
||||||
|
"zh-Hans": {
|
||||||
|
"abstained": false,
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.8278,
|
||||||
|
"falseNegative": 98,
|
||||||
|
"falsePositive": 11,
|
||||||
|
"precision": 0.9597,
|
||||||
|
"predictedPositive": 273,
|
||||||
|
"recall": 0.7278,
|
||||||
|
"total": 1485,
|
||||||
|
"trueNegative": 1114,
|
||||||
|
"truePositive": 262,
|
||||||
|
"wilsonPrecisionLower95": 0.9293
|
||||||
|
},
|
||||||
|
"threshold": 0.745
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"metrics": {
|
||||||
|
"f1": 0.8637,
|
||||||
|
"falseNegative": 153,
|
||||||
|
"falsePositive": 26,
|
||||||
|
"precision": 0.9562,
|
||||||
|
"predictedPositive": 593,
|
||||||
|
"recall": 0.7875,
|
||||||
|
"total": 2970,
|
||||||
|
"trueNegative": 2224,
|
||||||
|
"truePositive": 567,
|
||||||
|
"wilsonPrecisionLower95": 0.9365
|
||||||
|
},
|
||||||
|
"threshold": 0.755
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,129 @@
|
|||||||
|
{
|
||||||
|
"schemaVersion": 1,
|
||||||
|
"completedResearchRounds": 60,
|
||||||
|
"requestedCoreRounds": 20,
|
||||||
|
"seed": 20260827,
|
||||||
|
"data": {
|
||||||
|
"baseTrainingRecords": 11500,
|
||||||
|
"reachableLicensedOpenRecords": 3845,
|
||||||
|
"holdoutOverlapCounts": {
|
||||||
|
"comprehensiveOnline": 0,
|
||||||
|
"random": 0,
|
||||||
|
"targetedRelease": 0
|
||||||
|
},
|
||||||
|
"unavailableSources": [
|
||||||
|
"MASSIVE",
|
||||||
|
"CLINC150",
|
||||||
|
"CFPB"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"phases": [
|
||||||
|
{
|
||||||
|
"name": "shared-configuration-search",
|
||||||
|
"rounds": 20,
|
||||||
|
"selectedRound": 1,
|
||||||
|
"golden": {
|
||||||
|
"macroPrecision": 0.9366,
|
||||||
|
"macroRecall": 0.6801,
|
||||||
|
"macroF1": 0.7711
|
||||||
|
},
|
||||||
|
"randomHoldout": {
|
||||||
|
"macroPrecision": 0.8569,
|
||||||
|
"macroRecall": 0.4603,
|
||||||
|
"macroF1": 0.5709
|
||||||
|
},
|
||||||
|
"researchOnlyComprehensive": {
|
||||||
|
"macroPrecision": 0.4398,
|
||||||
|
"macroRecall": 0.2161,
|
||||||
|
"macroF1": 0.2383
|
||||||
|
},
|
||||||
|
"targetedRelease": {
|
||||||
|
"macroPrecision": 0.5489,
|
||||||
|
"macroRecall": 0.3139,
|
||||||
|
"macroF1": 0.3899
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "per-intent-fine-tuning",
|
||||||
|
"rounds": 20,
|
||||||
|
"selectedRoundsByIntent": {
|
||||||
|
"blessing": 19,
|
||||||
|
"complaint": 1,
|
||||||
|
"confirmationDecision": 3,
|
||||||
|
"followUpReminder": 1,
|
||||||
|
"invitation": 1,
|
||||||
|
"question": 1,
|
||||||
|
"replyableMessage": 1,
|
||||||
|
"scheduleNegotiation": 7,
|
||||||
|
"task": 15
|
||||||
|
},
|
||||||
|
"golden": {
|
||||||
|
"macroPrecision": 0.9377,
|
||||||
|
"macroRecall": 0.6819,
|
||||||
|
"macroF1": 0.7666
|
||||||
|
},
|
||||||
|
"randomHoldout": {
|
||||||
|
"macroPrecision": 0.8701,
|
||||||
|
"macroRecall": 0.4705,
|
||||||
|
"macroF1": 0.5770
|
||||||
|
},
|
||||||
|
"researchOnlyComprehensive": {
|
||||||
|
"macroPrecision": 0.4292,
|
||||||
|
"macroRecall": 0.2250,
|
||||||
|
"macroF1": 0.2645
|
||||||
|
},
|
||||||
|
"targetedRelease": {
|
||||||
|
"macroPrecision": 0.5467,
|
||||||
|
"macroRecall": 0.3375,
|
||||||
|
"macroF1": 0.4095
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "explicit-evidence-fine-tuning",
|
||||||
|
"rounds": 20,
|
||||||
|
"selectedRound": 1,
|
||||||
|
"golden": {
|
||||||
|
"macroPrecision": 0.9877,
|
||||||
|
"macroRecall": 0.4354,
|
||||||
|
"macroF1": 0.5476
|
||||||
|
},
|
||||||
|
"randomHoldout": {
|
||||||
|
"macroPrecision": 0.7731,
|
||||||
|
"macroRecall": 0.3029,
|
||||||
|
"macroF1": 0.3905
|
||||||
|
},
|
||||||
|
"researchOnlyComprehensive": {
|
||||||
|
"macroPrecision": 0.6097,
|
||||||
|
"macroRecall": 0.1647,
|
||||||
|
"macroF1": 0.2325
|
||||||
|
},
|
||||||
|
"targetedRelease": {
|
||||||
|
"macroPrecision": 0.5489,
|
||||||
|
"macroRecall": 0.2537,
|
||||||
|
"macroF1": 0.3209
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"currentDeployedReference": {
|
||||||
|
"randomHoldout": {
|
||||||
|
"macroPrecision": 0.9832,
|
||||||
|
"macroRecall": 0.6557,
|
||||||
|
"macroF1": 0.7669
|
||||||
|
},
|
||||||
|
"researchOnlyComprehensive": {
|
||||||
|
"macroPrecision": 0.2948,
|
||||||
|
"macroRecall": 0.3320,
|
||||||
|
"macroF1": 0.2783
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"findings": [
|
||||||
|
"The first low-complexity round generalized better than later self-training rounds.",
|
||||||
|
"Three-round high-confidence pseudo-labeling amplified source and template bias instead of improving frozen regressions.",
|
||||||
|
"Per-intent configuration selection improved the research-only comprehensive F1 from 0.2383 to 0.2645 but remained below the deployed reference F1 of 0.2783.",
|
||||||
|
"Explicit evidence gates raised golden precision to 0.9877 and research-only precision to 0.6097, but recall collapsed and the release gate still failed.",
|
||||||
|
"No surrogate candidate is eligible to replace or augment the deployed Apple models."
|
||||||
|
],
|
||||||
|
"releaseGatePassed": false,
|
||||||
|
"deploymentDecision": "retain-current-deployed-models",
|
||||||
|
"platformLimitation": "Linux cannot train, compile, run, or benchmark the deployable Apple Create ML models. These rounds are surrogate research and produce a macOS replay specification, not app-ready model assets."
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -122,6 +122,48 @@ Without human-reviewed gold labels, these results measure agreement with the
|
|||||||
model committee, not production truth. A passing verifier may enter shadow
|
model committee, not production truth. A passing verifier may enter shadow
|
||||||
deployment, but cannot be described as having 95% real-user precision.
|
deployment, but cannot be described as having 95% real-user precision.
|
||||||
|
|
||||||
|
## Iterative weakly supervised research
|
||||||
|
|
||||||
|
The portable research harness runs a fixed 20-round matrix over bilingual
|
||||||
|
character/word n-grams, sparse logistic classifiers, class balancing,
|
||||||
|
deterministic label-preserving augmentation, hard-example weighting, and
|
||||||
|
three-round high-confidence self-training consensus:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python3 -m pip install -r \
|
||||||
|
Scripts/clipboard_semantics/requirements-research.txt
|
||||||
|
python3 Scripts/clipboard_semantics/generate_open_training_corpus.py \
|
||||||
|
--allow-unavailable-sources
|
||||||
|
python3 Scripts/clipboard_semantics/run_iterative_retraining.py
|
||||||
|
```
|
||||||
|
|
||||||
|
The harness fits thresholds only on the generated validation split. Synthetic
|
||||||
|
test, golden, random, targeted-release, and comprehensive online corpora never
|
||||||
|
enter training or threshold calibration. Exact normalized overlap with every
|
||||||
|
frozen holdout is a fatal error.
|
||||||
|
|
||||||
|
The 2026-08-27 study completed the requested 20 rounds and two additional
|
||||||
|
20-round fine-tuning phases after the first phase missed its release target:
|
||||||
|
|
||||||
|
- The shared-configuration phase selected round 1. Random-holdout macro F1 was
|
||||||
|
`0.5709`; research-only comprehensive macro F1 was `0.2383`.
|
||||||
|
- Per-intent selection improved those values to `0.5770` and `0.2645`.
|
||||||
|
- Explicit evidence gates raised golden macro precision to `0.9877` and
|
||||||
|
research-only precision to `0.6097`, but reduced recall too severely.
|
||||||
|
- The current deployed reference remains stronger: random-holdout macro F1
|
||||||
|
`0.7669` and research-only comprehensive macro F1 `0.2783`.
|
||||||
|
|
||||||
|
All three phases failed the release gate, so no model was promoted. Detailed
|
||||||
|
rounds, final evaluations, source breakdowns, and macOS replay instructions are
|
||||||
|
stored under `IterativeResearch/`.
|
||||||
|
|
||||||
|
This harness is deliberately a Linux surrogate. It cannot emit the
|
||||||
|
`NLModel`-compatible Create ML artifacts used by the keyboard extension.
|
||||||
|
Deployable training, Core ML compilation, simulator regression, latency, and
|
||||||
|
memory checks still require macOS with Xcode 26+. A surrogate result can
|
||||||
|
nominate a data/threshold policy for macOS replay, but cannot authorize
|
||||||
|
automatic deployment.
|
||||||
|
|
||||||
## Deployment decision
|
## Deployment decision
|
||||||
|
|
||||||
Only maxEnt models are trained and deployed because they are self-contained in
|
Only maxEnt models are trained and deployed because they are self-contained in
|
||||||
|
|||||||
@@ -1,9 +1,9 @@
|
|||||||
{
|
{
|
||||||
"baseCorpusRecords": 17692,
|
"baseCorpusRecords": 17692,
|
||||||
"combinedCorpusSHA256": "70d1547b41b08e4417cfa86ad93dd5637b557d174c6503575c44fbf152cc3ee5",
|
"combinedCorpusSHA256": "f111bc7425912213082e01038b52dd3d25fe21418d156f532da8f01f27fa16e4",
|
||||||
"combinedRecords": 23918,
|
"combinedRecords": 21537,
|
||||||
"excludedBaseOverlap": 6,
|
"excludedBaseOverlap": 6,
|
||||||
"excludedHoldoutOverlap": 151,
|
"excludedHoldoutOverlap": 72,
|
||||||
"excludedSources": [
|
"excludedSources": [
|
||||||
{
|
{
|
||||||
"dataset": "MIDAS",
|
"dataset": "MIDAS",
|
||||||
@@ -38,72 +38,11 @@
|
|||||||
"reason": "No clean official train split independent from the frozen holdout."
|
"reason": "No clean official train split independent from the frozen holdout."
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"openTrainingRecords": 6226,
|
"openTrainingRecords": 3845,
|
||||||
"policy": "Licensed official training splits only. Per-record knownLabels prevent unannotated intents from becoming false negatives. Exact normalized text overlap with every local *holdout-corpus.jsonl file is excluded.",
|
"policy": "Licensed official training splits only. Per-record knownLabels prevent unannotated intents from becoming false negatives. Exact normalized text overlap with every local *holdout-corpus.jsonl file is excluded.",
|
||||||
"schemaVersion": 1,
|
"schemaVersion": 1,
|
||||||
"seed": 20260827,
|
"seed": 20260827,
|
||||||
"sources": [
|
"sources": [
|
||||||
{
|
|
||||||
"dataset": "AmazonScience/MASSIVE en-US",
|
|
||||||
"knownLabelCounts": {
|
|
||||||
"followUpReminder": 7,
|
|
||||||
"question": 223,
|
|
||||||
"sentiment": 4,
|
|
||||||
"task": 143
|
|
||||||
},
|
|
||||||
"languages": {
|
|
||||||
"en": 498
|
|
||||||
},
|
|
||||||
"license": "CC-BY-4.0",
|
|
||||||
"records": 498,
|
|
||||||
"revision": "1.1",
|
|
||||||
"url": "https://huggingface.co/datasets/AmazonScience/massive"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"dataset": "AmazonScience/MASSIVE zh-CN",
|
|
||||||
"knownLabelCounts": {
|
|
||||||
"followUpReminder": 3,
|
|
||||||
"question": 212,
|
|
||||||
"sentiment": 6,
|
|
||||||
"task": 165
|
|
||||||
},
|
|
||||||
"languages": {
|
|
||||||
"zh-Hans": 487
|
|
||||||
},
|
|
||||||
"license": "CC-BY-4.0",
|
|
||||||
"records": 487,
|
|
||||||
"revision": "1.1",
|
|
||||||
"url": "https://huggingface.co/datasets/AmazonScience/massive"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"dataset": "CFPB via hpe-ai/customer-complaints",
|
|
||||||
"knownLabelCounts": {
|
|
||||||
"complaint": 898,
|
|
||||||
"sentiment": 898
|
|
||||||
},
|
|
||||||
"languages": {
|
|
||||||
"en": 898
|
|
||||||
},
|
|
||||||
"license": "CC0-1.0 source / Apache-2.0 mirror",
|
|
||||||
"records": 898,
|
|
||||||
"revision": "e5fec64e1f0688e47699b9cf8c26fe4ed350123a",
|
|
||||||
"url": "https://www.consumerfinance.gov/data-research/consumer-complaints/search/detail/4696537"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"dataset": "CLINC150 UCI",
|
|
||||||
"knownLabelCounts": {
|
|
||||||
"followUpReminder": 5,
|
|
||||||
"question": 308,
|
|
||||||
"task": 143
|
|
||||||
},
|
|
||||||
"languages": {
|
|
||||||
"en": 498
|
|
||||||
},
|
|
||||||
"license": "CC-BY-3.0",
|
|
||||||
"records": 498,
|
|
||||||
"revision": "UCI-570-2020-05-07",
|
|
||||||
"url": "https://archive.ics.uci.edu/dataset/570/clinc150"
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"dataset": "CrossWOZ",
|
"dataset": "CrossWOZ",
|
||||||
"knownLabelCounts": {
|
"knownLabelCounts": {
|
||||||
@@ -121,13 +60,13 @@
|
|||||||
{
|
{
|
||||||
"dataset": "Google Research GoEmotions",
|
"dataset": "Google Research GoEmotions",
|
||||||
"knownLabelCounts": {
|
"knownLabelCounts": {
|
||||||
"sentiment": 799
|
"sentiment": 798
|
||||||
},
|
},
|
||||||
"languages": {
|
"languages": {
|
||||||
"en": 1199
|
"en": 1198
|
||||||
},
|
},
|
||||||
"license": "Apache-2.0",
|
"license": "Apache-2.0",
|
||||||
"records": 1199,
|
"records": 1198,
|
||||||
"revision": "5d8f4ac97c873bde3a792ba4628f00bb9103d3e6",
|
"revision": "5d8f4ac97c873bde3a792ba4628f00bb9103d3e6",
|
||||||
"url": "https://github.com/google-research/google-research/blob/5d8f4ac97c873bde3a792ba4628f00bb9103d3e6/goemotions/data/train.tsv"
|
"url": "https://github.com/google-research/google-research/blob/5d8f4ac97c873bde3a792ba4628f00bb9103d3e6/goemotions/data/train.tsv"
|
||||||
},
|
},
|
||||||
@@ -164,17 +103,31 @@
|
|||||||
{
|
{
|
||||||
"dataset": "MultiDoGO",
|
"dataset": "MultiDoGO",
|
||||||
"knownLabelCounts": {
|
"knownLabelCounts": {
|
||||||
"confirmationDecision": 28,
|
"confirmationDecision": 27,
|
||||||
"question": 83,
|
"question": 83,
|
||||||
"task": 222
|
"task": 222
|
||||||
},
|
},
|
||||||
"languages": {
|
"languages": {
|
||||||
"en": 464
|
"en": 465
|
||||||
},
|
},
|
||||||
"license": "CDLA-Permissive-1.0",
|
"license": "CDLA-Permissive-1.0",
|
||||||
"records": 464,
|
"records": 465,
|
||||||
"revision": "baa30639c4b271f394b81443c842193407cdf26d",
|
"revision": "baa30639c4b271f394b81443c842193407cdf26d",
|
||||||
"url": "https://github.com/awslabs/multi-domain-goal-oriented-dialogues-dataset/blob/baa30639c4b271f394b81443c842193407cdf26d/data/paper_splits/splits_annotated_at_turn_level/insurance/train.tsv"
|
"url": "https://github.com/awslabs/multi-domain-goal-oriented-dialogues-dataset/blob/baa30639c4b271f394b81443c842193407cdf26d/data/paper_splits/splits_annotated_at_turn_level/software/train.tsv"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"unavailableSources": [
|
||||||
|
{
|
||||||
|
"dataset": "MASSIVE",
|
||||||
|
"reason": "URLError: <urlopen error [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1000)>"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"dataset": "CLINC150",
|
||||||
|
"reason": "URLError: <urlopen error [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1000)>"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"dataset": "CFPB",
|
||||||
|
"reason": "URLError: <urlopen error [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1000)>"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user