Cursor: Apply local changes for cloud agent

Scripts/clipboard_semantics/apply_deployment_policy.py

@@ -0,0 +1,158 @@
+#!/usr/bin/env python3
+"""Apply the reviewed deployment thresholds and pinned model selections."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import shutil
+from pathlib import Path
+
+
+DEFAULT_RESOURCE_DIRECTORY = Path(
+    "OSGKeyboardShared/Resources/ClipboardSemantics"
+)
+DEFAULT_BASELINE_DIRECTORY = Path(
+    "ModelTraining/ClipboardSemantics/baselines/2026-08-26-nine-model-v1"
+)
+
+# Core thresholds were selected on the 680-record development holdout and
+# checked against the golden gate. Task and complaint were later tightened on
+# focused development gates; the 20260830 release profile is acceptance-only.
+THRESHOLD_POLICY = {
+    "task": {
+        "global": 0.88,
+        "byLanguage": {"en": 0.88, "zh-Hans": 0.73},
+    },
+    "complaint": {
+        "global": 0.82,
+        "byLanguage": {"en": 0.84, "zh-Hans": 0.82},
+    },
+    "scheduleNegotiation": {
+        "global": 0.68,
+        "byLanguage": {"en": 0.68, "zh-Hans": 0.53},
+    },
+    "confirmationDecision": {
+        "global": 0.72,
+        "byLanguage": {"en": 0.72, "zh-Hans": 0.72},
+    },
+    "followUpReminder": {
+        "global": 0.73,
+        "byLanguage": {"en": 0.73, "zh-Hans": 0.73},
+    },
+    "blessing": {
+        "global": 0.69,
+        "byLanguage": {"en": 0.69, "zh-Hans": 0.77},
+    },
+}
+
+PINNED_MODELS = {
+    "scheduleNegotiation": "ScheduleNegotiationIntentClassifier.mlmodel",
+}
+
+
+def parse_arguments() -> argparse.Namespace:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--resource-directory",
+        type=Path,
+        default=DEFAULT_RESOURCE_DIRECTORY,
+    )
+    parser.add_argument(
+        "--baseline-directory",
+        type=Path,
+        default=DEFAULT_BASELINE_DIRECTORY,
+    )
+    parser.add_argument(
+        "--candidate-resource-directory",
+        type=Path,
+        default=None,
+    )
+    parser.add_argument(
+        "--promote-classifier",
+        action="append",
+        default=[],
+    )
+    return parser.parse_args()
+
+
+def main() -> None:
+    arguments = parse_arguments()
+    resource_directory: Path = arguments.resource_directory
+    baseline_directory: Path = arguments.baseline_directory
+    manifest_path = resource_directory / "clipboard-semantic-models.json"
+    manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+    classifiers = {
+        classifier["id"]: classifier
+        for classifier in manifest["classifiers"]
+    }
+    promoted_classifiers = arguments.promote_classifier
+
+    if promoted_classifiers:
+        candidate_directory: Path | None = arguments.candidate_resource_directory
+        if candidate_directory is None:
+            raise RuntimeError(
+                "--candidate-resource-directory is required when promoting classifiers"
+            )
+        candidate_manifest_path = (
+            candidate_directory / "clipboard-semantic-models.json"
+        )
+        candidate_manifest = json.loads(
+            candidate_manifest_path.read_text(encoding="utf-8")
+        )
+        candidates = {
+            classifier["id"]: classifier
+            for classifier in candidate_manifest["classifiers"]
+        }
+        for classifier_id in promoted_classifiers:
+            if classifier_id not in candidates:
+                raise RuntimeError(
+                    f"Cannot promote missing classifier: {classifier_id}"
+                )
+            candidate = dict(candidates[classifier_id])
+            candidate["trainedAt"] = candidate_manifest["generatedAt"]
+            candidate["trainingCorpusRecordCount"] = candidate_manifest[
+                "corpusRecordCount"
+            ]
+            model_file = candidate["modelFile"]
+            shutil.copy2(
+                candidate_directory / model_file,
+                resource_directory / model_file,
+            )
+            if classifier_id in classifiers:
+                classifiers[classifier_id].clear()
+                classifiers[classifier_id].update(candidate)
+            else:
+                manifest["classifiers"].append(candidate)
+                classifiers[classifier_id] = candidate
+
+    missing = sorted(set(THRESHOLD_POLICY) - set(classifiers))
+    if missing:
+        raise RuntimeError(f"Manifest is missing classifiers: {missing}")
+
+    for classifier_id, policy in THRESHOLD_POLICY.items():
+        classifier = classifiers[classifier_id]
+        classifier["acceptedForAutomaticRouting"] = True
+        classifier["confidenceThreshold"] = policy["global"]
+        classifier["confidenceThresholdsByLanguage"] = policy["byLanguage"]
+
+    for classifier_id, model_file in PINNED_MODELS.items():
+        source = baseline_directory / model_file
+        destination = resource_directory / classifiers[classifier_id]["modelFile"]
+        if not source.is_file():
+            raise RuntimeError(f"Pinned model is missing: {source}")
+        shutil.copy2(source, destination)
+
+    manifest_path.write_text(
+        json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
+        encoding="utf-8",
+    )
+    print(
+        "DEPLOYMENT_POLICY_DONE "
+        f"thresholds={len(THRESHOLD_POLICY)} pinnedModels={len(PINNED_MODELS)} "
+        f"promotedModels={len(promoted_classifiers)}"
+    )
+
+
+if __name__ == "__main__":
+    main()

Scripts/clipboard_semantics/apply_verifier_deployment_policy.py

@@ -0,0 +1,158 @@
+#!/usr/bin/env python3
+"""Promote only acceptance-gated verifiers into privacy-safe shadow mode."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import shutil
+from datetime import datetime, timezone
+from pathlib import Path
+
+
+DEFAULT_CANDIDATE_DIRECTORY = Path(
+    "ModelTraining/ClipboardSemantics/VerifierCandidates"
+)
+DEFAULT_RESOURCE_DIRECTORY = Path(
+    "OSGKeyboardShared/Resources/ClipboardSemantics"
+)
+DEFAULT_REPORT = Path(
+    "ModelTraining/ClipboardSemantics/verifier-deployment-report.json"
+)
+
+
+def sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as stream:
+        for chunk in iter(lambda: stream.read(128 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def parse_arguments() -> argparse.Namespace:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--candidate-directory",
+        type=Path,
+        default=DEFAULT_CANDIDATE_DIRECTORY,
+    )
+    parser.add_argument(
+        "--resource-directory",
+        type=Path,
+        default=DEFAULT_RESOURCE_DIRECTORY,
+    )
+    parser.add_argument("--report", type=Path, default=DEFAULT_REPORT)
+    parser.add_argument(
+        "--apply",
+        action="store_true",
+        help="Copy eligible models and update the deployed manifest.",
+    )
+    return parser.parse_args()
+
+
+def main() -> None:
+    arguments = parse_arguments()
+    candidate_manifest_path = (
+        arguments.candidate_directory / "clipboard-semantic-models.json"
+    )
+    deployed_manifest_path = (
+        arguments.resource_directory / "clipboard-semantic-models.json"
+    )
+    candidate_manifest = json.loads(
+        candidate_manifest_path.read_text(encoding="utf-8")
+    )
+    deployed_manifest = json.loads(
+        deployed_manifest_path.read_text(encoding="utf-8")
+    )
+
+    candidate_verifiers = candidate_manifest.get("verifiers") or []
+    eligible = [
+        verifier
+        for verifier in candidate_verifiers
+        if verifier.get("acceptedForAutomaticRouting") is True
+        and verifier.get("deploymentMode") == "automatic"
+    ]
+    entries = []
+    for verifier in candidate_verifiers:
+        model_path = arguments.candidate_directory / verifier["modelFile"]
+        if not model_path.is_file():
+            raise FileNotFoundError(model_path)
+        entries.append(
+            {
+                "id": verifier["id"],
+                "eligible": verifier in eligible,
+                "candidateDeploymentMode": verifier.get("deploymentMode"),
+                "candidateSHA256": sha256(model_path),
+                "decision": (
+                    "promote-to-shadow"
+                    if verifier in eligible
+                    else "retain-current-models"
+                ),
+            }
+        )
+
+    applied = False
+    if arguments.apply and eligible:
+        promoted_verifiers = []
+        for verifier in eligible:
+            source = arguments.candidate_directory / verifier["modelFile"]
+            destination = arguments.resource_directory / verifier["modelFile"]
+            shutil.copy2(source, destination)
+            shadow = dict(verifier)
+            shadow["acceptedForAutomaticRouting"] = False
+            shadow["deploymentMode"] = "shadow"
+            shadow["candidatePassedAcceptance"] = True
+            promoted_verifiers.append(shadow)
+
+        # Always start from the deployed manifest so binary classifiers cannot
+        # be replaced by an experimental candidate as a side effect.
+        deployed_manifest["schemaVersion"] = 3
+        deployed_manifest["verifiers"] = promoted_verifiers
+        deployed_manifest["verifierGeneratedAt"] = candidate_manifest.get(
+            "verifierGeneratedAt"
+        )
+        deployed_manifest["verifierLabelPolicy"] = candidate_manifest.get(
+            "verifierLabelPolicy"
+        )
+        deployed_manifest_path.write_text(
+            json.dumps(
+                deployed_manifest,
+                ensure_ascii=False,
+                indent=2,
+                sort_keys=True,
+            )
+            + "\n",
+            encoding="utf-8",
+        )
+        applied = True
+
+    report = {
+        "schemaVersion": 1,
+        "generatedAt": datetime.now(timezone.utc).isoformat(),
+        "applyRequested": arguments.apply,
+        "applied": applied,
+        "eligibleVerifierCount": len(eligible),
+        "deployedClassifierCountPreserved": len(
+            deployed_manifest.get("classifiers") or []
+        ),
+        "policy": (
+            "Only acceptance-gated automatic candidates may be copied, and "
+            "their first deployed mode is forced to shadow. If none pass, the "
+            "deployed manifest and current binary models remain untouched."
+        ),
+        "verifiers": entries,
+    }
+    arguments.report.parent.mkdir(parents=True, exist_ok=True)
+    arguments.report.write_text(
+        json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
+        encoding="utf-8",
+    )
+    print(
+        "VERIFIER_DEPLOYMENT_POLICY "
+        f"eligible={len(eligible)} applied={str(applied).lower()}"
+    )
+
+
+if __name__ == "__main__":
+    main()

Scripts/clipboard_semantics/evaluate_random_holdout.swift

@@ -0,0 +1,915 @@
+#!/usr/bin/env swift
+
+import Foundation
+import NaturalLanguage
+
+private struct HoldoutRecord: Decodable {
+    let id: String
+    let text: String
+    let language: String
+    let split: String?
+    let family: String
+    let task: Bool
+    let question: Bool
+    let invitation: Bool
+    let complaint: Bool
+    let scheduleNegotiation: Bool
+    let confirmationDecision: Bool
+    let followUpReminder: Bool
+    let blessing: Bool?
+    let sentiment: String
+    let replyable: Bool
+    let sourceDataset: String?
+
+    func isPositive(for classifierID: String) -> Bool {
+        switch classifierID {
+        case "task": task
+        case "question": question
+        case "invitation": invitation
+        case "complaint": complaint
+        case "scheduleNegotiation": scheduleNegotiation
+        case "confirmationDecision": confirmationDecision
+        case "followUpReminder": followUpReminder
+        case "blessing": blessing ?? false
+        case "replyableMessage": replyable
+        default: false
+        }
+    }
+
+    func expectedVerifierLabel(for verifierID: String) -> String {
+        if verifierID == "action" {
+            if task && complaint {
+                return "both"
+            }
+            if task {
+                return "taskOnly"
+            }
+            if complaint {
+                return "complaintOnly"
+            }
+            if question {
+                return "questionRequest"
+            }
+            return "neither"
+        }
+        let coordinationLabels = [
+            invitation ? "invitation" : nil,
+            scheduleNegotiation ? "scheduleNegotiation" : nil,
+            confirmationDecision ? "confirmationDecision" : nil,
+            followUpReminder ? "followUpReminder" : nil
+        ].compactMap { $0 }
+        return coordinationLabels.count == 1 ? coordinationLabels[0] : "neither"
+    }
+}
+
+private struct TrainingRecord: Decodable {
+    let text: String
+}
+
+private struct Manifest: Decodable {
+    let schemaVersion: Int
+    let classifiers: [ManifestClassifier]
+    let verifiers: [ManifestVerifier]?
+}
+
+private struct ManifestClassifier: Decodable {
+    let id: String
+    let modelFile: String
+    let positiveLabel: String?
+    let confidenceThreshold: Double?
+    let confidenceThresholdsByLanguage: [String: Double]?
+    let acceptedForAutomaticRouting: Bool
+}
+
+private struct ManifestVerifier: Decodable {
+    let id: String
+    let modelFile: String
+    let confidenceThreshold: Double
+    let confidenceThresholdsByLanguage: [String: Double]?
+    let minimumMargin: Double
+    let minimumMarginsByLanguage: [String: Double]?
+    let acceptedForAutomaticRouting: Bool
+    let deploymentMode: String
+}
+
+private struct BinaryMetrics: Encodable {
+    let total: Int
+    let truePositive: Int
+    let trueNegative: Int
+    let falsePositive: Int
+    let falseNegative: Int
+    let accuracy: Double
+    let precision: Double
+    let recall: Double
+    let f1: Double
+}
+
+private struct ErrorExample: Encodable {
+    let id: String
+    let language: String
+    let family: String
+    let sourceDataset: String?
+    let text: String
+    let confidence: Double
+}
+
+private struct BinaryEvaluation: Encodable {
+    let id: String
+    let metrics: BinaryMetrics
+    let metricsByLanguage: [String: BinaryMetrics]
+    let metricsBySource: [String: BinaryMetrics]
+    let thresholdAt90Precision: ThresholdRecommendation?
+    let thresholdAt95Precision: ThresholdRecommendation?
+    let thresholdsAt90PrecisionByLanguage: [String: ThresholdRecommendation]
+    let thresholdsAt95PrecisionByLanguage: [String: ThresholdRecommendation]
+    let falsePositiveExamples: [ErrorExample]
+    let falseNegativeExamples: [ErrorExample]
+}
+
+private struct ThresholdRecommendation: Encodable {
+    let threshold: Double
+    let metrics: BinaryMetrics
+}
+
+private struct SentimentMetrics: Encodable {
+    let total: Int
+    let correct: Int
+    let unknown: Int
+    let accuracy: Double
+    let unknownRate: Double
+    let macroF1: Double
+    let perLabelF1: [String: Double]
+}
+
+private struct AggregateMetrics: Encodable {
+    let accuracy: Double
+    let precision: Double
+    let recall: Double
+    let f1: Double
+}
+
+private struct VerifierRoutingMetrics: Encodable {
+    let total: Int
+    let expectedSpecialized: Int
+    let stageACandidates: Int
+    let routed: Int
+    let correctRouted: Int
+    let falseRouted: Int
+    let stageARecall: Double
+    let stageBExactAccuracy: Double
+    let finalPrecision: Double
+    let finalPrecisionWilsonLower95: Double
+    let finalRecall: Double
+}
+
+private struct VerifierEvaluation: Encodable {
+    let id: String
+    let deploymentMode: String
+    let acceptedForAutomaticRouting: Bool
+    let metrics: VerifierRoutingMetrics
+    let metricsByLanguage: [String: VerifierRoutingMetrics]
+    let metricsBySource: [String: VerifierRoutingMetrics]
+    let leaveOneSourceOut: [String: VerifierRoutingMetrics]
+    let coldLoadMilliseconds: Double
+    let warmMedianMilliseconds: Double
+    let warmP95Milliseconds: Double
+}
+
+private struct Report: Encodable {
+    let generatedAt: String
+    let seed: Int
+    let corpusRecordCount: Int
+    let familyCount: Int
+    let languageCounts: [String: Int]
+    let exactTrainingOverlapCount: Int
+    let manifestSchemaVersion: Int
+    let binaryMacro: AggregateMetrics
+    let binaryMacroByLanguage: [String: AggregateMetrics]
+    let classifiers: [BinaryEvaluation]
+    let verifierLayers: [VerifierEvaluation]
+    let sentiment: SentimentMetrics
+    let sentimentBySource: [String: SentimentMetrics]
+}
+
+private struct BinaryObservation {
+    let record: HoldoutRecord
+    let expected: Bool
+    let predicted: Bool
+    let confidence: Double
+    let isSuppressed: Bool
+}
+
+private struct VerifierObservation {
+    let record: HoldoutRecord
+    let expectedLabel: String
+    let isStageACandidate: Bool
+    let predictedLabel: String
+    let confidence: Double
+    let margin: Double
+    let isRouted: Bool
+    let latencyMilliseconds: Double
+}
+
+private let fileManager = FileManager.default
+private let root = URL(fileURLWithPath: fileManager.currentDirectoryPath)
+private func argumentValue(after flag: String) -> String? {
+    guard let index = CommandLine.arguments.firstIndex(of: flag),
+          CommandLine.arguments.indices.contains(index + 1) else {
+        return nil
+    }
+    return CommandLine.arguments[index + 1]
+}
+
+private let corpusURL = argumentValue(after: "--corpus").map {
+    URL(fileURLWithPath: $0, relativeTo: root).standardizedFileURL
+} ?? root.appendingPathComponent(
+    "ModelTraining/ClipboardSemantics/random-holdout-corpus.jsonl"
+)
+private let trainingCorpusURL = root.appendingPathComponent(
+    "ModelTraining/ClipboardSemantics/clipboard_semantic_corpus.jsonl"
+)
+private let manifestURL = argumentValue(after: "--manifest").map {
+    URL(fileURLWithPath: $0, relativeTo: root).standardizedFileURL
+} ?? root.appendingPathComponent(
+    "OSGKeyboardShared/Resources/ClipboardSemantics/clipboard-semantic-models.json"
+)
+private let modelDirectory = argumentValue(after: "--models").map {
+    URL(fileURLWithPath: $0, relativeTo: root).standardizedFileURL
+} ?? manifestURL.deletingLastPathComponent()
+private let reportURL = argumentValue(after: "--report").map {
+    URL(fileURLWithPath: $0, relativeTo: root).standardizedFileURL
+} ?? root.appendingPathComponent(
+    "ModelTraining/ClipboardSemantics/random-holdout-report.json"
+)
+private let sentimentMinimumConfidence = 0.65
+private let sentimentMinimumMargin = 0.15
+private let holdoutSeed = argumentValue(after: "--seed").flatMap(Int.init) ?? 20260826
+private let includesRejectedModels = CommandLine.arguments.contains(
+    "--include-rejected-models"
+)
+private let requestedSplit = argumentValue(after: "--split")
+
+private func rounded(_ value: Double) -> Double {
+    guard value.isFinite else { return 0 }
+    return (value * 10_000).rounded() / 10_000
+}
+
+private func milliseconds(_ duration: Duration) -> Double {
+    Double(duration.components.seconds) * 1_000
+        + Double(duration.components.attoseconds) / 1_000_000_000_000_000
+}
+
+private func decodeJSONLines<T: Decodable>(_ type: T.Type, from url: URL) throws -> [T] {
+    let content = try String(contentsOf: url, encoding: .utf8)
+    let decoder = JSONDecoder()
+    return try content.split(separator: "\n").map {
+        try decoder.decode(type, from: Data($0.utf8))
+    }
+}
+
+private func normalized(_ text: String) -> String {
+    text
+        .folding(options: [.caseInsensitive, .diacriticInsensitive], locale: nil)
+        .split(whereSeparator: \.isWhitespace)
+        .joined(separator: " ")
+}
+
+private func compileModel(sourceURL: URL, outputDirectory: URL) throws -> URL {
+    let process = Process()
+    let outputPipe = Pipe()
+    process.executableURL = URL(fileURLWithPath: "/usr/bin/xcrun")
+    process.arguments = [
+        "coremlcompiler",
+        "compile",
+        sourceURL.path,
+        outputDirectory.path
+    ]
+    process.standardOutput = outputPipe
+    process.standardError = outputPipe
+    try process.run()
+    process.waitUntilExit()
+    guard process.terminationStatus == 0 else {
+        let data = outputPipe.fileHandleForReading.readDataToEndOfFile()
+        let output = String(bytes: data, encoding: .utf8) ?? ""
+        throw NSError(
+            domain: "RandomHoldoutEvaluation",
+            code: Int(process.terminationStatus),
+            userInfo: [NSLocalizedDescriptionKey: output]
+        )
+    }
+    let resourceName = sourceURL.deletingPathExtension().lastPathComponent
+    return outputDirectory.appendingPathComponent("\(resourceName).mlmodelc")
+}
+
+private func binaryMetrics(_ observations: [BinaryObservation]) -> BinaryMetrics {
+    var truePositive = 0
+    var trueNegative = 0
+    var falsePositive = 0
+    var falseNegative = 0
+    for observation in observations {
+        switch (observation.expected, observation.predicted) {
+        case (true, true): truePositive += 1
+        case (false, false): trueNegative += 1
+        case (false, true): falsePositive += 1
+        case (true, false): falseNegative += 1
+        }
+    }
+    let total = observations.count
+    let precision = truePositive + falsePositive > 0
+        ? Double(truePositive) / Double(truePositive + falsePositive)
+        : 0
+    let recall = truePositive + falseNegative > 0
+        ? Double(truePositive) / Double(truePositive + falseNegative)
+        : 0
+    let accuracy = total > 0
+        ? Double(truePositive + trueNegative) / Double(total)
+        : 0
+    let f1 = precision + recall > 0
+        ? 2 * precision * recall / (precision + recall)
+        : 0
+    return BinaryMetrics(
+        total: total,
+        truePositive: truePositive,
+        trueNegative: trueNegative,
+        falsePositive: falsePositive,
+        falseNegative: falseNegative,
+        accuracy: rounded(accuracy),
+        precision: rounded(precision),
+        recall: rounded(recall),
+        f1: rounded(f1)
+    )
+}
+
+private func thresholdRecommendation(
+    observations: [BinaryObservation],
+    minimumPrecision: Double
+) -> ThresholdRecommendation? {
+    let candidates = stride(from: 0.05, through: 0.99, by: 0.01).compactMap { threshold
+        -> ThresholdRecommendation? in
+        let adjusted = observations.map {
+            BinaryObservation(
+                record: $0.record,
+                expected: $0.expected,
+                predicted: !$0.isSuppressed && $0.confidence >= threshold,
+                confidence: $0.confidence,
+                isSuppressed: $0.isSuppressed
+            )
+        }
+        let metrics = binaryMetrics(adjusted)
+        guard metrics.truePositive > 0, metrics.precision >= minimumPrecision else {
+            return nil
+        }
+        return ThresholdRecommendation(
+            threshold: rounded(threshold),
+            metrics: metrics
+        )
+    }
+    return candidates.max {
+        if $0.metrics.recall != $1.metrics.recall {
+            return $0.metrics.recall < $1.metrics.recall
+        }
+        if $0.metrics.precision != $1.metrics.precision {
+            return $0.metrics.precision < $1.metrics.precision
+        }
+        return $0.threshold > $1.threshold
+    }
+}
+
+private func shouldSuppressTask(
+    text: String,
+    complaintConfidence: Double
+) -> Bool {
+    guard complaintConfidence >= 0.60 else { return false }
+    let normalized = text.lowercased()
+    let explicitTaskMarkers = [
+        "请", "麻烦", "能否", "可以请你", "由你", "交给你", "需要你",
+        "你负责", "下一步", "行动项", "please ", "can you", "could you",
+        "would you", "assigned to you", "you are responsible", "we need you",
+        "would like you", "counting on you", "take ownership", "your task",
+        "next action", "complete the", "finish the", "send it to",
+        "deliver it to"
+    ]
+    return !explicitTaskMarkers.contains { normalized.contains($0) }
+}
+
+private func hasExplicitBlessingMarker(in text: String) -> Bool {
+    let normalized = text.lowercased()
+    let quotedOrMetaContexts = [
+        "祝福模板", "祝福语模板", "文章引用", "搜索词", "系统正在检查",
+        "文档里收录", "贺卡名单", "收集祝福", "greeting template",
+        "message template", "the article quotes", "search phrase",
+        "system is checking", "document contains", "card list",
+        "quotes the phrase", "如何描述生日快乐", "怎么说生日快乐",
+        "如何写生日祝福", "how would you describe a happy birthday",
+        "how do you say happy birthday", "what does happy birthday mean",
+        "宁愿你", "祝你倒闭", "祝你立马倒闭", "祝你去死", "祝你倒霉",
+        "祝你失败", "祝你完蛋"
+    ]
+    guard !quotedOrMetaContexts.contains(where: { normalized.contains($0) }) else {
+        return false
+    }
+    let markers = [
+        "生日快乐", "新年快乐", "春节快乐", "节日快乐", "圣诞快乐",
+        "中秋快乐", "恭喜", "预祝", "祝你", "祝您", "祝大家", "祝他", "祝她",
+        "愿你", "愿您", "happy birthday", "happy new year",
+        "merry christmas", "happy holidays", "congratulations",
+        "congrats", "best wishes", "good luck", "wishing you",
+        "wish you", "wish him", "wish her", "wish them", "let us wish",
+        "let's wish", "we wish", "may you"
+    ]
+    return markers.contains { normalized.contains($0) }
+}
+
+private func aggregate(_ metrics: [BinaryMetrics]) -> AggregateMetrics {
+    let divisor = Double(max(metrics.count, 1))
+    return AggregateMetrics(
+        accuracy: rounded(metrics.reduce(0) { $0 + $1.accuracy } / divisor),
+        precision: rounded(metrics.reduce(0) { $0 + $1.precision } / divisor),
+        recall: rounded(metrics.reduce(0) { $0 + $1.recall } / divisor),
+        f1: rounded(metrics.reduce(0) { $0 + $1.f1 } / divisor)
+    )
+}
+
+private func errorExamples(
+    from observations: [BinaryObservation],
+    expected: Bool,
+    predicted: Bool
+) -> [ErrorExample] {
+    observations
+        .filter { $0.expected == expected && $0.predicted == predicted }
+        .sorted { $0.confidence > $1.confidence }
+        .prefix(5)
+        .map {
+            ErrorExample(
+                id: $0.record.id,
+                language: $0.record.language,
+                family: $0.record.family,
+                sourceDataset: $0.record.sourceDataset,
+                text: $0.record.text,
+                confidence: rounded($0.confidence)
+            )
+        }
+}
+
+private func wilsonLowerBound(successes: Int, total: Int) -> Double {
+    guard total > 0 else { return 0 }
+    let z = 1.959_963_984_540_054
+    let proportion = Double(successes) / Double(total)
+    let denominator = 1 + z * z / Double(total)
+    let center = proportion + z * z / (2 * Double(total))
+    let adjustment = z * sqrt(
+        (
+            proportion * (1 - proportion)
+                + z * z / (4 * Double(total))
+        ) / Double(total)
+    )
+    return rounded((center - adjustment) / denominator)
+}
+
+private func verifierMetrics(
+    _ observations: [VerifierObservation]
+) -> VerifierRoutingMetrics {
+    let expectedSpecialized = observations.filter {
+        $0.expectedLabel != "neither"
+    }.count
+    let stageACandidates = observations.filter(\.isStageACandidate).count
+    let stageATruePositives = observations.filter {
+        $0.isStageACandidate && $0.expectedLabel != "neither"
+    }.count
+    let stageBObservations = observations.filter(\.isStageACandidate)
+    let stageBCorrect = stageBObservations.filter {
+        $0.predictedLabel == $0.expectedLabel
+    }.count
+    let routed = observations.filter(\.isRouted)
+    let correctRouted = routed.filter {
+        $0.predictedLabel == $0.expectedLabel
+    }.count
+    let stageARecall = expectedSpecialized > 0
+        ? Double(stageATruePositives) / Double(expectedSpecialized)
+        : 0
+    let stageBAccuracy = stageBObservations.isEmpty
+        ? 0
+        : Double(stageBCorrect) / Double(stageBObservations.count)
+    let precision = routed.isEmpty
+        ? 0
+        : Double(correctRouted) / Double(routed.count)
+    let recall = expectedSpecialized > 0
+        ? Double(correctRouted) / Double(expectedSpecialized)
+        : 0
+    return VerifierRoutingMetrics(
+        total: observations.count,
+        expectedSpecialized: expectedSpecialized,
+        stageACandidates: stageACandidates,
+        routed: routed.count,
+        correctRouted: correctRouted,
+        falseRouted: routed.count - correctRouted,
+        stageARecall: rounded(stageARecall),
+        stageBExactAccuracy: rounded(stageBAccuracy),
+        finalPrecision: rounded(precision),
+        finalPrecisionWilsonLower95: wilsonLowerBound(
+            successes: correctRouted,
+            total: routed.count
+        ),
+        finalRecall: rounded(recall)
+    )
+}
+
+private func percentile(_ values: [Double], proportion: Double) -> Double {
+    guard !values.isEmpty else { return 0 }
+    let sorted = values.sorted()
+    let index = min(
+        Int((Double(sorted.count - 1) * proportion).rounded()),
+        sorted.count - 1
+    )
+    return rounded(sorted[index])
+}
+
+private func candidateClassifierIDs(for verifierID: String) -> [String] {
+    if verifierID == "action" {
+        return ["task", "question", "complaint"]
+    }
+    return [
+        "invitation",
+        "scheduleNegotiation",
+        "confirmationDecision",
+        "followUpReminder"
+    ]
+}
+
+private func isStageACandidate(
+    record: HoldoutRecord,
+    verifierID: String,
+    models: [String: NLModel],
+    configurations: [String: ManifestClassifier]
+) -> Bool {
+    candidateClassifierIDs(for: verifierID).contains { classifierID in
+        guard let model = models[classifierID],
+              let configuration = configurations[classifierID],
+              let positiveLabel = configuration.positiveLabel else {
+            return false
+        }
+        let confidence = model.predictedLabelHypotheses(
+            for: record.text,
+            maximumCount: 2
+        )[positiveLabel] ?? 0
+        let configuredThreshold = configuration
+            .confidenceThresholdsByLanguage?[record.language]
+            ?? configuration.confidenceThreshold
+            ?? 1
+        return confidence >= min(configuredThreshold, 0.50)
+    }
+}
+
+private func sentimentMetrics(
+    records: [HoldoutRecord],
+    model: NLModel
+) -> SentimentMetrics {
+    let labels = ["negative", "neutral", "positive"]
+    var confusion: [String: [String: Int]] = [:]
+    var correct = 0
+    var unknown = 0
+
+    for record in records {
+        let ranked = model.predictedLabelHypotheses(for: record.text, maximumCount: 3)
+            .sorted { $0.value > $1.value }
+        let winner = ranked.first
+        let runnerUp = ranked.dropFirst().first?.value ?? 0
+        let prediction: String
+        if let winner,
+           winner.value >= sentimentMinimumConfidence,
+           winner.value - runnerUp >= sentimentMinimumMargin {
+            prediction = winner.key
+        } else {
+            prediction = "unknown"
+            unknown += 1
+        }
+        confusion[record.sentiment, default: [:]][prediction, default: 0] += 1
+        if prediction == record.sentiment {
+            correct += 1
+        }
+    }
+
+    var perLabelF1: [String: Double] = [:]
+    for label in labels {
+        let truePositive = confusion[label]?[label] ?? 0
+        let falseNegative = (confusion[label] ?? [:])
+            .filter { $0.key != label }
+            .reduce(0) { $0 + $1.value }
+        let falsePositive = labels
+            .filter { $0 != label }
+            .reduce(0) { $0 + (confusion[$1]?[label] ?? 0) }
+        let precision = truePositive + falsePositive > 0
+            ? Double(truePositive) / Double(truePositive + falsePositive)
+            : 0
+        let recall = truePositive + falseNegative > 0
+            ? Double(truePositive) / Double(truePositive + falseNegative)
+            : 0
+        perLabelF1[label] = rounded(
+            precision + recall > 0
+                ? 2 * precision * recall / (precision + recall)
+                : 0
+        )
+    }
+    let macroF1 = perLabelF1.values.reduce(0, +) / Double(labels.count)
+    return SentimentMetrics(
+        total: records.count,
+        correct: correct,
+        unknown: unknown,
+        accuracy: rounded(Double(correct) / Double(max(records.count, 1))),
+        unknownRate: rounded(Double(unknown) / Double(max(records.count, 1))),
+        macroF1: rounded(macroF1),
+        perLabelF1: perLabelF1
+    )
+}
+
+private func writeJSON<T: Encodable>(_ value: T, to url: URL) throws {
+    let encoder = JSONEncoder()
+    encoder.outputFormatting = [.prettyPrinted, .sortedKeys, .withoutEscapingSlashes]
+    try encoder.encode(value).write(to: url, options: .atomic)
+}
+
+private func main() throws {
+    let decodedRecords = try decodeJSONLines(HoldoutRecord.self, from: corpusURL)
+    let records = requestedSplit.map { split in
+        decodedRecords.filter { $0.split == split }
+    } ?? decodedRecords
+    let trainingRecords = try decodeJSONLines(TrainingRecord.self, from: trainingCorpusURL)
+    let manifest = try JSONDecoder().decode(
+        Manifest.self,
+        from: Data(contentsOf: manifestURL)
+    )
+    let trainingTexts = Set(trainingRecords.map { normalized($0.text) })
+    let exactOverlapCount = records.filter { trainingTexts.contains(normalized($0.text)) }.count
+
+    let temporaryDirectory = fileManager.temporaryDirectory.appendingPathComponent(
+        "osg-random-holdout-\(UUID().uuidString)",
+        isDirectory: true
+    )
+    try fileManager.createDirectory(
+        at: temporaryDirectory,
+        withIntermediateDirectories: true
+    )
+    defer { try? fileManager.removeItem(at: temporaryDirectory) }
+
+    var models: [String: NLModel] = [:]
+    let classifierConfigurations = Dictionary(
+        uniqueKeysWithValues: manifest.classifiers.map { ($0.id, $0) }
+    )
+    for configuration in manifest.classifiers {
+        let sourceURL = modelDirectory.appendingPathComponent(configuration.modelFile)
+        let compiledURL = try compileModel(
+            sourceURL: sourceURL,
+            outputDirectory: temporaryDirectory
+        )
+        let model = try NLModel(contentsOf: compiledURL)
+        models[configuration.id] = model
+    }
+    var verifierModels: [String: NLModel] = [:]
+    var verifierLoadMilliseconds: [String: Double] = [:]
+    for configuration in manifest.verifiers ?? [] {
+        let sourceURL = modelDirectory.appendingPathComponent(configuration.modelFile)
+        let compiledURL = try compileModel(
+            sourceURL: sourceURL,
+            outputDirectory: temporaryDirectory
+        )
+        let startedAt = ContinuousClock.now
+        verifierModels[configuration.id] = try NLModel(contentsOf: compiledURL)
+        verifierLoadMilliseconds[configuration.id] = milliseconds(
+            startedAt.duration(to: .now)
+        )
+    }
+
+    var binaryEvaluations: [BinaryEvaluation] = []
+    let sentimentResult = models["sentiment"].map {
+        sentimentMetrics(records: records, model: $0)
+    }
+    for configuration in manifest.classifiers {
+        if configuration.id == "sentiment" {
+            continue
+        }
+        guard let model = models[configuration.id] else { continue }
+        guard let positiveLabel = configuration.positiveLabel else {
+            continue
+        }
+        let observations = records.map { record in
+            let confidence = model.predictedLabelHypotheses(
+                for: record.text,
+                maximumCount: 2
+            )[positiveLabel] ?? 0
+            let threshold = configuration.confidenceThresholdsByLanguage?[record.language]
+                ?? configuration.confidenceThreshold
+                ?? 1
+            let complaintConfidence: Double
+            if configuration.id == "task", let complaintModel = models["complaint"] {
+                complaintConfidence = complaintModel.predictedLabelHypotheses(
+                    for: record.text,
+                    maximumCount: 2
+                )["complaint"] ?? 0
+            } else {
+                complaintConfidence = 0
+            }
+            let hasBlessingMarker = configuration.id == "blessing"
+                && hasExplicitBlessingMarker(in: record.text)
+            let effectiveConfidence = hasBlessingMarker ? 1 : confidence
+            let isTaskSuppressed = configuration.id == "task"
+                && shouldSuppressTask(
+                    text: record.text,
+                    complaintConfidence: complaintConfidence
+                )
+            let isBlessingSuppressed = configuration.id == "blessing"
+                && !hasBlessingMarker
+            let isSuppressed = isTaskSuppressed || isBlessingSuppressed
+            return BinaryObservation(
+                record: record,
+                expected: record.isPositive(for: configuration.id),
+                predicted: (configuration.acceptedForAutomaticRouting || includesRejectedModels)
+                    && !isSuppressed
+                    && effectiveConfidence >= threshold,
+                confidence: effectiveConfidence,
+                isSuppressed: isSuppressed
+            )
+        }
+        let byLanguage = Dictionary(grouping: observations) { $0.record.language }
+        let metricsByLanguage = byLanguage.mapValues(binaryMetrics)
+        let metricsBySource = Dictionary(grouping: observations) {
+            $0.record.sourceDataset ?? $0.record.family
+        }.mapValues(binaryMetrics)
+        binaryEvaluations.append(
+            BinaryEvaluation(
+                id: configuration.id,
+                metrics: binaryMetrics(observations),
+                metricsByLanguage: metricsByLanguage,
+                metricsBySource: metricsBySource,
+                thresholdAt90Precision: thresholdRecommendation(
+                    observations: observations,
+                    minimumPrecision: 0.90
+                ),
+                thresholdAt95Precision: thresholdRecommendation(
+                    observations: observations,
+                    minimumPrecision: 0.95
+                ),
+                thresholdsAt90PrecisionByLanguage: byLanguage.compactMapValues {
+                    thresholdRecommendation(observations: $0, minimumPrecision: 0.90)
+                },
+                thresholdsAt95PrecisionByLanguage: byLanguage.compactMapValues {
+                    thresholdRecommendation(observations: $0, minimumPrecision: 0.95)
+                },
+                falsePositiveExamples: errorExamples(
+                    from: observations,
+                    expected: false,
+                    predicted: true
+                ),
+                falseNegativeExamples: errorExamples(
+                    from: observations,
+                    expected: true,
+                    predicted: false
+                )
+            )
+        )
+    }
+
+    let verifierEvaluations = (manifest.verifiers ?? []).compactMap { configuration
+        -> VerifierEvaluation? in
+        guard let model = verifierModels[configuration.id] else { return nil }
+        let observations = records.map { record -> VerifierObservation in
+            let candidate = isStageACandidate(
+                record: record,
+                verifierID: configuration.id,
+                models: models,
+                configurations: classifierConfigurations
+            )
+            guard candidate else {
+                return VerifierObservation(
+                    record: record,
+                    expectedLabel: record.expectedVerifierLabel(
+                        for: configuration.id
+                    ),
+                    isStageACandidate: false,
+                    predictedLabel: "neither",
+                    confidence: 0,
+                    margin: 0,
+                    isRouted: false,
+                    latencyMilliseconds: 0
+                )
+            }
+            let startedAt = ContinuousClock.now
+            let ranked = model.predictedLabelHypotheses(
+                for: record.text,
+                maximumCount: 2
+            ).sorted { $0.value > $1.value }
+            let elapsed = startedAt.duration(to: .now)
+            let winner = ranked.first ?? (key: "neither", value: 0)
+            let margin = winner.value - (ranked.dropFirst().first?.value ?? 0)
+            let threshold = configuration
+                .confidenceThresholdsByLanguage?[record.language]
+                ?? configuration.confidenceThreshold
+            let minimumMargin = configuration
+                .minimumMarginsByLanguage?[record.language]
+                ?? configuration.minimumMargin
+            return VerifierObservation(
+                record: record,
+                expectedLabel: record.expectedVerifierLabel(
+                    for: configuration.id
+                ),
+                isStageACandidate: true,
+                predictedLabel: winner.key,
+                confidence: winner.value,
+                margin: margin,
+                isRouted: winner.key != "neither"
+                    && winner.value >= threshold
+                    && margin >= minimumMargin,
+                latencyMilliseconds: milliseconds(
+                    elapsed
+                )
+            )
+        }
+        let byLanguage = Dictionary(grouping: observations) {
+            $0.record.language
+        }.mapValues(verifierMetrics)
+        let bySourceGroups = Dictionary(grouping: observations) {
+            $0.record.sourceDataset ?? $0.record.family
+        }
+        let bySource = bySourceGroups.mapValues(verifierMetrics)
+        let leaveOneSourceOut = Dictionary(uniqueKeysWithValues: bySourceGroups.keys.map { source
+            in
+            (
+                source,
+                verifierMetrics(
+                    observations.filter {
+                        ($0.record.sourceDataset ?? $0.record.family) != source
+                    }
+                )
+            )
+        })
+        let latencies = observations.filter(\.isStageACandidate)
+            .map(\.latencyMilliseconds)
+        return VerifierEvaluation(
+            id: configuration.id,
+            deploymentMode: configuration.deploymentMode,
+            acceptedForAutomaticRouting:
+                configuration.acceptedForAutomaticRouting,
+            metrics: verifierMetrics(observations),
+            metricsByLanguage: byLanguage,
+            metricsBySource: bySource,
+            leaveOneSourceOut: leaveOneSourceOut,
+            coldLoadMilliseconds: rounded(
+                verifierLoadMilliseconds[configuration.id] ?? 0
+            ),
+            warmMedianMilliseconds: percentile(latencies, proportion: 0.50),
+            warmP95Milliseconds: percentile(latencies, proportion: 0.95)
+        )
+    }
+
+    guard let sentimentResult else {
+        throw NSError(
+            domain: "RandomHoldoutEvaluation",
+            code: 2,
+            userInfo: [NSLocalizedDescriptionKey: "Sentiment model was not evaluated"]
+        )
+    }
+    let languages = Set(records.map(\.language)).sorted()
+    let macroByLanguage = Dictionary(uniqueKeysWithValues: languages.map { language in
+        let metrics = binaryEvaluations.compactMap {
+            $0.metricsByLanguage[language]
+        }
+        return (language, aggregate(metrics))
+    })
+    let sentimentModel = models["sentiment"]!
+    let sentimentBySource = Dictionary(grouping: records) {
+        $0.sourceDataset ?? $0.family
+    }.mapValues {
+        sentimentMetrics(records: $0, model: sentimentModel)
+    }
+    let report = Report(
+        generatedAt: ISO8601DateFormatter().string(from: Date()),
+        seed: holdoutSeed,
+        corpusRecordCount: records.count,
+        familyCount: Set(records.map(\.family)).count,
+        languageCounts: Dictionary(grouping: records, by: \.language).mapValues(\.count),
+        exactTrainingOverlapCount: exactOverlapCount,
+        manifestSchemaVersion: manifest.schemaVersion,
+        binaryMacro: aggregate(binaryEvaluations.map(\.metrics)),
+        binaryMacroByLanguage: macroByLanguage,
+        classifiers: binaryEvaluations,
+        verifierLayers: verifierEvaluations,
+        sentiment: sentimentResult,
+        sentimentBySource: sentimentBySource
+    )
+    try writeJSON(report, to: reportURL)
+    print(
+        "RANDOM_HOLDOUT_EVAL_DONE records=\(records.count) "
+            + "overlap=\(exactOverlapCount) "
+            + "macroPrecision=\(report.binaryMacro.precision) "
+            + "macroRecall=\(report.binaryMacro.recall) "
+            + "macroF1=\(report.binaryMacro.f1) "
+            + "verifiers=\(report.verifierLayers.count) "
+            + "sentimentMacroF1=\(report.sentiment.macroF1)"
+    )
+}
+
+do {
+    try main()
+} catch {
+    fputs("RANDOM_HOLDOUT_EVAL_FAILED \(error)\n", stderr)
+    exit(1)
+}

Scripts/clipboard_semantics/generate_consensus_labels.py

@@ -0,0 +1,658 @@
+#!/usr/bin/env python3
+"""Prepare public-text labeling queues and build deterministic consensus silver data."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import math
+import random
+import re
+import unicodedata
+from collections import Counter, defaultdict
+from pathlib import Path
+
+
+SEED = 20260827
+INTENT_LABELS = (
+    "task",
+    "question",
+    "invitation",
+    "complaint",
+    "scheduleNegotiation",
+    "confirmationDecision",
+    "followUpReminder",
+    "blessing",
+    "replyableMessage",
+)
+SPECIAL_FLAGS = ("ambiguous", "quotedOrMeta")
+ACTION_LABELS = (
+    "taskOnly",
+    "complaintOnly",
+    "both",
+    "questionRequest",
+    "neither",
+)
+COORDINATION_LABELS = (
+    "invitation",
+    "scheduleNegotiation",
+    "confirmationDecision",
+    "followUpReminder",
+    "neither",
+)
+PROMPT_VERSION = "clipboard-consensus-v1"
+DEFAULT_DIRECTORY = Path("ModelTraining/ClipboardSemantics/Consensus")
+DEFAULT_INPUT = Path("ModelTraining/ClipboardSemantics/open-training-corpus.jsonl")
+
+
+def normalized_text(value: str) -> str:
+    return " ".join(
+        unicodedata.normalize("NFKC", value)
+        .replace("\u0000", " ")
+        .split()
+    ).strip()
+
+
+def stable_hash(value: str) -> int:
+    return int.from_bytes(hashlib.sha256(value.encode()).digest()[:8], "big")
+
+
+def read_json_lines(path: Path) -> list[dict]:
+    return [
+        json.loads(line)
+        for line in path.read_text(encoding="utf-8").splitlines()
+        if line.strip()
+    ]
+
+
+def write_json_lines(path: Path, records: list[dict]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    serialized = "\n".join(
+        json.dumps(record, ensure_ascii=False, sort_keys=True)
+        for record in records
+    )
+    path.write_text(serialized + ("\n" if serialized else ""), encoding="utf-8")
+
+
+def queue_priority(
+    record: dict,
+    hard_negative_texts: set[str],
+) -> tuple[int, int]:
+    known_labels = set(record.get("knownLabels") or [])
+    positive_count = sum(bool(record.get(label)) for label in INTENT_LABELS)
+    confusion_priority = int(
+        bool(
+            known_labels
+            & {
+                "task",
+                "question",
+                "complaint",
+                "invitation",
+                "scheduleNegotiation",
+                "confirmationDecision",
+                "followUpReminder",
+            }
+        )
+    )
+    hard_negative_priority = int(
+        normalized_text(record["text"]).casefold() in hard_negative_texts
+    )
+    return (
+        hard_negative_priority * 10 + confusion_priority + positive_count,
+        stable_hash(record["id"]),
+    )
+
+
+def stratified_queue(
+    records: list[dict],
+    count: int,
+    seed: int,
+    hard_negative_texts: set[str],
+) -> list[dict]:
+    grouped: dict[tuple[str, str], list[dict]] = defaultdict(list)
+    for record in records:
+        grouped[
+            (
+                record.get("sourceDataset", "unknown"),
+                record.get("language", "unknown"),
+            )
+        ].append(record)
+    for key in grouped:
+        grouped[key].sort(
+            key=lambda record: queue_priority(record, hard_negative_texts),
+            reverse=True,
+        )
+
+    keys = sorted(grouped)
+    rng = random.Random(seed)
+    rng.shuffle(keys)
+    offsets = {key: 0 for key in keys}
+    selected: list[dict] = []
+    while len(selected) < count:
+        added = False
+        for key in keys:
+            offset = offsets[key]
+            values = grouped[key]
+            if offset >= len(values):
+                continue
+            selected.append(values[offset])
+            offsets[key] += 1
+            added = True
+            if len(selected) == count:
+                break
+        if not added:
+            break
+    return sorted(selected, key=lambda record: record["id"])
+
+
+def labeling_instructions() -> str:
+    return """# Clipboard semantic consensus labeling
+
+Prompt version: clipboard-consensus-v1
+
+Label every JSONL queue record independently. Use only the text itself; do not
+infer missing conversational context. Return one JSON object per input record:
+
+{"id":"same id","labels":["task"],"ambiguous":false,"quotedOrMeta":false,"confidence":0.98}
+
+Allowed labels:
+- task: another person is explicitly asked/assigned to perform an action.
+- question: a genuine information question, including request-shaped questions.
+- invitation: invitation to join an event or social activity.
+- complaint: present dissatisfaction, malfunction, bad service, or unresolved problem.
+- scheduleNegotiation: proposing, changing, or choosing between times; a fixed time is not enough.
+- confirmationDecision: explicit approval, rejection, or selection of an option.
+- followUpReminder: request to remind, check back, or follow up later/after a trigger.
+- blessing: a genuine birthday, holiday, congratulations, or good-wish message.
+- replyableMessage: a direct conversational message that naturally invites a response.
+
+Rules:
+- Multi-label is allowed. "Could you send it?" is task + question + replyableMessage.
+- Set quotedOrMeta when intent-like words are quoted, documented, searched, or discussed.
+- Set ambiguous when the text cannot be labeled without missing context.
+- Empty labels mean none of the product intents.
+- Negative sentiment alone is not complaint. Fixed appointments are not schedule negotiation.
+- Do not expose reasoning or add fields. Confidence must be between 0 and 1.
+"""
+
+
+def prepare(arguments: argparse.Namespace) -> None:
+    records = read_json_lines(arguments.input)
+    requested_hard_negatives: set[str] = set()
+    for path in arguments.hard_negative_report:
+        report = json.loads(path.read_text(encoding="utf-8"))
+        for classifier in report.get("classifiers") or []:
+            for example in classifier.get("falsePositiveExamples") or []:
+                requested_hard_negatives.add(
+                    normalized_text(example["text"]).casefold()
+                )
+    license_safe_texts = {
+        normalized_text(record["text"]).casefold() for record in records
+    }
+    matched_hard_negatives = requested_hard_negatives & license_safe_texts
+    queue = stratified_queue(
+        records,
+        arguments.count,
+        arguments.seed,
+        matched_hard_negatives,
+    )
+    queue_records = [
+        {
+            "id": record["id"],
+            "text": record["text"],
+            "language": record["language"],
+            "sourceDataset": record.get("sourceDataset"),
+            "sourceLicense": record.get("sourceLicense"),
+            "sourceURL": record.get("sourceURL"),
+            "sourceRevision": record.get("sourceRevision"),
+            "knownLabels": record.get("knownLabels", []),
+            "sourceLabels": {
+                label: bool(record.get(label))
+                for label in INTENT_LABELS
+                if label in set(record.get("knownLabels") or [])
+            },
+        }
+        for record in queue
+    ]
+    write_json_lines(arguments.queue, queue_records)
+    arguments.instructions.parent.mkdir(parents=True, exist_ok=True)
+    instructions = labeling_instructions()
+    arguments.instructions.write_text(instructions, encoding="utf-8")
+    manifest = {
+        "schemaVersion": 1,
+        "promptVersion": PROMPT_VERSION,
+        "promptSHA256": hashlib.sha256(instructions.encode()).hexdigest(),
+        "seed": arguments.seed,
+        "input": str(arguments.input),
+        "queue": str(arguments.queue),
+        "recordCount": len(queue_records),
+        "requestedHardNegativeCount": len(requested_hard_negatives),
+        "licenseSafeMatchedHardNegativeCount": len(matched_hard_negatives),
+        "sourceCounts": dict(
+            sorted(
+                Counter(
+                    record.get("sourceDataset") or "unknown"
+                    for record in queue_records
+                ).items()
+            )
+        ),
+        "languageCounts": dict(
+            sorted(Counter(record["language"] for record in queue_records).items())
+        ),
+    }
+    arguments.prepare_report.write_text(
+        json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
+        encoding="utf-8",
+    )
+    print(
+        f"CONSENSUS_QUEUE records={len(queue_records)} "
+        f"promptSHA256={manifest['promptSHA256']}"
+    )
+
+
+def parse_labeler_argument(value: str) -> tuple[str, Path]:
+    name, separator, raw_path = value.partition("=")
+    if not separator or not name or not raw_path:
+        raise argparse.ArgumentTypeError("Expected LABELER=PATH")
+    return name, Path(raw_path)
+
+
+def validate_labeler_record(record: dict, expected_ids: set[str]) -> dict:
+    record_id = record.get("id")
+    if record_id not in expected_ids:
+        raise ValueError(f"Unexpected labeler record id: {record_id}")
+    labels = record.get("labels")
+    if not isinstance(labels, list) or any(label not in INTENT_LABELS for label in labels):
+        raise ValueError(f"Unsupported labels for {record_id}: {labels}")
+    confidence = record.get("confidence")
+    if not isinstance(confidence, (int, float)) or not 0 <= confidence <= 1:
+        raise ValueError(f"Invalid confidence for {record_id}: {confidence}")
+    for flag in SPECIAL_FLAGS:
+        if not isinstance(record.get(flag), bool):
+            raise ValueError(f"Missing boolean {flag} for {record_id}")
+    return {
+        "id": record_id,
+        "labels": sorted(set(labels)),
+        "ambiguous": record["ambiguous"],
+        "quotedOrMeta": record["quotedOrMeta"],
+        "confidence": round(float(confidence), 4),
+    }
+
+
+def action_label(labels: set[str]) -> str:
+    if "task" in labels and "complaint" in labels:
+        return "both"
+    if "task" in labels:
+        return "taskOnly"
+    if "complaint" in labels:
+        return "complaintOnly"
+    if "question" in labels:
+        return "questionRequest"
+    return "neither"
+
+
+def coordination_label(labels: set[str]) -> str | None:
+    matches = [
+        label
+        for label in COORDINATION_LABELS
+        if label != "neither" and label in labels
+    ]
+    if len(matches) > 1:
+        return None
+    return matches[0] if matches else "neither"
+
+
+def cluster_signature(text: str) -> str:
+    normalized = unicodedata.normalize("NFKC", text).casefold()
+    normalized = re.sub(r"\d+", "<n>", normalized)
+    normalized = re.sub(r"[^\w\u4e00-\u9fff<>]+", " ", normalized)
+    tokens = normalized.split()
+    # Prefix and suffix retain intent-bearing wording while grouping slot variants.
+    skeleton = tokens[:10] + (["|"] + tokens[-6:] if len(tokens) > 16 else [])
+    return " ".join(skeleton)
+
+
+def split_for(record: dict) -> str:
+    bucket = stable_hash(cluster_signature(record["text"])) % 100
+    if bucket < 70:
+        return "silverTrain"
+    if bucket < 85:
+        return "silverCalibration"
+    return "silverAcceptance"
+
+
+def fleiss_kappa(
+    labeler_records: list[dict[str, dict]],
+    queue_ids: list[str],
+) -> float:
+    if len(labeler_records) < 2 or not queue_ids:
+        return 0
+    category_counts = [0, 0]
+    agreement_total = 0.0
+    item_count = 0
+    rater_count = len(labeler_records)
+    for record_id in queue_ids:
+        for label in INTENT_LABELS:
+            yes_count = sum(
+                label in set(labeler[record_id]["labels"])
+                for labeler in labeler_records
+            )
+            no_count = rater_count - yes_count
+            category_counts[0] += no_count
+            category_counts[1] += yes_count
+            agreement_total += (
+                no_count * (no_count - 1) + yes_count * (yes_count - 1)
+            ) / (rater_count * (rater_count - 1))
+            item_count += 1
+    observed = agreement_total / item_count
+    total_votes = sum(category_counts)
+    expected = sum((count / total_votes) ** 2 for count in category_counts)
+    if math.isclose(expected, 1):
+        return 1
+    return round((observed - expected) / (1 - expected), 4)
+
+
+def merge(arguments: argparse.Namespace) -> None:
+    queue = read_json_lines(arguments.queue)
+    queue_by_id = {record["id"]: record for record in queue}
+    if len(queue_by_id) != len(queue):
+        raise ValueError("Consensus queue contains duplicate ids")
+    expected_ids = set(queue_by_id)
+    if len(arguments.labeler) < 3:
+        raise ValueError("Consensus requires at least three independent labelers")
+
+    labelers: list[tuple[str, dict[str, dict]]] = []
+    for name, path in arguments.labeler:
+        values = [
+            validate_labeler_record(record, expected_ids)
+            for record in read_json_lines(path)
+        ]
+        by_id = {record["id"]: record for record in values}
+        missing = expected_ids.difference(by_id)
+        if missing:
+            raise ValueError(f"Labeler {name} is missing {len(missing)} records")
+        if len(by_id) != len(values):
+            raise ValueError(f"Labeler {name} contains duplicate ids")
+        labelers.append((name, by_id))
+
+    accepted: list[dict] = []
+    conflicts: list[dict] = []
+    for record_id in sorted(expected_ids):
+        queue_record = queue_by_id[record_id]
+        votes = Counter(
+            label
+            for _, records in labelers
+            for label in records[record_id]["labels"]
+        )
+        source_labels = {
+            label
+            for label, value in (queue_record.get("sourceLabels") or {}).items()
+            if value
+        }
+        consensus_labels = {
+            label
+            for label, count in votes.items()
+            if count == len(labelers) or count >= 2 and label in source_labels
+        }
+        ambiguous_votes = sum(
+            records[record_id]["ambiguous"] for _, records in labelers
+        )
+        quoted_votes = sum(
+            records[record_id]["quotedOrMeta"] for _, records in labelers
+        )
+        coordination = coordination_label(consensus_labels)
+        full_agreement = all(
+            set(records[record_id]["labels"]) == set(
+                labelers[0][1][record_id]["labels"]
+            )
+            and records[record_id]["ambiguous"]
+            == labelers[0][1][record_id]["ambiguous"]
+            and records[record_id]["quotedOrMeta"]
+            == labelers[0][1][record_id]["quotedOrMeta"]
+            for _, records in labelers[1:]
+        )
+        rejected_reason = None
+        if ambiguous_votes >= 2:
+            rejected_reason = "ambiguous-majority"
+        elif quoted_votes >= 2 and consensus_labels:
+            rejected_reason = "quoted-or-meta-intent"
+        elif coordination is None:
+            rejected_reason = "multiple-coordination-labels"
+        elif not consensus_labels and not full_agreement:
+            rejected_reason = "no-supported-consensus"
+
+        audit = {
+            "id": record_id,
+            "labelerVotes": dict(sorted(votes.items())),
+            "labelerConfidences": {
+                name: records[record_id]["confidence"]
+                for name, records in labelers
+            },
+            "labelerResponseHashes": {
+                name: hashlib.sha256(
+                    json.dumps(
+                        records[record_id],
+                        ensure_ascii=False,
+                        sort_keys=True,
+                        separators=(",", ":"),
+                    ).encode()
+                ).hexdigest()
+                for name, records in labelers
+            },
+            "sourceLabels": sorted(source_labels),
+            "ambiguousVotes": ambiguous_votes,
+            "quotedOrMetaVotes": quoted_votes,
+            "fullAgreement": full_agreement,
+        }
+        if rejected_reason:
+            conflicts.append(
+                {
+                    **queue_record,
+                    **audit,
+                    "rejectedReason": rejected_reason,
+                }
+            )
+            continue
+
+        split = split_for(queue_record)
+        labels = set() if quoted_votes >= 2 else consensus_labels
+        silver = {
+            "id": f"consensus-{record_id}",
+            "sourceRecordID": record_id,
+            "text": queue_record["text"],
+            "language": queue_record["language"],
+            "family": "consensus_public",
+            "split": split,
+            **{label: label in labels for label in INTENT_LABELS},
+            "sentiment": "neutral",
+            "replyable": "replyableMessage" in labels,
+            "actionVerifierLabel": action_label(labels),
+            "coordinationVerifierLabel": coordination_label(labels) or "neither",
+            "quotedOrMeta": quoted_votes >= 2,
+            "consensusAgreement": round(
+                max(
+                    [votes.get(label, 0) for label in INTENT_LABELS] + [
+                        len(labelers) if not labels and full_agreement else 0
+                    ]
+                )
+                / len(labelers),
+                4,
+            ),
+            "promptVersion": PROMPT_VERSION,
+            "sourceDataset": queue_record.get("sourceDataset"),
+            "sourceLicense": queue_record.get("sourceLicense"),
+            "sourceURL": queue_record.get("sourceURL"),
+            "sourceRevision": queue_record.get("sourceRevision"),
+            **audit,
+        }
+        accepted.append(silver)
+
+    write_json_lines(arguments.consensus, accepted)
+    write_json_lines(arguments.conflicts, conflicts)
+    for split, path in (
+        ("silverTrain", arguments.train),
+        ("silverCalibration", arguments.calibration),
+        ("silverAcceptance", arguments.acceptance),
+    ):
+        write_json_lines(
+            path,
+            [record for record in accepted if record["split"] == split],
+        )
+
+    labeler_maps = [records for _, records in labelers]
+    text_split_map: dict[str, set[str]] = defaultdict(set)
+    cluster_split_map: dict[str, set[str]] = defaultdict(set)
+    for record in accepted:
+        text_split_map[normalized_text(record["text"]).casefold()].add(record["split"])
+        cluster_split_map[cluster_signature(record["text"])].add(record["split"])
+    exact_overlap_count = sum(len(splits) > 1 for splits in text_split_map.values())
+    cluster_overlap_count = sum(
+        len(splits) > 1 for splits in cluster_split_map.values()
+    )
+    if exact_overlap_count or cluster_overlap_count:
+        raise ValueError("Silver split overlap validation failed")
+    report = {
+        "schemaVersion": 1,
+        "promptVersion": PROMPT_VERSION,
+        "promptSHA256": hashlib.sha256(labeling_instructions().encode()).hexdigest(),
+        "queueCount": len(queue),
+        "acceptedCount": len(accepted),
+        "conflictCount": len(conflicts),
+        "conflictRate": round(len(conflicts) / max(len(queue), 1), 4),
+        "duplicateTextRate": round(
+            (len(accepted) - len(text_split_map)) / max(len(accepted), 1),
+            4,
+        ),
+        "overlapChecks": {
+            "exactTextAcrossSplits": exact_overlap_count,
+            "nearDuplicateClusterAcrossSplits": cluster_overlap_count,
+        },
+        "fullAgreementCount": sum(record["fullAgreement"] for record in accepted)
+        + sum(record["fullAgreement"] for record in conflicts),
+        "fleissKappa": fleiss_kappa(
+            labeler_maps,
+            sorted(expected_ids),
+        ),
+        "labelers": [name for name, _ in labelers],
+        "splitCounts": dict(sorted(Counter(record["split"] for record in accepted).items())),
+        "actionLabelCounts": dict(
+            sorted(Counter(record["actionVerifierLabel"] for record in accepted).items())
+        ),
+        "coordinationLabelCounts": dict(
+            sorted(
+                Counter(
+                    record["coordinationVerifierLabel"] for record in accepted
+                ).items()
+            )
+        ),
+        "intentPositiveCounts": {
+            label: sum(bool(record[label]) for record in accepted)
+            for label in INTENT_LABELS
+        },
+        "languageCounts": dict(
+            sorted(Counter(record["language"] for record in accepted).items())
+        ),
+        "sourceCounts": dict(
+            sorted(
+                Counter(
+                    record.get("sourceDataset") or "unknown"
+                    for record in accepted
+                ).items()
+            )
+        ),
+    }
+    arguments.report.write_text(
+        json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
+        encoding="utf-8",
+    )
+    print(
+        f"CONSENSUS_DONE accepted={len(accepted)} conflicts={len(conflicts)} "
+        f"kappa={report['fleissKappa']}"
+    )
+
+
+def parser() -> argparse.ArgumentParser:
+    root = argparse.ArgumentParser()
+    subparsers = root.add_subparsers(dest="command", required=True)
+
+    prepare_parser = subparsers.add_parser("prepare")
+    prepare_parser.add_argument("--input", type=Path, default=DEFAULT_INPUT)
+    prepare_parser.add_argument(
+        "--queue",
+        type=Path,
+        default=DEFAULT_DIRECTORY / "labeling-queue.jsonl",
+    )
+    prepare_parser.add_argument(
+        "--instructions",
+        type=Path,
+        default=DEFAULT_DIRECTORY / "labeling-instructions.md",
+    )
+    prepare_parser.add_argument(
+        "--prepare-report",
+        type=Path,
+        default=DEFAULT_DIRECTORY / "labeling-queue-report.json",
+    )
+    prepare_parser.add_argument("--count", type=int, default=360)
+    prepare_parser.add_argument("--seed", type=int, default=SEED)
+    prepare_parser.add_argument(
+        "--hard-negative-report",
+        action="append",
+        type=Path,
+        default=[],
+    )
+    prepare_parser.set_defaults(handler=prepare)
+
+    merge_parser = subparsers.add_parser("merge")
+    merge_parser.add_argument(
+        "--queue",
+        type=Path,
+        default=DEFAULT_DIRECTORY / "labeling-queue.jsonl",
+    )
+    merge_parser.add_argument(
+        "--labeler",
+        action="append",
+        type=parse_labeler_argument,
+        required=True,
+    )
+    merge_parser.add_argument(
+        "--consensus",
+        type=Path,
+        default=DEFAULT_DIRECTORY / "consensus-silver.jsonl",
+    )
+    merge_parser.add_argument(
+        "--conflicts",
+        type=Path,
+        default=DEFAULT_DIRECTORY / "consensus-conflicts.jsonl",
+    )
+    merge_parser.add_argument(
+        "--train",
+        type=Path,
+        default=DEFAULT_DIRECTORY / "silver-train.jsonl",
+    )
+    merge_parser.add_argument(
+        "--calibration",
+        type=Path,
+        default=DEFAULT_DIRECTORY / "silver-calibration.jsonl",
+    )
+    merge_parser.add_argument(
+        "--acceptance",
+        type=Path,
+        default=DEFAULT_DIRECTORY / "silver-acceptance.jsonl",
+    )
+    merge_parser.add_argument(
+        "--report",
+        type=Path,
+        default=DEFAULT_DIRECTORY / "consensus-report.json",
+    )
+    merge_parser.set_defaults(handler=merge)
+    return root
+
+
+def main() -> None:
+    arguments = parser().parse_args()
+    arguments.handler(arguments)
+
+
+if __name__ == "__main__":
+    main()

Scripts/clipboard_semantics/generate_random_holdout.py

@@ -0,0 +1,926 @@
+#!/usr/bin/env python3
+"""Generate a reproducible blind random-combination corpus for deployed models.
+
+This corpus is evaluation-only. Its templates and slot vocabulary are kept
+separate from generate_corpus.py and must never be added to model training.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import random
+from dataclasses import asdict, dataclass
+from pathlib import Path
+
+
+DEFAULT_SEED = 20260826
+DEFAULT_SAMPLES_PER_FAMILY = 20
+
+
+@dataclass(frozen=True)
+class Labels:
+    task: bool = False
+    question: bool = False
+    invitation: bool = False
+    complaint: bool = False
+    scheduleNegotiation: bool = False
+    confirmationDecision: bool = False
+    followUpReminder: bool = False
+    sentiment: str = "neutral"
+    replyable: bool = False
+
+
+@dataclass(frozen=True)
+class Record:
+    id: str
+    text: str
+    language: str
+    split: str
+    family: str
+    task: bool
+    question: bool
+    invitation: bool
+    complaint: bool
+    scheduleNegotiation: bool
+    confirmationDecision: bool
+    followUpReminder: bool
+    sentiment: str
+    replyable: bool
+
+
+SLOTS = {
+    "zh-Hans": {
+        "discourse": ["", "对了，", "另外，", "补充一下，", "还有一件事，", "先说重点，", "顺便提一句，", "刚想到，", "简单说，", "单独记一下，"],
+        "owner": ["设计同学", "财务", "供应商", "项目负责人", "客服团队", "运营"],
+        "artifact": ["验收截图", "预算备注", "测试结论", "签字页", "交付清单", "复盘摘要"],
+        "deadline": ["午休前", "明晚之前", "本周收尾时", "下次评审前", "两个工作日内", "月底前"],
+        "channel": ["群里", "工单下方", "邮件线程里", "共享文档中", "项目卡片上"],
+        "topic": ["费用调整", "权限开通", "排期变动", "售后范围", "资料归档", "版本上线"],
+        "event": ["参加小范围评审", "吃顿便饭", "看内部演示", "碰面聊方案", "参加庆功会"],
+        "time": ["周六傍晚", "明天十点半", "下周三午后", "今晚九点", "周一早会后"],
+        "alternate": ["周二四点半", "周四午休后", "明早第一段时间", "周五临下班前", "下周一下午"],
+        "place": ["楼下咖啡店", "三号会议室", "园区北门", "线上会议室", "客户办公室"],
+        "issue": ["页面一直空白", "付款结果重复扣款", "附件始终打不开", "物流状态停了五天", "账号又被锁住"],
+        "impact": ["工作完全卡住", "客户已经在催", "我无法继续操作", "交付时间被耽误", "家人收不到商品"],
+        "option": ["轻量版本", "第二套报价", "供应商丙", "季度结算", "先灰度发布", "线下处理方案"],
+        "trigger": ["客户回信", "补丁上线", "款项入账", "复诊结束", "合同盖章", "样品送达"],
+        "person": ["客户经理", "医生", "仓库负责人", "法务", "房东", "招聘方"],
+        "thing": ["最终结论", "下一步安排", "到账情况", "补充材料", "交付日期", "处理进度"],
+        "update": ["我刚到酒店", "演示比预想顺利", "路上有点堵", "今天终于忙完了", "刚看到你发的照片"],
+        "positive": ["这次响应非常快", "新流程顺手多了", "处理结果超出预期", "讲解特别清楚", "修复后体验很好"],
+        "negative_news": ["行业指数连续回落", "昨夜航班大面积延误", "原材料价格再次上涨", "部分门店暂停营业", "天气预警已经升级"],
+        "fact": ["仓库共有三层", "合同附件为 PDF", "当前版本号是 2.1", "展厅周一闭馆", "蓝色标签表示已归档"],
+    },
+    "en": {
+        "discourse": ["", "Also, ", "One more thing: ", "For context, ", "The main point is this: ", "Just to add, ", "By the way, ", "A quick note: ", "In short, ", "For the record, "],
+        "owner": ["Design", "Finance", "the vendor", "the project lead", "Support", "Operations"],
+        "artifact": ["acceptance screenshots", "budget notes", "test findings", "signature page", "delivery checklist", "retro summary"],
+        "deadline": ["before lunch", "by tomorrow evening", "as this week closes", "before the next review", "within two business days", "before month-end"],
+        "channel": ["in the group thread", "under the ticket", "in the email chain", "inside the shared document", "on the project card"],
+        "topic": ["the fee adjustment", "access activation", "the timeline change", "support coverage", "document archiving", "the release"],
+        "event": ["join a small review", "have a casual dinner", "watch the internal demo", "meet to discuss the proposal", "attend the celebration"],
+        "time": ["Saturday evening", "tomorrow at 10:30", "next Wednesday afternoon", "tonight at nine", "after Monday's stand-up"],
+        "alternate": ["Tuesday at 4:30", "after lunch on Thursday", "first thing tomorrow", "late Friday afternoon", "next Monday afternoon"],
+        "place": ["the cafe downstairs", "meeting room three", "the north campus gate", "the online room", "the client office"],
+        "issue": ["the page stays blank", "the payment was charged twice", "the attachment never opens", "tracking has not moved for five days", "the account is locked again"],
+        "impact": ["all work is blocked", "the client is already chasing us", "I cannot continue", "delivery is now delayed", "my family cannot receive the item"],
+        "option": ["the lightweight version", "the second quote", "vendor C", "quarterly billing", "a limited rollout first", "the offline resolution"],
+        "trigger": ["the client replies", "the patch ships", "the payment lands", "the checkup ends", "the contract is signed", "the sample arrives"],
+        "person": ["the account manager", "the doctor", "the warehouse lead", "Legal", "the landlord", "the recruiter"],
+        "thing": ["the final decision", "next steps", "payment status", "the missing documents", "the delivery date", "resolution progress"],
+        "update": ["I just reached the hotel", "the demo went better than expected", "traffic is a little slow", "I finally wrapped up today", "I just saw the photo you sent"],
+        "positive": ["the response was exceptionally fast", "the new flow is much easier", "the outcome exceeded expectations", "the explanation was crystal clear", "the fix feels solid"],
+        "negative_news": ["the industry index fell again", "many flights were delayed overnight", "raw material prices rose again", "several stores paused operations", "the weather alert was upgraded"],
+        "fact": ["the warehouse has three floors", "the contract attachment is a PDF", "the current version is 2.1", "the showroom closes on Mondays", "a blue label means archived"],
+    },
+}
+
+
+def labels(**values: object) -> Labels:
+    return Labels(**values)
+
+
+FAMILIES = {
+    "task_request": {
+        "labels": labels(task=True, replyable=True),
+        "templates": {
+            "zh-Hans": [
+                "{owner}还缺{artifact}，麻烦{deadline}补齐后在{channel}留言。",
+            ],
+            "en": [
+                "{owner} still needs the {artifact}; please add it {deadline} and leave a note {channel}.",
+            ],
+        },
+    },
+    "task_next_action": {
+        "labels": labels(
+            task=True,
+            followUpReminder=True,
+            replyable=True,
+        ),
+        "templates": {
+            "zh-Hans": [
+                "下一步由你整理{artifact}，请在{deadline}交给{owner}。",
+            ],
+            "en": [
+                "Your next action is to prepare the {artifact} {deadline} for {owner}.",
+            ],
+        },
+    },
+    "information_question": {
+        "labels": labels(question=True, replyable=True),
+        "templates": {
+            "zh-Hans": ["想核实一下，{topic}现在由谁拍板？", "{topic}目前走到哪一步了，方便说明吗？"],
+            "en": ["Quick check: who owns the final call on {topic}?", "Where does {topic} stand right now?"],
+        },
+    },
+    "fixed_invitation": {
+        "labels": labels(question=True, invitation=True, replyable=True),
+        "templates": {
+            "zh-Hans": ["我给你留了位置，{time}到{place}{event}，能来不？", "{time}我们在{place}{event}，要不要一起？"],
+            "en": ["I saved you a spot to {event} {time} at {place}; can you make it?", "Want to {event} with us {time} at {place}?"],
+        },
+    },
+    "complaint_support": {
+        "labels": labels(question=True, complaint=True, sentiment="negative", replyable=True),
+        "templates": {
+            "zh-Hans": ["我已经重试三次，还是{issue}，导致{impact}。请问什么时候能处理？", "{issue}到现在没解决，{impact}，能给一个明确答复吗？"],
+            "en": ["I have tried three times and {issue}; {impact}. When will this be fixed?", "{issue} is still unresolved and {impact}. Can I get a clear answer?"],
+        },
+    },
+    "schedule_negotiation": {
+        "labels": labels(question=True, scheduleNegotiation=True, replyable=True),
+        "templates": {
+            "zh-Hans": ["原来的时段卡住了，我只能{alternate}，能不能对调？", "{time}赶不过去，换成{alternate}你觉得可行吗？"],
+            "en": ["The original slot is blocked for me; could we swap to {alternate}?", "I cannot make {time}. Would {alternate} be workable instead?"],
+        },
+    },
+    "confirmation_decision": {
+        "labels": labels(confirmationDecision=True, replyable=True),
+        "templates": {
+            "zh-Hans": ["不用再比较了，就选{option}，后续都按这个口径走。", "我正式确认{option}，请{deadline}启动后续安排。"],
+            "en": ["No more comparisons: choose {option} and use it as the final direction.", "I formally approve {option}; start the next steps {deadline}."],
+        },
+    },
+    "follow_up_instruction": {
+        "labels": labels(
+            task=True,
+            followUpReminder=True,
+            replyable=True,
+        ),
+        "templates": {
+            "zh-Hans": ["等{trigger}后第二天再找{person}确认{thing}，别漏了。"],
+            "en": ["Once {trigger}, check with {person} the next day about {thing}."],
+        },
+    },
+    "follow_up_personal_reminder": {
+        "labels": labels(followUpReminder=True, replyable=False),
+        "templates": {
+            "zh-Hans": ["个人备忘：提醒我{deadline}联系{person}，追一下{thing}。"],
+            "en": ["Note to self: remind me to contact {person} {deadline} about {thing}."],
+        },
+    },
+    "conversation": {
+        "labels": labels(replyable=True),
+        "templates": {
+            "zh-Hans": ["跟你说一声，{update}，晚点再聊。", "{update}，突然想到你可能会想知道。"],
+            "en": ["Just letting you know, {update}; we can talk later.", "{update}, and I thought you might want to know."],
+        },
+    },
+    "acknowledgment_boundary": {
+        "labels": labels(),
+        "templates": {
+            "zh-Hans": ["看到了，先这样，不需要回复。", "嗯，内容已收到，我只是确认一下。"],
+            "en": ["Seen, that is all for now; no reply needed.", "Okay, I received it. This is only an acknowledgment."],
+        },
+    },
+    "quoted_question_boundary": {
+        "labels": labels(),
+        "templates": {
+            "zh-Hans": ["文档标题是“{topic}怎么办？”，这里不是在问你。", "纪要保留了“谁负责{topic}？”这句话，答案已经写在后面。"],
+            "en": ["The document heading says “What about {topic}?”, but it is not asking you.", "The notes preserve the question “Who owns {topic}?”, which is answered below."],
+        },
+    },
+    "event_boundary": {
+        "labels": labels(),
+        "templates": {
+            "zh-Hans": ["公告只记录一件事：{event}定在{time}，地点是{place}。", "历史行程显示他们曾在{place}{event}，没有邀请任何人。"],
+            "en": ["The notice only records that they will {event} {time} at {place}.", "The old itinerary says they went to {place} to {event}; nobody is being invited."],
+        },
+    },
+    "vague_future_boundary": {
+        "labels": labels(),
+        "templates": {
+            "zh-Hans": ["哪天有心情再找{person}聊{thing}吧。", "以后也许会看看{topic}，目前没有安排。"],
+            "en": ["Maybe I will talk to {person} about {thing} someday.", "I may revisit {topic} eventually, but there is no plan."],
+        },
+    },
+    "positive_feedback": {
+        "labels": labels(sentiment="positive", replyable=True),
+        "templates": {
+            "zh-Hans": ["必须夸一下，{positive}，谢谢你们。", "{positive}，整个过程让人很安心。"],
+            "en": ["Credit where it is due: {positive}. Thank you.", "{positive}, and the whole process felt reassuring."],
+        },
+    },
+    "negative_news": {
+        "labels": labels(sentiment="negative"),
+        "templates": {
+            "zh-Hans": ["新闻简报显示，{negative_news}，本文仅陈述情况。", "数据显示{negative_news}，没有提出处理诉求。"],
+            "en": ["The news brief reports that {negative_news}; this is only a factual summary.", "Data shows that {negative_news}, with no request for support."],
+        },
+    },
+    "neutral_fact": {
+        "labels": labels(),
+        "templates": {
+            "zh-Hans": ["资料页写明：{fact}。", "当前记录只包含一个事实：{fact}。"],
+            "en": ["The reference page states that {fact}.", "The current record contains one fact: {fact}."],
+        },
+    },
+}
+
+FRESH_FAMILIES = {
+    "fresh_task_assignment": {
+        "labels": labels(task=True, replyable=True),
+        "templates": {
+            "zh-Hans": [
+                "这份交付由你收尾：把{artifact}整理好，最迟{deadline}交给{owner}。",
+                "{owner}把{artifact}分给你处理，完成时间不能晚于{deadline}。",
+            ],
+            "en": [
+                "You are closing out this deliverable: finish the {artifact} and give it to {owner} {deadline}.",
+                "{owner} assigned the {artifact} to you, with completion required {deadline}.",
+            ],
+        },
+    },
+    "fresh_task_question": {
+        "labels": labels(task=True, question=True, replyable=True),
+        "templates": {
+            "zh-Hans": [
+                "{artifact}能由你在{deadline}前收尾吗？完成后告诉{owner}。",
+                "这项分工你能接吗：{deadline}整理{artifact}？",
+            ],
+            "en": [
+                "Can you close out the {artifact} {deadline} and update {owner}?",
+                "Can you take this assignment and finish the {artifact} {deadline}?",
+            ],
+        },
+    },
+    "fresh_self_plan_boundary": {
+        "labels": labels(),
+        "templates": {
+            "zh-Hans": [
+                "我可能自己看看{topic}，但还没决定什么时候做。",
+                "只是个人想法：以后也许整理{artifact}，目前没有安排。",
+            ],
+            "en": [
+                "I may look into {topic} myself, but I have not decided when.",
+                "This is only a personal idea: perhaps I will organize the {artifact} someday.",
+            ],
+        },
+    },
+    "fresh_complaint": {
+        "labels": labels(complaint=True, sentiment="negative", replyable=True),
+        "templates": {
+            "zh-Hans": [
+                "本来承诺今天解决，结果还是{issue}，现在{impact}。",
+                "同样的故障第三次出现，{issue}，整个事情已经{impact}。",
+            ],
+            "en": [
+                "This was promised for today, yet {issue}, and now {impact}.",
+                "The same failure has happened a third time: {issue}, so {impact}.",
+            ],
+        },
+    },
+    "fresh_complaint_request": {
+        "labels": labels(
+            question=True,
+            complaint=True,
+            sentiment="negative",
+            replyable=True,
+        ),
+        "templates": {
+            "zh-Hans": [
+                "{issue}已经影响到{impact}，你们准备哪天真正解决？",
+                "因为{issue}，现在{impact}，能不能给出明确处理时限？",
+            ],
+            "en": [
+                "{issue} has reached the point where {impact}. When will you actually resolve it?",
+                "Because {issue}, {impact}. Can you provide a firm resolution date?",
+            ],
+        },
+    },
+    "fresh_negative_report_boundary": {
+        "labels": labels(sentiment="negative"),
+        "templates": {
+            "zh-Hans": [
+                "行业通报称{negative_news}，这里只摘录公开信息。",
+                "统计报告记录了{negative_news}，没有客户投诉。",
+            ],
+            "en": [
+                "The industry bulletin says that {negative_news}; this only quotes public information.",
+                "The statistical report records that {negative_news}, with no customer complaint.",
+            ],
+        },
+    },
+    "fresh_confirmation": {
+        "labels": labels(confirmationDecision=True, replyable=True),
+        "templates": {
+            "zh-Hans": [
+                "评审结束，我的最终选择是{option}，其他方案关闭。",
+                "结论正式生效：{topic}采用{option}。",
+            ],
+            "en": [
+                "The review is over; my final selection is {option}, and the alternatives are closed.",
+                "The decision is now official: use {option} for {topic}.",
+            ],
+        },
+    },
+    "fresh_pending_boundary": {
+        "labels": labels(),
+        "templates": {
+            "zh-Hans": [
+                "我看过{option}了，但还没有选择，等明天再决定。",
+                "{topic}仍在评审，当前没有批准结论。",
+            ],
+            "en": [
+                "I reviewed {option}, but have not selected it; the decision waits until tomorrow.",
+                "{topic} remains under review, with no approval yet.",
+            ],
+        },
+    },
+    "fresh_follow_up": {
+        "labels": labels(
+            task=True,
+            followUpReminder=True,
+            replyable=True,
+        ),
+        "templates": {
+            "zh-Hans": [
+                "等{trigger}满两天后，再找{person}核实{thing}并记录结果。",
+                "当前事项结束以后，回头联系{person}追踪{thing}。",
+            ],
+            "en": [
+                "Two days after {trigger}, check with {person} again about {thing} and record the outcome.",
+                "After the current item closes, return to {person} and track {thing}.",
+            ],
+        },
+    },
+    "fresh_personal_reminder": {
+        "labels": labels(followUpReminder=True),
+        "templates": {
+            "zh-Hans": [
+                "给自己设一条后续提醒：{deadline}找{person}确认{thing}。",
+                "个人行动项：{trigger}以后再次检查{thing}。",
+            ],
+            "en": [
+                "Set myself a follow-up reminder to ask {person} about {thing} {deadline}.",
+                "Personal action item: check {thing} again after {trigger}.",
+            ],
+        },
+    },
+    "fresh_plain_task_boundary": {
+        "labels": labels(task=True, replyable=True),
+        "templates": {
+            "zh-Hans": [
+                "现在请直接整理{artifact}并发给{owner}，不需要后续回访。",
+                "一次性完成{artifact}即可，交给{owner}后任务结束。",
+            ],
+            "en": [
+                "Prepare the {artifact} now and send it to {owner}; no later follow-up is needed.",
+                "Complete the {artifact} once, deliver it to {owner}, and close the task.",
+            ],
+        },
+    },
+    "fresh_vague_follow_boundary": {
+        "labels": labels(),
+        "templates": {
+            "zh-Hans": [
+                "以后说不定会再问{person}，目前没有后续计划。",
+                "{thing}哪天想起来再看，现在不用提醒。",
+            ],
+            "en": [
+                "I might ask {person} again someday, but there is no follow-up plan.",
+                "Maybe I will revisit {thing} whenever it comes to mind; no reminder is needed.",
+            ],
+        },
+    },
+}
+
+TARGETED_FINAL_FAMILIES = {
+    "final_task_direct": {
+        "labels": labels(task=True, replyable=True),
+        "templates": {
+            "zh-Hans": [
+                "{artifact}现在明确由你负责，{deadline}前交给{owner}。",
+                "请你承担{artifact}的交付，完成后在{channel}确认。",
+            ],
+            "en": [
+                "Ownership of the {artifact} now belongs to you; deliver it to {owner} {deadline}.",
+                "You are responsible for delivering the {artifact}; confirm completion {channel}.",
+            ],
+        },
+    },
+    "final_task_indirect": {
+        "labels": labels(task=True, replyable=True),
+        "templates": {
+            "zh-Hans": [
+                "{owner}希望你能接下{artifact}，并在{deadline}前收尾。",
+                "这件事想交给你继续推进：整理{artifact}。",
+            ],
+            "en": [
+                "{owner} is counting on you to pick up the {artifact} and close it out {deadline}.",
+                "We would like you to carry this forward by preparing the {artifact}.",
+            ],
+        },
+    },
+    "final_task_question": {
+        "labels": labels(task=True, question=True, replyable=True),
+        "templates": {
+            "zh-Hans": [
+                "能请你负责{artifact}并在{deadline}前完成吗？",
+                "{artifact}接下来可以由你收尾吗？",
+            ],
+            "en": [
+                "Would you be able to own the {artifact} and finish it {deadline}?",
+                "Could the {artifact} be left with you for final completion?",
+            ],
+        },
+    },
+    "final_personal_action_boundary": {
+        "labels": labels(followUpReminder=True),
+        "templates": {
+            "zh-Hans": [
+                "这是我自己的行动清单：{trigger}后检查{thing}。",
+                "个人记录，不是委派：{deadline}找{person}确认{thing}。",
+            ],
+            "en": [
+                "This is on my own action list: check {thing} after {trigger}.",
+                "Personal note, not delegated work: ask {person} about {thing} {deadline}.",
+            ],
+        },
+    },
+    "final_self_intent_boundary": {
+        "labels": labels(),
+        "templates": {
+            "zh-Hans": [
+                "我自己可能会整理{artifact}，但还没有正式计划。",
+                "以后有空我再看{topic}，目前不用安排。",
+            ],
+            "en": [
+                "I may organize the {artifact} myself, but there is no firm plan.",
+                "I might look at {topic} when I have time; nothing is scheduled.",
+            ],
+        },
+    },
+    "final_event_boundary": {
+        "labels": labels(),
+        "templates": {
+            "zh-Hans": [
+                "{owner}在{deadline}举行评审，这只是日程通知。",
+                "{topic}会议已经定在{deadline}，没有分配任务。",
+            ],
+            "en": [
+                "{owner} is holding a review {deadline}; this is only a calendar notice.",
+                "The meeting about {topic} is set for {deadline}, with no assignment.",
+            ],
+        },
+    },
+    "final_complaint_implicit": {
+        "labels": labels(complaint=True, sentiment="negative", replyable=True),
+        "templates": {
+            "zh-Hans": [
+                "我不应该为同一个问题反复追问，{issue}，现在{impact}。",
+                "又是同样的结果：{issue}，已经连续影响到{impact}。",
+            ],
+            "en": [
+                "I should not have to chase the same issue repeatedly: {issue}, and now {impact}.",
+                "It is the same outcome again: {issue}, repeatedly leaving me with {impact}.",
+            ],
+        },
+    },
+    "final_complaint_short": {
+        "labels": labels(complaint=True, sentiment="negative", replyable=True),
+        "templates": {
+            "zh-Hans": [
+                "{issue}，已经第三次了。",
+                "等了这么久还是{issue}，实在无法接受。",
+            ],
+            "en": [
+                "{issue}. This is already the third time.",
+                "After all this waiting, {issue}; this is not acceptable.",
+            ],
+        },
+    },
+    "final_complaint_request": {
+        "labels": labels(
+            question=True,
+            complaint=True,
+            sentiment="negative",
+            replyable=True,
+        ),
+        "templates": {
+            "zh-Hans": [
+                "{issue}一直没有变化，什么时候才能真正处理？",
+                "因为{issue}，现在{impact}，谁能给出解决结果？",
+            ],
+            "en": [
+                "Nothing has changed with this problem: {issue}. When will it actually be handled?",
+                "Because {issue}, {impact}. Who can provide a real resolution?",
+            ],
+        },
+    },
+    "final_negative_fact_boundary": {
+        "labels": labels(sentiment="negative"),
+        "templates": {
+            "zh-Hans": [
+                "公开报告显示{negative_news}，这里没有服务申诉。",
+                "资料仅记录{negative_news}，不涉及个人问题。",
+            ],
+            "en": [
+                "The public report shows that {negative_news}; there is no service grievance here.",
+                "The document only records that {negative_news}, not an individual problem.",
+            ],
+        },
+    },
+    "final_resolved_boundary": {
+        "labels": labels(),
+        "templates": {
+            "zh-Hans": [
+                "之前的{issue}已经彻底恢复，这是一条结案记录。",
+                "{topic}的问题处理完毕，现在不需要任何支持。",
+            ],
+            "en": [
+                "The earlier issue where {issue} is fully resolved; this is a closure record.",
+                "The problem involving {topic} is complete, with no support needed now.",
+            ],
+        },
+    },
+    "final_operational_task_boundary": {
+        "labels": labels(task=True, replyable=True),
+        "templates": {
+            "zh-Hans": [
+                "请完成{artifact}后直接结项，不需要投诉或后续回访。",
+                "{deadline}前把{artifact}交给{owner}，随后流程关闭。",
+            ],
+            "en": [
+                "Complete the {artifact} and close the item; no complaint or later follow-up is involved.",
+                "Deliver the {artifact} to {owner} {deadline}, then close the workflow.",
+            ],
+        },
+    },
+}
+
+TARGETED_CONFIRMATION_FAMILIES = {
+    "confirmation_task_assignment": {
+        "labels": labels(task=True, replyable=True),
+        "templates": {
+            "zh-Hans": [
+                "请你把{artifact}负责到底，并在{deadline}向{owner}交付。",
+                "{artifact}接下来归你处理，完成后在{channel}更新状态。",
+            ],
+            "en": [
+                "Please see the {artifact} through and deliver it to {owner} {deadline}.",
+                "The {artifact} is yours to handle next; update the status {channel} when complete.",
+            ],
+        },
+    },
+    "confirmation_task_request": {
+        "labels": labels(task=True, question=True, replyable=True),
+        "templates": {
+            "zh-Hans": [
+                "可以请你接管{artifact}并在{deadline}前处理好吗？",
+                "你愿意负责{artifact}的最后交付吗？",
+            ],
+            "en": [
+                "Could I ask you to take care of the {artifact} {deadline}?",
+                "Would you own the final delivery of the {artifact}?",
+            ],
+        },
+    },
+    "confirmation_operational_task": {
+        "labels": labels(task=True, replyable=True),
+        "templates": {
+            "zh-Hans": [
+                "把{artifact}交给{owner}后直接关闭事项，不安排回访。",
+                "当前只需要完成{artifact}，没有投诉处理。",
+            ],
+            "en": [
+                "Close the item after sending the {artifact} to {owner}; do not schedule a follow-up.",
+                "The only requirement is to complete the {artifact}; no grievance handling is involved.",
+            ],
+        },
+    },
+    "confirmation_personal_boundary": {
+        "labels": labels(followUpReminder=True),
+        "templates": {
+            "zh-Hans": [
+                "我自己的行动备忘：{trigger}后查看{thing}。",
+                "仅记录个人计划，{deadline}联系{person}。",
+            ],
+            "en": [
+                "My private action note is to review {thing} after {trigger}.",
+                "This only records my personal plan to contact {person} {deadline}.",
+            ],
+        },
+    },
+    "confirmation_self_plan_boundary": {
+        "labels": labels(),
+        "templates": {
+            "zh-Hans": [
+                "我也许会自己整理{artifact}，目前没有确定安排。",
+                "{topic}以后再考虑，现在谁都不用处理。",
+            ],
+            "en": [
+                "I may prepare the {artifact} myself, but nothing is arranged.",
+                "{topic} can be considered later; nobody needs to handle it now.",
+            ],
+        },
+    },
+    "confirmation_complaint_implicit": {
+        "labels": labels(complaint=True, sentiment="negative", replyable=True),
+        "templates": {
+            "zh-Hans": [
+                "我到现在还在面对{issue}，而且{impact}。",
+                "处理承诺没有兑现，结果仍然是{issue}。",
+            ],
+            "en": [
+                "I am still dealing with the fact that {issue}, and {impact}.",
+                "The promised fix never materialized; the result is still that {issue}.",
+            ],
+        },
+    },
+    "confirmation_complaint_request": {
+        "labels": labels(
+            question=True,
+            complaint=True,
+            sentiment="negative",
+            replyable=True,
+        ),
+        "templates": {
+            "zh-Hans": [
+                "{issue}已经持续很久，究竟什么时候恢复？",
+                "现在因为{issue}而{impact}，谁来负责处理？",
+            ],
+            "en": [
+                "This has continued for far too long: {issue}. When will it be restored?",
+                "Because {issue}, {impact}. Who is responsible for fixing it?",
+            ],
+        },
+    },
+    "confirmation_complaint_short": {
+        "labels": labels(complaint=True, sentiment="negative", replyable=True),
+        "templates": {
+            "zh-Hans": [
+                "还是{issue}，完全没有改善。",
+                "{issue}又来了，不能一直这样。",
+            ],
+            "en": [
+                "It is still the case that {issue}, with no improvement.",
+                "The problem is back again: {issue}. This cannot keep happening.",
+            ],
+        },
+    },
+    "confirmation_negative_fact": {
+        "labels": labels(sentiment="negative"),
+        "templates": {
+            "zh-Hans": [
+                "研究资料指出{negative_news}，这不是客户反馈。",
+                "行业统计记录了{negative_news}，没有服务诉求。",
+            ],
+            "en": [
+                "The research notes that {negative_news}; this is not customer feedback.",
+                "Industry statistics record that {negative_news}, with no service request.",
+            ],
+        },
+    },
+    "confirmation_resolved_boundary": {
+        "labels": labels(),
+        "templates": {
+            "zh-Hans": [
+                "{issue}已经恢复，{owner}确认事项关闭。",
+                "{topic}当前运行正常，不需要补救。",
+            ],
+            "en": [
+                "The earlier state where {issue} is resolved, and {owner} confirmed closure.",
+                "{topic} is operating normally now, with no remedy needed.",
+            ],
+        },
+    },
+}
+
+TARGETED_RELEASE_FAMILIES = {
+    "release_task_owner": {
+        "labels": labels(task=True, replyable=True),
+        "templates": {
+            "zh-Hans": [
+                "{artifact}的负责人就是你，请在{deadline}完成交付。",
+                "接下来请你推进{artifact}，并向{owner}汇报结果。",
+            ],
+            "en": [
+                "You are the owner of the {artifact}; complete delivery {deadline}.",
+                "Please move the {artifact} forward and report the outcome to {owner}.",
+            ],
+        },
+    },
+    "release_task_polite_request": {
+        "labels": labels(task=True, question=True, replyable=True),
+        "templates": {
+            "zh-Hans": [
+                "能麻烦你把{artifact}负责到交付完成吗？",
+                "你可以在{deadline}前处理好{artifact}吗？",
+            ],
+            "en": [
+                "May I ask you to own the {artifact} through delivery?",
+                "Can you have the {artifact} completed {deadline}?",
+            ],
+        },
+    },
+    "release_personal_boundary": {
+        "labels": labels(followUpReminder=True),
+        "templates": {
+            "zh-Hans": [
+                "这是我的私人待办：{trigger}之后确认{thing}。",
+                "个人备忘，不交给任何人：{deadline}联系{person}。",
+            ],
+            "en": [
+                "This is my private todo: confirm {thing} after {trigger}.",
+                "Personal reminder, assigned to nobody else: contact {person} {deadline}.",
+            ],
+        },
+    },
+    "release_complaint_unspoken": {
+        "labels": labels(complaint=True, sentiment="negative", replyable=True),
+        "templates": {
+            "zh-Hans": [
+                "从上次反馈以后仍然{issue}，现在已经{impact}。",
+                "说好的处理没有发生，我看到的还是{issue}。",
+            ],
+            "en": [
+                "Since my last report, {issue}, and it has now reached the point where {impact}.",
+                "The promised handling did not happen; I am still seeing that {issue}.",
+            ],
+        },
+    },
+    "release_complaint_question": {
+        "labels": labels(
+            question=True,
+            complaint=True,
+            sentiment="negative",
+            replyable=True,
+        ),
+        "templates": {
+            "zh-Hans": [
+                "{issue}到现在都没处理，什么时候能恢复正常？",
+                "目前因为{issue}而{impact}，可以给个处理结果吗？",
+            ],
+            "en": [
+                "The problem where {issue} has not been addressed. When will normal service return?",
+                "At the moment, {issue}, so {impact}. Can I get an actual resolution?",
+            ],
+        },
+    },
+    "release_complaint_brief": {
+        "labels": labels(complaint=True, sentiment="negative", replyable=True),
+        "templates": {
+            "zh-Hans": [
+                "{issue}，到现在还是老样子。",
+                "又一次{issue}，这已经影响正常使用。",
+            ],
+            "en": [
+                "{issue}, and nothing has changed.",
+                "Once again, {issue}; normal use is now affected.",
+            ],
+        },
+    },
+    "release_negative_boundary": {
+        "labels": labels(sentiment="negative"),
+        "templates": {
+            "zh-Hans": [
+                "分析报告提到{negative_news}，不代表用户投诉。",
+                "公开数据包含{negative_news}，这里只做事实引用。",
+            ],
+            "en": [
+                "The analysis mentions that {negative_news}; it does not represent a user complaint.",
+                "Public data includes the fact that {negative_news}; this is only a factual citation.",
+            ],
+        },
+    },
+    "release_resolved_boundary": {
+        "labels": labels(),
+        "templates": {
+            "zh-Hans": [
+                "{issue}的问题已经结束，目前状态稳定。",
+                "{topic}已确认恢复，{owner}不需要继续介入。",
+            ],
+            "en": [
+                "The earlier problem where {issue} is over, and the current state is stable.",
+                "{topic} is confirmed restored, so {owner} does not need to intervene.",
+            ],
+        },
+    },
+}
+
+
+def render(template: str, slots: dict[str, list[str]], rng: random.Random) -> str:
+    values = {key: rng.choice(options) for key, options in slots.items()}
+    return values["discourse"] + template.format(**values)
+
+
+def generate(
+    seed: int,
+    samples_per_family: int,
+    families: dict[str, dict[str, object]],
+) -> list[Record]:
+    rng = random.Random(seed)
+    records: list[Record] = []
+    seen: set[str] = set()
+
+    for language in ("zh-Hans", "en"):
+        slots = SLOTS[language]
+        for family, configuration in families.items():
+            family_labels: Labels = configuration["labels"]
+            templates: list[str] = configuration["templates"][language]
+            generated = 0
+            attempts = 0
+            while generated < samples_per_family:
+                attempts += 1
+                if attempts > samples_per_family * 200:
+                    raise RuntimeError(f"Unable to generate unique samples for {language}/{family}")
+                text = render(rng.choice(templates), slots, rng)
+                normalized = " ".join(text.casefold().split())
+                if normalized in seen:
+                    continue
+                seen.add(normalized)
+                record_id = f"random-{seed}-{language}-{family}-{generated + 1:03d}"
+                records.append(
+                    Record(
+                        id=record_id,
+                        text=text,
+                        language=language,
+                        split="randomHoldout",
+                        family=family,
+                        **asdict(family_labels),
+                    )
+                )
+                generated += 1
+
+    rng.shuffle(records)
+    return records
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--seed", type=int, default=DEFAULT_SEED)
+    parser.add_argument("--samples-per-family", type=int, default=DEFAULT_SAMPLES_PER_FAMILY)
+    parser.add_argument(
+        "--profile",
+        choices=(
+            "development",
+            "fresh",
+            "targeted-final",
+            "targeted-confirmation",
+            "targeted-release",
+        ),
+        default="development",
+    )
+    parser.add_argument(
+        "--output",
+        type=Path,
+        default=None,
+    )
+    arguments = parser.parse_args()
+
+    profiles = {
+        "development": FAMILIES,
+        "fresh": FRESH_FAMILIES,
+        "targeted-final": TARGETED_FINAL_FAMILIES,
+        "targeted-confirmation": TARGETED_CONFIRMATION_FAMILIES,
+        "targeted-release": TARGETED_RELEASE_FAMILIES,
+    }
+    families = profiles[arguments.profile]
+    default_outputs = {
+        "development": "random-holdout-corpus.jsonl",
+        "fresh": "fresh-metric-holdout-corpus.jsonl",
+        "targeted-final": "targeted-final-holdout-corpus.jsonl",
+        "targeted-confirmation": "targeted-confirmation-holdout-corpus.jsonl",
+        "targeted-release": "targeted-release-holdout-corpus.jsonl",
+    }
+    output_path = arguments.output or Path(
+        "ModelTraining/ClipboardSemantics/" + default_outputs[arguments.profile]
+    )
+    records = generate(arguments.seed, arguments.samples_per_family, families)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    with output_path.open("w", encoding="utf-8") as output:
+        for record in records:
+            output.write(json.dumps(asdict(record), ensure_ascii=False, sort_keys=True) + "\n")
+
+    print(
+        f"RANDOM_HOLDOUT_DONE records={len(records)} "
+        f"families={len(families)} profile={arguments.profile} "
+        f"seed={arguments.seed} output={output_path}"
+    )
+
+
+if __name__ == "__main__":
+    main()

Scripts/clipboard_semantics/tests/test_apply_verifier_deployment_policy.py

@@ -0,0 +1,109 @@
+import json
+import subprocess
+import tempfile
+import unittest
+from pathlib import Path
+
+
+SCRIPT = Path(__file__).resolve().parents[1] / "apply_verifier_deployment_policy.py"
+
+
+class VerifierDeploymentPolicyTests(unittest.TestCase):
+    def test_rejected_candidate_cannot_change_deployed_manifest(self):
+        with self._fixture(accepted=False) as fixture:
+            self._run(fixture)
+
+            deployed = self._read(fixture["deployed_manifest"])
+            report = self._read(fixture["report"])
+            self.assertEqual(2, deployed["schemaVersion"])
+            self.assertFalse(report["applied"])
+            self.assertEqual(0, report["eligibleVerifierCount"])
+
+    def test_passing_candidate_enters_shadow_and_preserves_classifiers(self):
+        with self._fixture(accepted=True) as fixture:
+            self._run(fixture)
+
+            deployed = self._read(fixture["deployed_manifest"])
+            report = self._read(fixture["report"])
+            self.assertEqual(3, deployed["schemaVersion"])
+            self.assertEqual("task", deployed["classifiers"][0]["id"])
+            self.assertEqual("shadow", deployed["verifiers"][0]["deploymentMode"])
+            self.assertFalse(
+                deployed["verifiers"][0]["acceptedForAutomaticRouting"]
+            )
+            self.assertTrue(report["applied"])
+
+    def _fixture(self, accepted):
+        temporary = tempfile.TemporaryDirectory()
+        root = Path(temporary.name)
+        candidate = root / "candidate"
+        resource = root / "resource"
+        candidate.mkdir()
+        resource.mkdir()
+        verifier = {
+            "id": "action",
+            "modelFile": "ActionIntentVerifier.mlmodel",
+            "acceptedForAutomaticRouting": accepted,
+            "deploymentMode": "automatic" if accepted else "shadow",
+        }
+        (candidate / verifier["modelFile"]).write_bytes(b"model")
+        self._write(
+            candidate / "clipboard-semantic-models.json",
+            {
+                "schemaVersion": 3,
+                "classifiers": [{"id": "untrusted-candidate"}],
+                "verifiers": [verifier],
+            },
+        )
+        deployed_manifest = resource / "clipboard-semantic-models.json"
+        self._write(
+            deployed_manifest,
+            {
+                "schemaVersion": 2,
+                "classifiers": [{"id": "task"}],
+            },
+        )
+        fixture = {
+            "temporary": temporary,
+            "candidate": candidate,
+            "resource": resource,
+            "deployed_manifest": deployed_manifest,
+            "report": root / "report.json",
+        }
+
+        class Context:
+            def __enter__(self):
+                return fixture
+
+            def __exit__(self, *unused):
+                temporary.cleanup()
+
+        return Context()
+
+    def _run(self, fixture):
+        subprocess.run(
+            [
+                "python3",
+                str(SCRIPT),
+                "--candidate-directory",
+                str(fixture["candidate"]),
+                "--resource-directory",
+                str(fixture["resource"]),
+                "--report",
+                str(fixture["report"]),
+                "--apply",
+            ],
+            check=True,
+            capture_output=True,
+            text=True,
+        )
+
+    def _write(self, path, value):
+        path.write_text(json.dumps(value), encoding="utf-8")
+
+    def _read(self, path):
+        return json.loads(path.read_text(encoding="utf-8"))
+
+
+if __name__ == "__main__":
+    unittest.main()

Scripts/clipboard_semantics/tests/test_generate_consensus_labels.py

@@ -0,0 +1,138 @@
+import json
+import tempfile
+import unittest
+from pathlib import Path
+from types import SimpleNamespace
+
+import sys
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+import generate_consensus_labels as consensus
+
+
+class ConsensusLabelTests(unittest.TestCase):
+    def test_requires_source_support_for_two_of_three_vote(self):
+        report, accepted, conflicts = self._merge(
+            source_labels={},
+            label_sets=[["task"], ["task"], []],
+        )
+
+        self.assertEqual([], accepted)
+        self.assertEqual("no-supported-consensus", conflicts[0]["rejectedReason"])
+        self.assertEqual(1, report["conflictCount"])
+
+    def test_accepts_two_of_three_when_official_label_supports_vote(self):
+        report, accepted, conflicts = self._merge(
+            source_labels={"task": True},
+            label_sets=[["task"], ["task"], []],
+        )
+
+        self.assertEqual([], conflicts)
+        self.assertEqual("taskOnly", accepted[0]["actionVerifierLabel"])
+        self.assertTrue(accepted[0]["task"])
+        self.assertEqual(1, report["acceptedCount"])
+
+    def test_rejects_multi_coordination_consensus(self):
+        _, accepted, conflicts = self._merge(
+            source_labels={"invitation": True, "scheduleNegotiation": True},
+            label_sets=[
+                ["invitation", "scheduleNegotiation"],
+                ["invitation", "scheduleNegotiation"],
+                ["invitation", "scheduleNegotiation"],
+            ],
+        )
+
+        self.assertEqual([], accepted)
+        self.assertEqual(
+            "multiple-coordination-labels",
+            conflicts[0]["rejectedReason"],
+        )
+
+    def test_near_duplicate_slot_variants_share_split(self):
+        first = {
+            "text": "Could you send report 123 before Friday?",
+        }
+        second = {
+            "text": "Could you send report 456 before Friday?",
+        }
+
+        self.assertEqual(
+            consensus.split_for(first),
+            consensus.split_for(second),
+        )
+
+    def _merge(self, source_labels, label_sets):
+        with tempfile.TemporaryDirectory() as raw_directory:
+            directory = Path(raw_directory)
+            queue_path = directory / "queue.jsonl"
+            self._write_json_lines(
+                queue_path,
+                [
+                    {
+                        "id": "record-1",
+                        "text": "Could you send the report?",
+                        "language": "en",
+                        "sourceDataset": "fixture",
+                        "sourceLicense": "MIT",
+                        "sourceLabels": source_labels,
+                    }
+                ],
+            )
+            labelers = []
+            for index, labels in enumerate(label_sets):
+                path = directory / f"labeler-{index}.jsonl"
+                self._write_json_lines(
+                    path,
+                    [
+                        {
+                            "id": "record-1",
+                            "labels": labels,
+                            "ambiguous": False,
+                            "quotedOrMeta": False,
+                            "confidence": 0.95,
+                        }
+                    ],
+                )
+                labelers.append((f"labeler-{index}", path))
+            paths = {
+                name: directory / f"{name}.jsonl"
+                for name in (
+                    "consensus",
+                    "conflicts",
+                    "train",
+                    "calibration",
+                    "acceptance",
+                )
+            }
+            report_path = directory / "report.json"
+            consensus.merge(
+                SimpleNamespace(
+                    queue=queue_path,
+                    labeler=labelers,
+                    report=report_path,
+                    **paths,
+                )
+            )
+            return (
+                json.loads(report_path.read_text(encoding="utf-8")),
+                self._read_json_lines(paths["consensus"]),
+                self._read_json_lines(paths["conflicts"]),
+            )
+
+    def _write_json_lines(self, path, records):
+        path.write_text(
+            "\n".join(json.dumps(record) for record in records) + "\n",
+            encoding="utf-8",
+        )
+
+    def _read_json_lines(self, path):
+        return [
+            json.loads(line)
+            for line in path.read_text(encoding="utf-8").splitlines()
+            if line
+        ]
+
+
+if __name__ == "__main__":
+    unittest.main()

Scripts/clipboard_semantics/train_models.swift

@@ -9,10 +9,16 @@ private struct CorpusRecord: Codable {
     let language: String
     let split: String
     let family: String
+    let knownLabels: Set<String>?
+    let sourceDataset: String?
     let task: Bool
     let question: Bool
     let invitation: Bool
     let complaint: Bool
+    let scheduleNegotiation: Bool
+    let confirmationDecision: Bool
+    let followUpReminder: Bool
+    let blessing: Bool
     let sentiment: String
     let replyable: Bool
 }
@@ -44,6 +50,7 @@ private struct CandidateReport: Codable {
     let balancedTrainingCount: Int
     let balancedValidationCount: Int
     let threshold: Double?
+    let confidenceThresholdsByLanguage: [String: Double]?
     let acceptedForAutomaticRouting: Bool
     let validationBinary: BinaryMetrics?
     let testBinary: BinaryMetrics?
@@ -53,6 +60,7 @@ private struct CandidateReport: Codable {
     let goldenFalsePositiveExamples: [String]?
     let goldenFalseNegativeExamples: [String]?
     let binaryByLanguage: [String: BinaryMetrics]?
+    let goldenBinaryByLanguage: [String: BinaryMetrics]?
     let validationMulticlass: MulticlassMetrics?
     let testMulticlass: MulticlassMetrics?
     let goldenMulticlass: MulticlassMetrics?
@@ -87,6 +95,7 @@ private struct ManifestClassifier: Codable {
     let labels: [String]
     let positiveLabel: String?
     let confidenceThreshold: Double?
+    let confidenceThresholdsByLanguage: [String: Double]?
     let acceptedForAutomaticRouting: Bool
 }
 
@@ -118,11 +127,19 @@ private enum CandidateAlgorithm: String, CaseIterable {
     }
 }
 
+private let usesBaselineNegativePolicy = CommandLine.arguments.contains(
+    "--baseline-negative-policy"
+)
+
 private enum ClassifierID: String, CaseIterable {
     case task
     case question
     case invitation
     case complaint
+    case scheduleNegotiation
+    case confirmationDecision
+    case followUpReminder
+    case blessing
     case replyableMessage
     case sentiment
 
@@ -132,6 +149,10 @@ private enum ClassifierID: String, CaseIterable {
         case .question: "QuestionIntentClassifier"
         case .invitation: "InvitationIntentClassifier"
         case .complaint: "ComplaintIntentClassifier"
+        case .scheduleNegotiation: "ScheduleNegotiationIntentClassifier"
+        case .confirmationDecision: "ConfirmationDecisionIntentClassifier"
+        case .followUpReminder: "FollowUpReminderIntentClassifier"
+        case .blessing: "BlessingIntentClassifier"
         case .replyableMessage: "ConversationalReplyIntentClassifier"
         case .sentiment: "SentimentClassifier"
         }
@@ -143,6 +164,10 @@ private enum ClassifierID: String, CaseIterable {
         case .question: ["notQuestion", "question"]
         case .invitation: ["notInvitation", "invitation"]
         case .complaint: ["notComplaint", "complaint"]
+        case .scheduleNegotiation: ["notScheduleNegotiation", "scheduleNegotiation"]
+        case .confirmationDecision: ["notConfirmationDecision", "confirmationDecision"]
+        case .followUpReminder: ["notFollowUpReminder", "followUpReminder"]
+        case .blessing: ["notBlessing", "blessing"]
         case .replyableMessage: ["notReplyableMessage", "replyableMessage"]
         case .sentiment: ["negative", "neutral", "positive"]
         }
@@ -154,21 +179,168 @@ private enum ClassifierID: String, CaseIterable {
         case .question: "question"
         case .invitation: "invitation"
         case .complaint: "complaint"
+        case .scheduleNegotiation: "scheduleNegotiation"
+        case .confirmationDecision: "confirmationDecision"
+        case .followUpReminder: "followUpReminder"
+        case .blessing: "blessing"
         case .replyableMessage: "replyableMessage"
         case .sentiment: nil
         }
     }
 
+    var hardNegativeFamilies: Set<String> {
+        switch self {
+        case .task:
+            return [
+                "complaint_implicit_failure",
+                "complaint_incident_diverse",
+                "complaint_request",
+                "complaint_statement",
+                "confirmation_decision",
+                "confirmation_selection_short",
+                "event_statement",
+                "follow_up_personal_reminder",
+                "neutral_fact",
+                "personal_action_item_boundary",
+                "resolved_issue_boundary",
+                "self_plan"
+            ]
+        case .invitation:
+            return [
+                "event_statement",
+                "schedule_negotiation",
+                "task_question",
+                "task_statement"
+            ]
+        case .complaint:
+            return [
+                "information_question",
+                "negative_news",
+                "neutral_fact",
+                "personal_action_item_boundary",
+                "positive_feedback",
+                "quoted_question",
+                "resolved_issue_boundary",
+                "self_plan",
+                "task_assignment_diverse",
+                "task_completion_boundary",
+                "task_indirect_assignment",
+                "task_indirect_question",
+                "task_statement"
+            ]
+        case .scheduleNegotiation:
+            if usesBaselineNegativePolicy {
+                return [
+                    "event_statement",
+                    "information_question",
+                    "invitation_question",
+                    "schedule_fixed_invitation_boundary",
+                    "task_question",
+                    "task_statement",
+                    "vague_future_boundary"
+                ]
+            }
+            return [
+                "event_statement",
+                "confirmation_decision",
+                "confirmation_selection_short",
+                "follow_up_action",
+                "follow_up_triggered",
+                "information_question",
+                "invitation_question",
+                "schedule_fixed_invitation_boundary",
+                "task_question",
+                "task_statement",
+                "vague_future_boundary"
+            ]
+        case .confirmationDecision:
+            return [
+                "acknowledgment_decision_boundary",
+                "event_statement",
+                "follow_up_action",
+                "follow_up_personal_reminder",
+                "follow_up_triggered",
+                "neutral_fact",
+                "negative_news",
+                "schedule_negotiation",
+                "task_assignment_diverse",
+                "task_statement",
+                "vague_future_boundary"
+            ]
+        case .followUpReminder:
+            return [
+                "acknowledgment",
+                "complaint_request",
+                "confirmation_decision",
+                "confirmation_selection_short",
+                "event_statement",
+                "invitation_question",
+                "neutral_fact",
+                "positive_feedback",
+                "schedule_negotiation",
+                "self_plan",
+                "task_assignment_diverse",
+                "task_question",
+                "task_statement",
+                "vague_future_boundary"
+            ]
+        case .blessing:
+            return [
+                "acknowledgment",
+                "blessing_boundary",
+                "conversational_message",
+                "event_statement",
+                "invitation_question",
+                "neutral_fact",
+                "positive_feedback",
+                "quoted_question",
+                "task_question",
+                "task_statement"
+            ]
+        case .question, .replyableMessage, .sentiment:
+            return []
+        }
+    }
+
+    var hardNegativeFraction: Double {
+        switch self {
+        case .task:
+            0.65
+        case .complaint, .blessing:
+            0.65
+        case .invitation, .followUpReminder:
+            0.50
+        case .scheduleNegotiation:
+            0.75
+        case .confirmationDecision:
+            0.90
+        case .question, .replyableMessage, .sentiment:
+            0
+        }
+    }
+
     func label(for record: CorpusRecord) -> String {
         switch self {
         case .task: record.task ? "task" : "notTask"
         case .question: record.question ? "question" : "notQuestion"
         case .invitation: record.invitation ? "invitation" : "notInvitation"
         case .complaint: record.complaint ? "complaint" : "notComplaint"
+        case .scheduleNegotiation:
+            record.scheduleNegotiation ? "scheduleNegotiation" : "notScheduleNegotiation"
+        case .confirmationDecision:
+            record.confirmationDecision ? "confirmationDecision" : "notConfirmationDecision"
+        case .followUpReminder:
+            record.followUpReminder ? "followUpReminder" : "notFollowUpReminder"
+        case .blessing:
+            record.blessing ? "blessing" : "notBlessing"
         case .replyableMessage: record.replyable ? "replyableMessage" : "notReplyableMessage"
         case .sentiment: record.sentiment
         }
     }
+
+    func hasKnownLabel(in record: CorpusRecord) -> Bool {
+        record.knownLabels?.contains(rawValue) ?? true
+    }
 }
 
 private struct SeededGenerator: RandomNumberGenerator {
@@ -195,14 +367,36 @@ private struct TrainedCandidate {
 
 private let fileManager = FileManager.default
 private let repositoryRoot = URL(fileURLWithPath: fileManager.currentDirectoryPath)
-private let corpusURL = repositoryRoot
-    .appendingPathComponent("ModelTraining/ClipboardSemantics/clipboard_semantic_corpus.jsonl")
-private let candidateDirectory = repositoryRoot
-    .appendingPathComponent("ModelTraining/ClipboardSemantics/Candidates")
-private let resourceDirectory = repositoryRoot
-    .appendingPathComponent("OSGKeyboardShared/Resources/ClipboardSemantics")
-private let reportURL = repositoryRoot
-    .appendingPathComponent("ModelTraining/ClipboardSemantics/evaluation-report.json")
+
+private func commandLineValue(after flag: String) -> String? {
+    guard let index = CommandLine.arguments.firstIndex(of: flag),
+          CommandLine.arguments.indices.contains(index + 1) else {
+        return nil
+    }
+    return CommandLine.arguments[index + 1]
+}
+
+private func resolvedURL(flag: String, defaultPath: String) -> URL {
+    let path = commandLineValue(after: flag) ?? defaultPath
+    return URL(fileURLWithPath: path, relativeTo: repositoryRoot).standardizedFileURL
+}
+
+private let corpusURL = resolvedURL(
+    flag: "--corpus",
+    defaultPath: "ModelTraining/ClipboardSemantics/clipboard_semantic_corpus.jsonl"
+)
+private let candidateDirectory = resolvedURL(
+    flag: "--candidate-directory",
+    defaultPath: "ModelTraining/ClipboardSemantics/Candidates"
+)
+private let resourceDirectory = resolvedURL(
+    flag: "--resource-directory",
+    defaultPath: "OSGKeyboardShared/Resources/ClipboardSemantics"
+)
+private let reportURL = resolvedURL(
+    flag: "--report",
+    defaultPath: "ModelTraining/ClipboardSemantics/evaluation-report.json"
+)
 
 private func loadCorpus() throws -> [CorpusRecord] {
     let content = try String(contentsOf: corpusURL, encoding: .utf8)
@@ -219,6 +413,85 @@ private func stableSeed(for classifier: ClassifierID, split: String) -> UInt64 {
     }
 }
 
+private func sourceBalancedPrefix(
+    _ records: [CorpusRecord],
+    limit: Int,
+    classifier: ClassifierID,
+    label: String
+) -> [CorpusRecord] {
+    guard records.count > limit else { return records }
+    var grouped = Dictionary(grouping: records) {
+        $0.sourceDataset ?? "generated"
+    }
+    for source in grouped.keys.sorted() {
+        var generator = SeededGenerator(
+            seed: stableSeed(
+                for: classifier,
+                split: "open|\(label)|\(source)"
+            )
+        )
+        grouped[source]?.shuffle(using: &generator)
+    }
+    let sources = grouped.keys.sorted()
+    var offsets = Dictionary(uniqueKeysWithValues: sources.map { ($0, 0) })
+    var selected: [CorpusRecord] = []
+    while selected.count < limit {
+        var addedRecord = false
+        for source in sources where selected.count < limit {
+            let offset = offsets[source] ?? 0
+            guard let values = grouped[source], values.indices.contains(offset) else {
+                continue
+            }
+            selected.append(values[offset])
+            offsets[source] = offset + 1
+            addedRecord = true
+        }
+        if !addedRecord {
+            break
+        }
+    }
+    return selected
+}
+
+private func curatedTrainingRecords(
+    _ records: [CorpusRecord],
+    classifier: ClassifierID
+) -> [CorpusRecord] {
+    let knownRecords = records.filter {
+        classifier.hasKnownLabel(in: $0)
+    }
+    let generatedRecords = knownRecords.filter { $0.sourceDataset == nil }
+    let openRecords = knownRecords.filter { $0.sourceDataset != nil }
+    guard !openRecords.isEmpty else { return generatedRecords }
+
+    let generatedByLabel = Dictionary(grouping: generatedRecords) {
+        classifier.label(for: $0)
+    }
+    let openByLabel = Dictionary(grouping: openRecords) {
+        classifier.label(for: $0)
+    }
+    let multiplier = switch classifier {
+    case .blessing:
+        2.0
+    case .task, .question, .complaint, .confirmationDecision, .sentiment:
+        1.0
+    case .invitation, .scheduleNegotiation, .followUpReminder, .replyableMessage:
+        0.5
+    }
+
+    let selectedOpenRecords = classifier.labels.flatMap { label in
+        let generatedCount = generatedByLabel[label]?.count ?? 0
+        let limit = max(1, Int((Double(generatedCount) * multiplier).rounded()))
+        return sourceBalancedPrefix(
+            openByLabel[label] ?? [],
+            limit: limit,
+            classifier: classifier,
+            label: label
+        )
+    }
+    return generatedRecords + selectedOpenRecords
+}
+
 private func balancedTexts(
     records: [CorpusRecord],
     classifier: ClassifierID,
@@ -236,10 +509,31 @@ private func balancedTexts(
         var generator = SeededGenerator(
             seed: stableSeed(for: classifier, split: split) &+ UInt64(offset)
         )
-        let texts = (grouped[label] ?? [])
-            .map(\.text)
-            .shuffled(using: &generator)
-        result[label] = Array(texts.prefix(minimumCount))
+        let candidates = grouped[label] ?? []
+        if label != classifier.positiveLabel,
+           !classifier.hardNegativeFamilies.isEmpty,
+           classifier.hardNegativeFraction > 0 {
+            var hardNegatives = candidates
+                .filter { classifier.hardNegativeFamilies.contains($0.family) }
+                .map(\.text)
+                .shuffled(using: &generator)
+            var remaining = candidates
+                .filter { !classifier.hardNegativeFamilies.contains($0.family) }
+                .map(\.text)
+                .shuffled(using: &generator)
+            let requestedHardNegatives = Int(
+                (Double(minimumCount) * classifier.hardNegativeFraction).rounded(.down)
+            )
+            let hardNegativeCount = min(hardNegatives.count, requestedHardNegatives)
+            hardNegatives = Array(hardNegatives.prefix(hardNegativeCount))
+            remaining = Array(remaining.prefix(minimumCount - hardNegativeCount))
+            result[label] = hardNegatives + remaining
+        } else {
+            let texts = candidates
+                .map(\.text)
+                .shuffled(using: &generator)
+            result[label] = Array(texts.prefix(minimumCount))
+        }
     }
     return result
 }
@@ -305,6 +599,57 @@ private func binaryMetrics(
     )
 }
 
+private func binaryMetrics(
+    records: [CorpusRecord],
+    classifier: ClassifierID,
+    positiveLabel: String,
+    globalThreshold: Double,
+    thresholdsByLanguage: [String: Double],
+    scores: [Double]
+) -> BinaryMetrics {
+    precondition(records.count == scores.count)
+    let predictions = zip(records, scores).map { record, score in
+        score >= (thresholdsByLanguage[record.language] ?? globalThreshold)
+    }
+    var truePositive = 0
+    var trueNegative = 0
+    var falsePositive = 0
+    var falseNegative = 0
+    for (record, predictedPositive) in zip(records, predictions) {
+        let expectedPositive = classifier.label(for: record) == positiveLabel
+        switch (expectedPositive, predictedPositive) {
+        case (true, true): truePositive += 1
+        case (false, false): trueNegative += 1
+        case (false, true): falsePositive += 1
+        case (true, false): falseNegative += 1
+        }
+    }
+    let total = records.count
+    let precision = truePositive + falsePositive > 0
+        ? Double(truePositive) / Double(truePositive + falsePositive)
+        : 0
+    let recall = truePositive + falseNegative > 0
+        ? Double(truePositive) / Double(truePositive + falseNegative)
+        : 0
+    return BinaryMetrics(
+        total: total,
+        truePositive: truePositive,
+        trueNegative: trueNegative,
+        falsePositive: falsePositive,
+        falseNegative: falseNegative,
+        accuracy: rounded(
+            total > 0 ? Double(truePositive + trueNegative) / Double(total) : 0
+        ),
+        precision: rounded(precision),
+        recall: rounded(recall),
+        f1: rounded(
+            precision + recall > 0
+                ? 2 * precision * recall / (precision + recall)
+                : 0
+        )
+    )
+}
+
 private func scores(
     classifier: MLTextClassifier,
     records: [CorpusRecord],
@@ -320,6 +665,7 @@ private func binaryErrorExamples(
     classifier: ClassifierID,
     positiveLabel: String,
     threshold: Double,
+    thresholdsByLanguage: [String: Double] = [:],
     scores: [Double],
     expectedPositive: Bool,
     predictedPositive: Bool,
@@ -327,7 +673,8 @@ private func binaryErrorExamples(
 ) -> [String] {
     zip(records, scores).compactMap { record, score -> String? in
         let isExpectedPositive = classifier.label(for: record) == positiveLabel
-        let isPredictedPositive = score >= threshold
+        let effectiveThreshold = thresholdsByLanguage[record.language] ?? threshold
+        let isPredictedPositive = score >= effectiveThreshold
         guard isExpectedPositive == expectedPositive,
               isPredictedPositive == predictedPositive else {
             return nil
@@ -347,7 +694,15 @@ private func calibratedThreshold(
     var candidates: [(Double, BinaryMetrics)] = []
     // Low-confidence positives are too unstable for automatic keyboard
     // routing even when a synthetic validation split happens to accept them.
-    for integer in 60...99 {
+    let minimumThreshold = switch classifierID {
+    case .scheduleNegotiation, .confirmationDecision:
+        30
+    case .followUpReminder:
+        58
+    default:
+        60
+    }
+    for integer in minimumThreshold...99 {
         let threshold = Double(integer) / 100
         candidates.append(
             (
@@ -366,6 +721,11 @@ private func calibratedThreshold(
     let highPrecision = candidates.filter { $0.1.precision >= 0.97 }
     if let best = highPrecision.max(by: {
         if $0.1.recall == $1.1.recall {
+            if $0.1.precision == $1.1.precision {
+                // Prefer the lowest threshold on an identical validation
+                // plateau so held-out paraphrases are not needlessly lost.
+                return $0.0 > $1.0
+            }
             return $0.1.precision < $1.1.precision
         }
         return $0.1.recall < $1.1.recall
@@ -382,6 +742,40 @@ private func calibratedThreshold(
         ))
 }
 
+private func calibratedThresholdsByLanguage(
+    records: [CorpusRecord],
+    classifierID: ClassifierID,
+    positiveLabel: String,
+    scores: [Double],
+    minimumPerClass: Int = 20
+) -> [String: Double] {
+    var result: [String: Double] = [:]
+    for language in Set(records.map(\.language)).sorted() {
+        let indexed = records.enumerated().filter { $0.element.language == language }
+        let languageRecords = indexed.map(\.element)
+        let positiveCount = languageRecords.filter {
+            classifierID.label(for: $0) == positiveLabel
+        }.count
+        let negativeCount = languageRecords.count - positiveCount
+        guard positiveCount >= minimumPerClass, negativeCount >= minimumPerClass else {
+            print(
+                "CALIBRATION_SKIPPED classifier=\(classifierID.rawValue) "
+                    + "language=\(language) positives=\(positiveCount) negatives=\(negativeCount)"
+            )
+            continue
+        }
+        let languageScores = indexed.map { scores[$0.offset] }
+        let calibration = calibratedThreshold(
+            records: languageRecords,
+            classifierID: classifierID,
+            positiveLabel: positiveLabel,
+            scores: languageScores
+        )
+        result[language] = rounded(calibration.threshold)
+    }
+    return result
+}
+
 private func multiclassMetrics(
     records: [CorpusRecord],
     classifierID: ClassifierID,
@@ -462,8 +856,16 @@ private func train(
     testRecords: [CorpusRecord],
     goldenRecords: [CorpusRecord]
 ) throws -> TrainedCandidate {
+    // Open datasets often annotate only a subset of product intents. Excluding
+    // unknown labels prevents an unannotated intent from becoming a false negative.
+    // Source-balanced caps then preserve the reviewed base corpus as the boundary
+    // anchor instead of allowing one large dataset to dominate model weights.
+    let knownTrainingRecords = curatedTrainingRecords(
+        trainingRecords,
+        classifier: classifierID
+    )
     let trainingTexts = balancedTexts(
-        records: trainingRecords,
+        records: knownTrainingRecords,
         classifier: classifierID,
         split: "train"
     )
@@ -524,6 +926,15 @@ private func train(
             positiveLabel: positiveLabel,
             scores: validationScores
         )
+        let calibratedLanguageThresholds = calibratedThresholdsByLanguage(
+            records: validationRecords,
+            classifierID: classifierID,
+            positiveLabel: positiveLabel,
+            scores: validationScores
+        )
+        let thresholdsByLanguage = calibratedLanguageThresholds.mapValues {
+            max($0, rounded(calibration.threshold))
+        }
         let testScores = try scores(
             classifier: classifier,
             records: testRecords,
@@ -533,7 +944,8 @@ private func train(
             records: testRecords,
             classifier: classifierID,
             positiveLabel: positiveLabel,
-            threshold: calibration.threshold,
+            globalThreshold: calibration.threshold,
+            thresholdsByLanguage: thresholdsByLanguage,
             scores: testScores
         )
         let goldenScores = try scores(
@@ -545,7 +957,8 @@ private func train(
             records: goldenRecords,
             classifier: classifierID,
             positiveLabel: positiveLabel,
-            threshold: calibration.threshold,
+            globalThreshold: calibration.threshold,
+            thresholdsByLanguage: thresholdsByLanguage,
             scores: goldenScores
         )
         var byLanguage: [String: BinaryMetrics] = [:]
@@ -557,10 +970,21 @@ private func train(
                 records: records,
                 classifier: classifierID,
                 positiveLabel: positiveLabel,
-                threshold: calibration.threshold,
+                threshold: thresholdsByLanguage[language] ?? calibration.threshold,
                 scores: languageScores
             )
         }
+        var goldenByLanguage: [String: BinaryMetrics] = [:]
+        for language in Set(goldenRecords.map(\.language)).sorted() {
+            let indexed = goldenRecords.enumerated().filter { $0.element.language == language }
+            goldenByLanguage[language] = binaryMetrics(
+                records: indexed.map(\.element),
+                classifier: classifierID,
+                positiveLabel: positiveLabel,
+                threshold: thresholdsByLanguage[language] ?? calibration.threshold,
+                scores: indexed.map { goldenScores[$0.offset] }
+            )
+        }
         report = CandidateReport(
             algorithm: algorithm.rawValue,
             modelBytes: modelFileSize(at: modelURL),
@@ -568,6 +992,8 @@ private func train(
             balancedTrainingCount: totalCount(trainingTexts),
             balancedValidationCount: totalCount(validationTexts),
             threshold: rounded(calibration.threshold),
+            confidenceThresholdsByLanguage:
+                thresholdsByLanguage.isEmpty ? nil : thresholdsByLanguage,
             acceptedForAutomaticRouting: algorithm == .maxEnt
                 && calibration.metrics.precision >= 0.97
                 && testMetrics.precision >= 0.90
@@ -580,6 +1006,7 @@ private func train(
                 classifier: classifierID,
                 positiveLabel: positiveLabel,
                 threshold: calibration.threshold,
+                thresholdsByLanguage: thresholdsByLanguage,
                 scores: testScores,
                 expectedPositive: false,
                 predictedPositive: true
@@ -589,6 +1016,7 @@ private func train(
                 classifier: classifierID,
                 positiveLabel: positiveLabel,
                 threshold: calibration.threshold,
+                thresholdsByLanguage: thresholdsByLanguage,
                 scores: testScores,
                 expectedPositive: true,
                 predictedPositive: false
@@ -598,6 +1026,7 @@ private func train(
                 classifier: classifierID,
                 positiveLabel: positiveLabel,
                 threshold: calibration.threshold,
+                thresholdsByLanguage: thresholdsByLanguage,
                 scores: goldenScores,
                 expectedPositive: false,
                 predictedPositive: true
@@ -607,11 +1036,13 @@ private func train(
                 classifier: classifierID,
                 positiveLabel: positiveLabel,
                 threshold: calibration.threshold,
+                thresholdsByLanguage: thresholdsByLanguage,
                 scores: goldenScores,
                 expectedPositive: true,
                 predictedPositive: false
             ),
             binaryByLanguage: byLanguage,
+            goldenBinaryByLanguage: goldenByLanguage,
             validationMulticlass: nil,
             testMulticlass: nil,
             goldenMulticlass: nil,
@@ -658,6 +1089,7 @@ private func train(
             balancedTrainingCount: totalCount(trainingTexts),
             balancedValidationCount: totalCount(validationTexts),
             threshold: nil,
+            confidenceThresholdsByLanguage: nil,
             acceptedForAutomaticRouting: algorithm == .maxEnt
                 && validationMetrics.macroF1 >= 0.85
                 && testMetrics.macroF1 >= 0.85
@@ -670,6 +1102,7 @@ private func train(
             goldenFalsePositiveExamples: nil,
             goldenFalseNegativeExamples: nil,
             binaryByLanguage: nil,
+            goldenBinaryByLanguage: nil,
             validationMulticlass: validationMetrics,
             testMulticlass: testMetrics,
             goldenMulticlass: goldenMetrics,
@@ -723,7 +1156,7 @@ private func selectedAlgorithms() -> [CandidateAlgorithm] {
     guard let index = CommandLine.arguments.firstIndex(of: "--algorithms"),
           CommandLine.arguments.indices.contains(index + 1)
     else {
-        return CandidateAlgorithm.allCases
+        return [.maxEnt]
     }
     let requested = Set(
         CommandLine.arguments[index + 1]
@@ -834,6 +1267,8 @@ private func main() throws {
                 labels: classifierID.labels,
                 positiveLabel: classifierID.positiveLabel,
                 confidenceThreshold: selected.report.threshold,
+                confidenceThresholdsByLanguage:
+                    selected.report.confidenceThresholdsByLanguage,
                 acceptedForAutomaticRouting: selected.report.acceptedForAutomaticRouting
             )
         )
@@ -845,14 +1280,18 @@ private func main() throws {
 
     let report = TrainingReport(
         generatedAt: generatedAt,
-        corpusPath: "ModelTraining/ClipboardSemantics/clipboard_semantic_corpus.jsonl",
+        corpusPath: corpusURL.path,
         corpusCount: records.count,
         trainingCount: trainingRecords.count,
         validationCount: validationRecords.count,
         testCount: testRecords.count,
         goldenCount: goldenRecords.count,
         selectionPolicy:
-            "Validation only: binary models require precision >= 0.97, then maximize recall; "
+            "Open records with unknown labels are excluded per classifier, and source-balanced "
+            + "caps anchor each label to the reviewed generated corpus size. "
+            + "Validation only: global and per-language binary thresholds require precision "
+            + ">= 0.97, then maximize recall; languages with fewer than 20 examples per class "
+            + "fall back to the global threshold. "
             + "sentiment prioritizes macro-F1. Automatic routing also requires a self-contained "
             + "maxEnt model because BERT embedding assets are not guaranteed in extensions. "
             + "Test and golden data gate deployment but never tune model weights.",
@@ -861,7 +1300,7 @@ private func main() throws {
     try writeJSON(report, to: reportURL)
     try writeJSON(
         ModelManifest(
-            schemaVersion: 1,
+            schemaVersion: 2,
             generatedAt: generatedAt,
             corpusRecordCount: records.count,
             classifiers: manifestClassifiers

Scripts/clipboard_semantics/train_verifiers.swift

@@ -0,0 +1,716 @@
+#!/usr/bin/env xcrun swift
+
+import CreateML
+import Foundation
+
+private struct BaseRecord: Decodable {
+    let id: String
+    let text: String
+    let language: String
+    let split: String
+    let task: Bool
+    let question: Bool
+    let invitation: Bool
+    let complaint: Bool
+    let scheduleNegotiation: Bool
+    let confirmationDecision: Bool
+    let followUpReminder: Bool
+}
+
+private struct SilverRecord: Decodable {
+    let id: String
+    let text: String
+    let language: String
+    let split: String
+    let actionVerifierLabel: String
+    let coordinationVerifierLabel: String
+    let sourceDataset: String?
+}
+
+private struct VerifierExample {
+    let id: String
+    let text: String
+    let language: String
+    let split: String
+    let label: String
+    let sourceDataset: String?
+}
+
+private struct RoutingMetrics: Encodable {
+    let total: Int
+    let expectedSpecialized: Int
+    let routed: Int
+    let correctRouted: Int
+    let falseRouted: Int
+    let missedSpecialized: Int
+    let precision: Double
+    let recall: Double
+    let f1: Double
+    let routedByLabel: [String: Int]
+    let correctByLabel: [String: Int]
+}
+
+private struct ThresholdSelection: Encodable {
+    let confidenceThreshold: Double
+    let minimumMargin: Double
+    let metrics: RoutingMetrics
+}
+
+private struct GateResult: Encodable {
+    let accepted: Bool
+    let reason: String
+    let minimumPredictedPositivesPerLabelAndLanguage: Int
+    let supportByLabelAndLanguage: [String: Int]
+    let precisionByLabelAndLanguage: [String: Double]
+    let maximumSourceShareByLabelAndLanguage: [String: Double]
+}
+
+private struct VerifierReport: Encodable {
+    let id: String
+    let labels: [String]
+    let modelFile: String
+    let modelBytes: Int
+    let trainingCount: Int
+    let calibrationCount: Int
+    let thresholdCalibrationCount: Int
+    let acceptanceCount: Int
+    let threshold: ThresholdSelection
+    let thresholdsByLanguage: [String: ThresholdSelection]
+    let calibrationMetrics: RoutingMetrics
+    let acceptanceMetrics: RoutingMetrics
+    let acceptanceByLanguage: [String: RoutingMetrics]
+    let gate: GateResult
+}
+
+private struct TrainingReport: Encodable {
+    let generatedAt: String
+    let baseCorpusPath: String
+    let silverDirectoryPath: String
+    let selectionPolicy: String
+    let verifiers: [VerifierReport]
+}
+
+private struct Observation {
+    let example: VerifierExample
+    let predictedLabel: String
+    let confidence: Double
+    let margin: Double
+}
+
+private enum VerifierID: String, CaseIterable {
+    case action
+    case coordination
+
+    var labels: [String] {
+        switch self {
+        case .action:
+            ["taskOnly", "complaintOnly", "both", "questionRequest", "neither"]
+        case .coordination:
+            [
+                "invitation",
+                "scheduleNegotiation",
+                "confirmationDecision",
+                "followUpReminder",
+                "neither"
+            ]
+        }
+    }
+
+    var modelFile: String {
+        switch self {
+        case .action: "ActionIntentVerifier.mlmodel"
+        case .coordination: "CoordinationIntentVerifier.mlmodel"
+        }
+    }
+
+    func baseLabel(for record: BaseRecord) -> String? {
+        switch self {
+        case .action:
+            if record.task && record.complaint {
+                return "both"
+            }
+            if record.task {
+                return "taskOnly"
+            }
+            if record.complaint {
+                return "complaintOnly"
+            }
+            if record.question {
+                return "questionRequest"
+            }
+            return "neither"
+        case .coordination:
+            let matches = [
+                record.invitation ? "invitation" : nil,
+                record.scheduleNegotiation ? "scheduleNegotiation" : nil,
+                record.confirmationDecision ? "confirmationDecision" : nil,
+                record.followUpReminder ? "followUpReminder" : nil
+            ].compactMap { $0 }
+            guard matches.count <= 1 else { return nil }
+            return matches.first ?? "neither"
+        }
+    }
+
+    func silverLabel(for record: SilverRecord) -> String {
+        switch self {
+        case .action: record.actionVerifierLabel
+        case .coordination: record.coordinationVerifierLabel
+        }
+    }
+}
+
+private struct SeededGenerator: RandomNumberGenerator {
+    private var state: UInt64
+
+    init(seed: UInt64) {
+        state = seed
+    }
+
+    mutating func next() -> UInt64 {
+        state &+= 0x9E37_79B9_7F4A_7C15
+        var value = state
+        value = (value ^ (value >> 30)) &* 0xBF58_476D_1CE4_E5B9
+        value = (value ^ (value >> 27)) &* 0x94D0_49BB_1331_11EB
+        return value ^ (value >> 31)
+    }
+}
+
+private let fileManager = FileManager.default
+private let root = URL(fileURLWithPath: fileManager.currentDirectoryPath)
+
+private func argumentValue(after flag: String) -> String? {
+    guard let index = CommandLine.arguments.firstIndex(of: flag),
+          CommandLine.arguments.indices.contains(index + 1) else {
+        return nil
+    }
+    return CommandLine.arguments[index + 1]
+}
+
+private func resolvedURL(flag: String, defaultPath: String) -> URL {
+    URL(
+        fileURLWithPath: argumentValue(after: flag) ?? defaultPath,
+        relativeTo: root
+    ).standardizedFileURL
+}
+
+private let baseCorpusURL = resolvedURL(
+    flag: "--base-corpus",
+    defaultPath: "ModelTraining/ClipboardSemantics/clipboard_semantic_corpus.jsonl"
+)
+private let silverDirectoryURL = resolvedURL(
+    flag: "--silver-directory",
+    defaultPath: "ModelTraining/ClipboardSemantics/Consensus"
+)
+private let baseManifestURL = resolvedURL(
+    flag: "--base-manifest",
+    defaultPath:
+        "OSGKeyboardShared/Resources/ClipboardSemantics/clipboard-semantic-models.json"
+)
+private let outputDirectoryURL = resolvedURL(
+    flag: "--output-directory",
+    defaultPath: "ModelTraining/ClipboardSemantics/VerifierCandidates"
+)
+private let reportURL = resolvedURL(
+    flag: "--report",
+    defaultPath: "ModelTraining/ClipboardSemantics/verifier-training-report.json"
+)
+
+private func readJSONLines<T: Decodable>(_ type: T.Type, from url: URL) throws -> [T] {
+    let content = try String(contentsOf: url, encoding: .utf8)
+    let decoder = JSONDecoder()
+    return try content.split(separator: "\n").map {
+        try decoder.decode(type, from: Data($0.utf8))
+    }
+}
+
+private func rounded(_ value: Double) -> Double {
+    guard value.isFinite else { return 0 }
+    return (value * 10_000).rounded() / 10_000
+}
+
+private func stableSeed(_ value: String) -> UInt64 {
+    value.utf8.reduce(0xcbf2_9ce4_8422_2325) { partial, byte in
+        (partial ^ UInt64(byte)) &* 0x0000_0100_0000_01B3
+    }
+}
+
+private func examples(
+    verifier: VerifierID,
+    baseRecords: [BaseRecord],
+    silverRecords: [SilverRecord],
+    baseSplit: String,
+    silverSplit: String
+) -> [VerifierExample] {
+    let base = baseRecords.compactMap { record -> VerifierExample? in
+        guard record.split == baseSplit,
+              let label = verifier.baseLabel(for: record) else {
+            return nil
+        }
+        return VerifierExample(
+            id: record.id,
+            text: record.text,
+            language: record.language,
+            split: baseSplit,
+            label: label,
+            sourceDataset: nil
+        )
+    }
+    let silver = silverRecords.compactMap { record -> VerifierExample? in
+        guard record.split == silverSplit else { return nil }
+        return VerifierExample(
+            id: record.id,
+            text: record.text,
+            language: record.language,
+            split: silverSplit,
+            label: verifier.silverLabel(for: record),
+            sourceDataset: record.sourceDataset
+        )
+    }
+    return base + silver
+}
+
+private func balancedTexts(
+    examples: [VerifierExample],
+    verifier: VerifierID,
+    split: String
+) -> [String: [String]] {
+    var grouped = Dictionary(grouping: examples, by: \.label)
+    if verifier == .action {
+        let taskExamples = grouped["taskOnly"] ?? []
+        let complaintExamples = grouped["complaintOnly"] ?? []
+        let targetCount = min(taskExamples.count, complaintExamples.count)
+        var bothExamples = grouped["both"] ?? []
+        if !taskExamples.isEmpty, !complaintExamples.isEmpty {
+            for index in bothExamples.count..<targetCount {
+                let task = taskExamples[index % taskExamples.count]
+                let complaint = complaintExamples[
+                    Int(
+                        stableSeed("\(split)|both|\(index)")
+                            % UInt64(complaintExamples.count)
+                    )
+                ]
+                bothExamples.append(
+                    VerifierExample(
+                        id: "composed-both-\(split)-\(index)",
+                        text: "\(complaint.text)\n\(task.text)",
+                        language: task.language,
+                        split: split,
+                        label: "both",
+                        sourceDataset: "generated-composition"
+                    )
+                )
+            }
+            grouped["both"] = bothExamples
+        }
+    }
+    let minimumCount = verifier.labels.compactMap { grouped[$0]?.count }.min() ?? 0
+    precondition(minimumCount > 0, "Missing \(verifier.rawValue) label in \(split)")
+    return Dictionary(uniqueKeysWithValues: verifier.labels.enumerated().map { offset, label in
+        var generator = SeededGenerator(
+            seed: stableSeed("\(verifier.rawValue)|\(split)|\(label)|\(offset)")
+        )
+        let texts = (grouped[label] ?? []).map(\.text).shuffled(using: &generator)
+        return (label, Array(texts.prefix(minimumCount)))
+    })
+}
+
+private func observations(
+    model: MLTextClassifier,
+    examples: [VerifierExample]
+) throws -> [Observation] {
+    try examples.map { example in
+        let hypotheses = try model.predictionWithConfidence(from: example.text)
+            .sorted { $0.value > $1.value }
+        let winner = hypotheses.first ?? (key: "neither", value: 0)
+        let runnerUp = hypotheses.dropFirst().first?.value ?? 0
+        return Observation(
+            example: example,
+            predictedLabel: winner.key,
+            confidence: winner.value,
+            margin: winner.value - runnerUp
+        )
+    }
+}
+
+private func routingMetrics(
+    observations: [Observation],
+    confidenceThreshold: Double,
+    minimumMargin: Double
+) -> RoutingMetrics {
+    var routed = 0
+    var correctRouted = 0
+    var expectedSpecialized = 0
+    var missedSpecialized = 0
+    var routedByLabel: [String: Int] = [:]
+    var correctByLabel: [String: Int] = [:]
+    for observation in observations {
+        let expectedIsSpecialized = observation.example.label != "neither"
+        if expectedIsSpecialized {
+            expectedSpecialized += 1
+        }
+        let shouldRoute = observation.predictedLabel != "neither"
+            && observation.confidence >= confidenceThreshold
+            && observation.margin >= minimumMargin
+        if shouldRoute {
+            routed += 1
+            routedByLabel[observation.predictedLabel, default: 0] += 1
+            if observation.predictedLabel == observation.example.label {
+                correctRouted += 1
+                correctByLabel[observation.predictedLabel, default: 0] += 1
+            }
+        } else if expectedIsSpecialized {
+            missedSpecialized += 1
+        }
+    }
+    let precision = routed > 0 ? Double(correctRouted) / Double(routed) : 0
+    let recall = expectedSpecialized > 0
+        ? Double(correctRouted) / Double(expectedSpecialized)
+        : 0
+    return RoutingMetrics(
+        total: observations.count,
+        expectedSpecialized: expectedSpecialized,
+        routed: routed,
+        correctRouted: correctRouted,
+        falseRouted: routed - correctRouted,
+        missedSpecialized: missedSpecialized,
+        precision: rounded(precision),
+        recall: rounded(recall),
+        f1: rounded(
+            precision + recall > 0
+                ? 2 * precision * recall / (precision + recall)
+                : 0
+        ),
+        routedByLabel: routedByLabel,
+        correctByLabel: correctByLabel
+    )
+}
+
+private func selectedThreshold(
+    observations: [Observation]
+) -> ThresholdSelection {
+    var selections: [ThresholdSelection] = []
+    for confidenceStep in 50...99 {
+        for marginStep in 0...10 {
+            let confidence = Double(confidenceStep) / 100
+            let margin = Double(marginStep) / 20
+            let metrics = routingMetrics(
+                observations: observations,
+                confidenceThreshold: confidence,
+                minimumMargin: margin
+            )
+            if metrics.precision >= 0.95 {
+                selections.append(
+                    ThresholdSelection(
+                        confidenceThreshold: confidence,
+                        minimumMargin: margin,
+                        metrics: metrics
+                    )
+                )
+            }
+        }
+    }
+    return selections.max {
+        if $0.metrics.recall != $1.metrics.recall {
+            return $0.metrics.recall < $1.metrics.recall
+        }
+        if $0.metrics.routed != $1.metrics.routed {
+            return $0.metrics.routed < $1.metrics.routed
+        }
+        if $0.confidenceThreshold != $1.confidenceThreshold {
+            return $0.confidenceThreshold > $1.confidenceThreshold
+        }
+        return $0.minimumMargin > $1.minimumMargin
+    } ?? ThresholdSelection(
+        confidenceThreshold: 1,
+        minimumMargin: 1,
+        metrics: routingMetrics(
+            observations: observations,
+            confidenceThreshold: 1,
+            minimumMargin: 1
+        )
+    )
+}
+
+private func gateResult(
+    verifier: VerifierID,
+    observations: [Observation],
+    thresholdsByLanguage: [String: ThresholdSelection],
+    globalThreshold: ThresholdSelection
+) -> GateResult {
+    var support: [String: Int] = [:]
+    var correct: [String: Int] = [:]
+    var sourceSupport: [String: [String: Int]] = [:]
+    for observation in observations {
+        let selection = thresholdsByLanguage[observation.example.language]
+            ?? globalThreshold
+        let routed = observation.predictedLabel != "neither"
+            && observation.confidence >= selection.confidenceThreshold
+            && observation.margin >= selection.minimumMargin
+        guard routed else { continue }
+        let key = "\(observation.example.language)|\(observation.predictedLabel)"
+        support[key, default: 0] += 1
+        let source = observation.example.sourceDataset ?? "generated-base"
+        sourceSupport[key, default: [:]][source, default: 0] += 1
+        if observation.predictedLabel == observation.example.label {
+            correct[key, default: 0] += 1
+        }
+    }
+    let precision = Dictionary(uniqueKeysWithValues: support.map { key, count in
+        (
+            key,
+            rounded(Double(correct[key] ?? 0) / Double(max(count, 1)))
+        )
+    })
+    let maximumSourceShare = Dictionary(uniqueKeysWithValues: support.map { key, count in
+        let maximum = sourceSupport[key]?.values.max() ?? 0
+        return (key, rounded(Double(maximum) / Double(max(count, 1))))
+    })
+    let requiredKeys = ["en", "zh-Hans"].flatMap { language in
+        verifier.labels
+            .filter { $0 != "neither" }
+            .map { "\(language)|\($0)" }
+    }
+    let accepted = requiredKeys.allSatisfy {
+        support[$0, default: 0] >= 100
+            && precision[$0, default: 0] >= 0.95
+            && maximumSourceShare[$0, default: 1] <= 0.65
+    }
+    return GateResult(
+        accepted: accepted,
+        reason: accepted
+            ? "Every routed label/language meets precision, support, and source-diversity gates."
+            : "Shadow only: at least one label/language misses precision, support, or source-diversity gates.",
+        minimumPredictedPositivesPerLabelAndLanguage: 100,
+        supportByLabelAndLanguage: support,
+        precisionByLabelAndLanguage: precision,
+        maximumSourceShareByLabelAndLanguage: maximumSourceShare
+    )
+}
+
+private func writeJSON<T: Encodable>(_ value: T, to url: URL) throws {
+    let encoder = JSONEncoder()
+    encoder.outputFormatting = [.prettyPrinted, .sortedKeys, .withoutEscapingSlashes]
+    try encoder.encode(value).write(to: url, options: .atomic)
+}
+
+private func train(
+    verifier: VerifierID,
+    baseRecords: [BaseRecord],
+    silverRecords: [SilverRecord]
+) throws -> (VerifierReport, [String: Any]) {
+    let training = examples(
+        verifier: verifier,
+        baseRecords: baseRecords,
+        silverRecords: silverRecords,
+        baseSplit: "train",
+        silverSplit: "silverTrain"
+    )
+    let calibration = examples(
+        verifier: verifier,
+        baseRecords: baseRecords,
+        silverRecords: silverRecords,
+        baseSplit: "validation",
+        silverSplit: "silverCalibration"
+    )
+    let acceptance = examples(
+        verifier: verifier,
+        baseRecords: baseRecords,
+        silverRecords: silverRecords,
+        baseSplit: "golden",
+        silverSplit: "silverAcceptance"
+    )
+    let trainingTexts = balancedTexts(
+        examples: training,
+        verifier: verifier,
+        split: "train"
+    )
+    let calibrationTexts = balancedTexts(
+        examples: calibration,
+        verifier: verifier,
+        split: "calibration"
+    )
+    let parameters = MLTextClassifier.ModelParameters(
+        validation: .dictionary(calibrationTexts),
+        algorithm: .maxEnt(revision: 1)
+    )
+    let model = try MLTextClassifier(
+        trainingData: trainingTexts,
+        parameters: parameters
+    )
+    let modelURL = outputDirectoryURL.appendingPathComponent(verifier.modelFile)
+    try model.write(to: modelURL)
+    let calibrationObservations = try observations(model: model, examples: calibration)
+    let acceptanceObservations = try observations(model: model, examples: acceptance)
+    let externalCalibrationObservations = calibrationObservations.filter {
+        $0.example.sourceDataset != nil
+    }
+    let thresholdObservations = externalCalibrationObservations.count >= 40
+        ? externalCalibrationObservations
+        : calibrationObservations
+    let globalThreshold = selectedThreshold(observations: thresholdObservations)
+    let languages = Set(calibration.map(\.language)).sorted()
+    let thresholdsByLanguage = Dictionary(uniqueKeysWithValues: languages.map { language in
+        let externalLanguageObservations = thresholdObservations.filter {
+            $0.example.language == language
+        }
+        let languageObservations = externalLanguageObservations.count >= 15
+            ? externalLanguageObservations
+            : calibrationObservations.filter {
+                $0.example.language == language
+            }
+        return (
+            language,
+            selectedThreshold(
+                observations: languageObservations
+            )
+        )
+    })
+    let finalAcceptanceMetrics = routingMetrics(
+        observations: acceptanceObservations,
+        confidenceThreshold: globalThreshold.confidenceThreshold,
+        minimumMargin: globalThreshold.minimumMargin
+    )
+    let acceptanceByLanguage: [String: RoutingMetrics] = Dictionary(
+        uniqueKeysWithValues: languages.map { language in
+            let threshold = thresholdsByLanguage[language] ?? globalThreshold
+            return (
+                language,
+                routingMetrics(
+                    observations: acceptanceObservations.filter {
+                        $0.example.language == language
+                    },
+                    confidenceThreshold: threshold.confidenceThreshold,
+                    minimumMargin: threshold.minimumMargin
+                )
+            )
+        }
+    )
+    let gate = gateResult(
+        verifier: verifier,
+        observations: acceptanceObservations,
+        thresholdsByLanguage: thresholdsByLanguage,
+        globalThreshold: globalThreshold
+    )
+    let modelBytes = (
+        try fileManager.attributesOfItem(atPath: modelURL.path)[.size] as? NSNumber
+    )?.intValue ?? 0
+    let report = VerifierReport(
+        id: verifier.rawValue,
+        labels: verifier.labels,
+        modelFile: verifier.modelFile,
+        modelBytes: modelBytes,
+        trainingCount: trainingTexts.values.reduce(0) { $0 + $1.count },
+        calibrationCount: calibration.count,
+        thresholdCalibrationCount: thresholdObservations.count,
+        acceptanceCount: acceptance.count,
+        threshold: globalThreshold,
+        thresholdsByLanguage: thresholdsByLanguage,
+        calibrationMetrics: globalThreshold.metrics,
+        acceptanceMetrics: finalAcceptanceMetrics,
+        acceptanceByLanguage: acceptanceByLanguage,
+        gate: gate
+    )
+    let configuration: [String: Any] = [
+        "id": verifier.rawValue,
+        "modelFile": verifier.modelFile,
+        "labels": verifier.labels,
+        "confidenceThreshold": globalThreshold.confidenceThreshold,
+        "confidenceThresholdsByLanguage": thresholdsByLanguage.mapValues {
+            $0.confidenceThreshold
+        },
+        "minimumMargin": globalThreshold.minimumMargin,
+        "minimumMarginsByLanguage": thresholdsByLanguage.mapValues {
+            $0.minimumMargin
+        },
+        "acceptedForAutomaticRouting": gate.accepted,
+        "deploymentMode": gate.accepted ? "automatic" : "shadow"
+    ]
+    return (report, configuration)
+}
+
+private func main() throws {
+    try fileManager.createDirectory(
+        at: outputDirectoryURL,
+        withIntermediateDirectories: true
+    )
+    let baseRecords = try readJSONLines(BaseRecord.self, from: baseCorpusURL)
+    let silverURLs = [
+        silverDirectoryURL.appendingPathComponent("silver-train.jsonl"),
+        silverDirectoryURL.appendingPathComponent("silver-calibration.jsonl"),
+        silverDirectoryURL.appendingPathComponent("silver-acceptance.jsonl")
+    ]
+    let silverRecords = try silverURLs.flatMap {
+        try readJSONLines(SilverRecord.self, from: $0)
+    }
+    var reports: [VerifierReport] = []
+    var configurations: [[String: Any]] = []
+    for verifier in VerifierID.allCases {
+        let (report, configuration) = try train(
+            verifier: verifier,
+            baseRecords: baseRecords,
+            silverRecords: silverRecords
+        )
+        reports.append(report)
+        configurations.append(configuration)
+        print(
+            "VERIFIER_SELECTED id=\(verifier.rawValue) "
+                + "accepted=\(report.gate.accepted) "
+                + "precision=\(report.acceptanceMetrics.precision)"
+        )
+    }
+
+    let generatedAt = ISO8601DateFormatter().string(from: Date())
+    let report = TrainingReport(
+        generatedAt: generatedAt,
+        baseCorpusPath: baseCorpusURL.path,
+        silverDirectoryPath: silverDirectoryURL.path,
+        selectionPolicy:
+            "Thresholds and top-1/top-2 margins are calibrated without acceptance data. "
+            + "Automatic routing requires >=95% exact-route precision and >=100 routed "
+            + "positives for every specialized label in both English and Simplified Chinese. "
+            + "Without human gold, passing values remain consensus-relative.",
+        verifiers: reports
+    )
+    try writeJSON(report, to: reportURL)
+
+    let manifestData = try Data(contentsOf: baseManifestURL)
+    guard var manifest = try JSONSerialization.jsonObject(
+        with: manifestData
+    ) as? [String: Any] else {
+        throw NSError(
+            domain: "VerifierTraining",
+            code: 1,
+            userInfo: [NSLocalizedDescriptionKey: "Invalid base manifest"]
+        )
+    }
+    for classifier in manifest["classifiers"] as? [[String: Any]] ?? [] {
+        guard let modelFile = classifier["modelFile"] as? String else { continue }
+        let source = baseManifestURL.deletingLastPathComponent()
+            .appendingPathComponent(modelFile)
+        let destination = outputDirectoryURL.appendingPathComponent(modelFile)
+        if fileManager.fileExists(atPath: destination.path) {
+            try fileManager.removeItem(at: destination)
+        }
+        try fileManager.copyItem(at: source, to: destination)
+    }
+    manifest["schemaVersion"] = 3
+    manifest["verifiers"] = configurations
+    manifest["verifierGeneratedAt"] = generatedAt
+    manifest["verifierLabelPolicy"] = "multi-model-consensus-without-human-gold"
+    let outputManifestURL = outputDirectoryURL.appendingPathComponent(
+        "clipboard-semantic-models.json"
+    )
+    let outputManifestData = try JSONSerialization.data(
+        withJSONObject: manifest,
+        options: [.prettyPrinted, .sortedKeys, .withoutEscapingSlashes]
+    )
+    try outputManifestData.write(to: outputManifestURL, options: .atomic)
+    print("VERIFIER_REPORT \(reportURL.path)")
+}
+
+do {
+    try main()
+} catch {
+    fputs("Verifier training failed: \(error)\n", stderr)
+    exit(1)
+}