OSGKeyboard/Scripts/clipboard_semantics/train_verifiers.swift

#!/usr/bin/env xcrun swift

import CreateML
import Foundation

private struct BaseRecord: Decodable {
    let id: String
    let text: String
    let language: String
    let split: String
    let task: Bool
    let question: Bool
    let invitation: Bool
    let complaint: Bool
    let scheduleNegotiation: Bool
    let confirmationDecision: Bool
    let followUpReminder: Bool
}

private struct SilverRecord: Decodable {
    let id: String
    let text: String
    let language: String
    let split: String
    let actionVerifierLabel: String
    let coordinationVerifierLabel: String
    let sourceDataset: String?
}

private struct VerifierExample {
    let id: String
    let text: String
    let language: String
    let split: String
    let label: String
    let sourceDataset: String?
}

private struct RoutingMetrics: Encodable {
    let total: Int
    let expectedSpecialized: Int
    let routed: Int
    let correctRouted: Int
    let falseRouted: Int
    let missedSpecialized: Int
    let precision: Double
    let recall: Double
    let f1: Double
    let routedByLabel: [String: Int]
    let correctByLabel: [String: Int]
}

private struct ThresholdSelection: Encodable {
    let confidenceThreshold: Double
    let minimumMargin: Double
    let metrics: RoutingMetrics
}

private struct GateResult: Encodable {
    let accepted: Bool
    let reason: String
    let minimumPredictedPositivesPerLabelAndLanguage: Int
    let supportByLabelAndLanguage: [String: Int]
    let precisionByLabelAndLanguage: [String: Double]
    let maximumSourceShareByLabelAndLanguage: [String: Double]
}

private struct VerifierReport: Encodable {
    let id: String
    let labels: [String]
    let modelFile: String
    let modelBytes: Int
    let trainingCount: Int
    let calibrationCount: Int
    let thresholdCalibrationCount: Int
    let acceptanceCount: Int
    let threshold: ThresholdSelection
    let thresholdsByLanguage: [String: ThresholdSelection]
    let calibrationMetrics: RoutingMetrics
    let acceptanceMetrics: RoutingMetrics
    let acceptanceByLanguage: [String: RoutingMetrics]
    let gate: GateResult
}

private struct TrainingReport: Encodable {
    let generatedAt: String
    let baseCorpusPath: String
    let silverDirectoryPath: String
    let selectionPolicy: String
    let verifiers: [VerifierReport]
}

private struct Observation {
    let example: VerifierExample
    let predictedLabel: String
    let confidence: Double
    let margin: Double
}

private enum VerifierID: String, CaseIterable {
    case action
    case coordination

    var labels: [String] {
        switch self {
        case .action:
            ["taskOnly", "complaintOnly", "both", "questionRequest", "neither"]
        case .coordination:
            [
                "invitation",
                "scheduleNegotiation",
                "confirmationDecision",
                "followUpReminder",
                "neither"
            ]
        }
    }

    var modelFile: String {
        switch self {
        case .action: "ActionIntentVerifier.mlmodel"
        case .coordination: "CoordinationIntentVerifier.mlmodel"
        }
    }

    func baseLabel(for record: BaseRecord) -> String? {
        switch self {
        case .action:
            if record.task && record.complaint {
                return "both"
            }
            if record.task {
                return "taskOnly"
            }
            if record.complaint {
                return "complaintOnly"
            }
            if record.question {
                return "questionRequest"
            }
            return "neither"
        case .coordination:
            let matches = [
                record.invitation ? "invitation" : nil,
                record.scheduleNegotiation ? "scheduleNegotiation" : nil,
                record.confirmationDecision ? "confirmationDecision" : nil,
                record.followUpReminder ? "followUpReminder" : nil
            ].compactMap { $0 }
            guard matches.count <= 1 else { return nil }
            return matches.first ?? "neither"
        }
    }

    func silverLabel(for record: SilverRecord) -> String {
        switch self {
        case .action: record.actionVerifierLabel
        case .coordination: record.coordinationVerifierLabel
        }
    }
}

private struct SeededGenerator: RandomNumberGenerator {
    private var state: UInt64

    init(seed: UInt64) {
        state = seed
    }

    mutating func next() -> UInt64 {
        state &+= 0x9E37_79B9_7F4A_7C15
        var value = state
        value = (value ^ (value >> 30)) &* 0xBF58_476D_1CE4_E5B9
        value = (value ^ (value >> 27)) &* 0x94D0_49BB_1331_11EB
        return value ^ (value >> 31)
    }
}

private let fileManager = FileManager.default
private let root = URL(fileURLWithPath: fileManager.currentDirectoryPath)

private func argumentValue(after flag: String) -> String? {
    guard let index = CommandLine.arguments.firstIndex(of: flag),
          CommandLine.arguments.indices.contains(index + 1) else {
        return nil
    }
    return CommandLine.arguments[index + 1]
}

private func resolvedURL(flag: String, defaultPath: String) -> URL {
    URL(
        fileURLWithPath: argumentValue(after: flag) ?? defaultPath,
        relativeTo: root
    ).standardizedFileURL
}

private let baseCorpusURL = resolvedURL(
    flag: "--base-corpus",
    defaultPath: "ModelTraining/ClipboardSemantics/clipboard_semantic_corpus.jsonl"
)
private let silverDirectoryURL = resolvedURL(
    flag: "--silver-directory",
    defaultPath: "ModelTraining/ClipboardSemantics/Consensus"
)
private let baseManifestURL = resolvedURL(
    flag: "--base-manifest",
    defaultPath:
        "OSGKeyboardShared/Resources/ClipboardSemantics/clipboard-semantic-models.json"
)
private let outputDirectoryURL = resolvedURL(
    flag: "--output-directory",
    defaultPath: "ModelTraining/ClipboardSemantics/VerifierCandidates"
)
private let reportURL = resolvedURL(
    flag: "--report",
    defaultPath: "ModelTraining/ClipboardSemantics/verifier-training-report.json"
)

private func readJSONLines<T: Decodable>(_ type: T.Type, from url: URL) throws -> [T] {
    let content = try String(contentsOf: url, encoding: .utf8)
    let decoder = JSONDecoder()
    return try content.split(separator: "\n").map {
        try decoder.decode(type, from: Data($0.utf8))
    }
}

private func rounded(_ value: Double) -> Double {
    guard value.isFinite else { return 0 }
    return (value * 10_000).rounded() / 10_000
}

private func stableSeed(_ value: String) -> UInt64 {
    value.utf8.reduce(0xcbf2_9ce4_8422_2325) { partial, byte in
        (partial ^ UInt64(byte)) &* 0x0000_0100_0000_01B3
    }
}

private func examples(
    verifier: VerifierID,
    baseRecords: [BaseRecord],
    silverRecords: [SilverRecord],
    baseSplit: String,
    silverSplit: String
) -> [VerifierExample] {
    let base = baseRecords.compactMap { record -> VerifierExample? in
        guard record.split == baseSplit,
              let label = verifier.baseLabel(for: record) else {
            return nil
        }
        return VerifierExample(
            id: record.id,
            text: record.text,
            language: record.language,
            split: baseSplit,
            label: label,
            sourceDataset: nil
        )
    }
    let silver = silverRecords.compactMap { record -> VerifierExample? in
        guard record.split == silverSplit else { return nil }
        return VerifierExample(
            id: record.id,
            text: record.text,
            language: record.language,
            split: silverSplit,
            label: verifier.silverLabel(for: record),
            sourceDataset: record.sourceDataset
        )
    }
    return base + silver
}

private func balancedTexts(
    examples: [VerifierExample],
    verifier: VerifierID,
    split: String
) -> [String: [String]] {
    var grouped = Dictionary(grouping: examples, by: \.label)
    if verifier == .action {
        let taskExamples = grouped["taskOnly"] ?? []
        let complaintExamples = grouped["complaintOnly"] ?? []
        let targetCount = min(taskExamples.count, complaintExamples.count)
        var bothExamples = grouped["both"] ?? []
        if !taskExamples.isEmpty, !complaintExamples.isEmpty {
            for index in bothExamples.count..<targetCount {
                let task = taskExamples[index % taskExamples.count]
                let complaint = complaintExamples[
                    Int(
                        stableSeed("\(split)|both|\(index)")
                            % UInt64(complaintExamples.count)
                    )
                ]
                bothExamples.append(
                    VerifierExample(
                        id: "composed-both-\(split)-\(index)",
                        text: "\(complaint.text)\n\(task.text)",
                        language: task.language,
                        split: split,
                        label: "both",
                        sourceDataset: "generated-composition"
                    )
                )
            }
            grouped["both"] = bothExamples
        }
    }
    let minimumCount = verifier.labels.compactMap { grouped[$0]?.count }.min() ?? 0
    precondition(minimumCount > 0, "Missing \(verifier.rawValue) label in \(split)")
    return Dictionary(uniqueKeysWithValues: verifier.labels.enumerated().map { offset, label in
        var generator = SeededGenerator(
            seed: stableSeed("\(verifier.rawValue)|\(split)|\(label)|\(offset)")
        )
        let texts = (grouped[label] ?? []).map(\.text).shuffled(using: &generator)
        return (label, Array(texts.prefix(minimumCount)))
    })
}

private func observations(
    model: MLTextClassifier,
    examples: [VerifierExample]
) throws -> [Observation] {
    try examples.map { example in
        let hypotheses = try model.predictionWithConfidence(from: example.text)
            .sorted { $0.value > $1.value }
        let winner = hypotheses.first ?? (key: "neither", value: 0)
        let runnerUp = hypotheses.dropFirst().first?.value ?? 0
        return Observation(
            example: example,
            predictedLabel: winner.key,
            confidence: winner.value,
            margin: winner.value - runnerUp
        )
    }
}

private func routingMetrics(
    observations: [Observation],
    confidenceThreshold: Double,
    minimumMargin: Double
) -> RoutingMetrics {
    var routed = 0
    var correctRouted = 0
    var expectedSpecialized = 0
    var missedSpecialized = 0
    var routedByLabel: [String: Int] = [:]
    var correctByLabel: [String: Int] = [:]
    for observation in observations {
        let expectedIsSpecialized = observation.example.label != "neither"
        if expectedIsSpecialized {
            expectedSpecialized += 1
        }
        let shouldRoute = observation.predictedLabel != "neither"
            && observation.confidence >= confidenceThreshold
            && observation.margin >= minimumMargin
        if shouldRoute {
            routed += 1
            routedByLabel[observation.predictedLabel, default: 0] += 1
            if observation.predictedLabel == observation.example.label {
                correctRouted += 1
                correctByLabel[observation.predictedLabel, default: 0] += 1
            }
        } else if expectedIsSpecialized {
            missedSpecialized += 1
        }
    }
    let precision = routed > 0 ? Double(correctRouted) / Double(routed) : 0
    let recall = expectedSpecialized > 0
        ? Double(correctRouted) / Double(expectedSpecialized)
        : 0
    return RoutingMetrics(
        total: observations.count,
        expectedSpecialized: expectedSpecialized,
        routed: routed,
        correctRouted: correctRouted,
        falseRouted: routed - correctRouted,
        missedSpecialized: missedSpecialized,
        precision: rounded(precision),
        recall: rounded(recall),
        f1: rounded(
            precision + recall > 0
                ? 2 * precision * recall / (precision + recall)
                : 0
        ),
        routedByLabel: routedByLabel,
        correctByLabel: correctByLabel
    )
}

private func selectedThreshold(
    observations: [Observation]
) -> ThresholdSelection {
    var selections: [ThresholdSelection] = []
    for confidenceStep in 50...99 {
        for marginStep in 0...10 {
            let confidence = Double(confidenceStep) / 100
            let margin = Double(marginStep) / 20
            let metrics = routingMetrics(
                observations: observations,
                confidenceThreshold: confidence,
                minimumMargin: margin
            )
            if metrics.precision >= 0.95 {
                selections.append(
                    ThresholdSelection(
                        confidenceThreshold: confidence,
                        minimumMargin: margin,
                        metrics: metrics
                    )
                )
            }
        }
    }
    return selections.max {
        if $0.metrics.recall != $1.metrics.recall {
            return $0.metrics.recall < $1.metrics.recall
        }
        if $0.metrics.routed != $1.metrics.routed {
            return $0.metrics.routed < $1.metrics.routed
        }
        if $0.confidenceThreshold != $1.confidenceThreshold {
            return $0.confidenceThreshold > $1.confidenceThreshold
        }
        return $0.minimumMargin > $1.minimumMargin
    } ?? ThresholdSelection(
        confidenceThreshold: 1,
        minimumMargin: 1,
        metrics: routingMetrics(
            observations: observations,
            confidenceThreshold: 1,
            minimumMargin: 1
        )
    )
}

private func gateResult(
    verifier: VerifierID,
    observations: [Observation],
    thresholdsByLanguage: [String: ThresholdSelection],
    globalThreshold: ThresholdSelection
) -> GateResult {
    var support: [String: Int] = [:]
    var correct: [String: Int] = [:]
    var sourceSupport: [String: [String: Int]] = [:]
    for observation in observations {
        let selection = thresholdsByLanguage[observation.example.language]
            ?? globalThreshold
        let routed = observation.predictedLabel != "neither"
            && observation.confidence >= selection.confidenceThreshold
            && observation.margin >= selection.minimumMargin
        guard routed else { continue }
        let key = "\(observation.example.language)|\(observation.predictedLabel)"
        support[key, default: 0] += 1
        let source = observation.example.sourceDataset ?? "generated-base"
        sourceSupport[key, default: [:]][source, default: 0] += 1
        if observation.predictedLabel == observation.example.label {
            correct[key, default: 0] += 1
        }
    }
    let precision = Dictionary(uniqueKeysWithValues: support.map { key, count in
        (
            key,
            rounded(Double(correct[key] ?? 0) / Double(max(count, 1)))
        )
    })
    let maximumSourceShare = Dictionary(uniqueKeysWithValues: support.map { key, count in
        let maximum = sourceSupport[key]?.values.max() ?? 0
        return (key, rounded(Double(maximum) / Double(max(count, 1))))
    })
    let requiredKeys = ["en", "zh-Hans"].flatMap { language in
        verifier.labels
            .filter { $0 != "neither" }
            .map { "\(language)|\($0)" }
    }
    let accepted = requiredKeys.allSatisfy {
        support[$0, default: 0] >= 100
            && precision[$0, default: 0] >= 0.95
            && maximumSourceShare[$0, default: 1] <= 0.65
    }
    return GateResult(
        accepted: accepted,
        reason: accepted
            ? "Every routed label/language meets precision, support, and source-diversity gates."
            : "Shadow only: at least one label/language misses precision, support, or source-diversity gates.",
        minimumPredictedPositivesPerLabelAndLanguage: 100,
        supportByLabelAndLanguage: support,
        precisionByLabelAndLanguage: precision,
        maximumSourceShareByLabelAndLanguage: maximumSourceShare
    )
}

private func writeJSON<T: Encodable>(_ value: T, to url: URL) throws {
    let encoder = JSONEncoder()
    encoder.outputFormatting = [.prettyPrinted, .sortedKeys, .withoutEscapingSlashes]
    try encoder.encode(value).write(to: url, options: .atomic)
}

private func train(
    verifier: VerifierID,
    baseRecords: [BaseRecord],
    silverRecords: [SilverRecord]
) throws -> (VerifierReport, [String: Any]) {
    let training = examples(
        verifier: verifier,
        baseRecords: baseRecords,
        silverRecords: silverRecords,
        baseSplit: "train",
        silverSplit: "silverTrain"
    )
    let calibration = examples(
        verifier: verifier,
        baseRecords: baseRecords,
        silverRecords: silverRecords,
        baseSplit: "validation",
        silverSplit: "silverCalibration"
    )
    let acceptance = examples(
        verifier: verifier,
        baseRecords: baseRecords,
        silverRecords: silverRecords,
        baseSplit: "golden",
        silverSplit: "silverAcceptance"
    )
    let trainingTexts = balancedTexts(
        examples: training,
        verifier: verifier,
        split: "train"
    )
    let calibrationTexts = balancedTexts(
        examples: calibration,
        verifier: verifier,
        split: "calibration"
    )
    let parameters = MLTextClassifier.ModelParameters(
        validation: .dictionary(calibrationTexts),
        algorithm: .maxEnt(revision: 1)
    )
    let model = try MLTextClassifier(
        trainingData: trainingTexts,
        parameters: parameters
    )
    let modelURL = outputDirectoryURL.appendingPathComponent(verifier.modelFile)
    try model.write(to: modelURL)
    let calibrationObservations = try observations(model: model, examples: calibration)
    let acceptanceObservations = try observations(model: model, examples: acceptance)
    let externalCalibrationObservations = calibrationObservations.filter {
        $0.example.sourceDataset != nil
    }
    let thresholdObservations = externalCalibrationObservations.count >= 40
        ? externalCalibrationObservations
        : calibrationObservations
    let globalThreshold = selectedThreshold(observations: thresholdObservations)
    let languages = Set(calibration.map(\.language)).sorted()
    let thresholdsByLanguage = Dictionary(uniqueKeysWithValues: languages.map { language in
        let externalLanguageObservations = thresholdObservations.filter {
            $0.example.language == language
        }
        let languageObservations = externalLanguageObservations.count >= 15
            ? externalLanguageObservations
            : calibrationObservations.filter {
                $0.example.language == language
            }
        return (
            language,
            selectedThreshold(
                observations: languageObservations
            )
        )
    })
    let finalAcceptanceMetrics = routingMetrics(
        observations: acceptanceObservations,
        confidenceThreshold: globalThreshold.confidenceThreshold,
        minimumMargin: globalThreshold.minimumMargin
    )
    let acceptanceByLanguage: [String: RoutingMetrics] = Dictionary(
        uniqueKeysWithValues: languages.map { language in
            let threshold = thresholdsByLanguage[language] ?? globalThreshold
            return (
                language,
                routingMetrics(
                    observations: acceptanceObservations.filter {
                        $0.example.language == language
                    },
                    confidenceThreshold: threshold.confidenceThreshold,
                    minimumMargin: threshold.minimumMargin
                )
            )
        }
    )
    let gate = gateResult(
        verifier: verifier,
        observations: acceptanceObservations,
        thresholdsByLanguage: thresholdsByLanguage,
        globalThreshold: globalThreshold
    )
    let modelBytes = (
        try fileManager.attributesOfItem(atPath: modelURL.path)[.size] as? NSNumber
    )?.intValue ?? 0
    let report = VerifierReport(
        id: verifier.rawValue,
        labels: verifier.labels,
        modelFile: verifier.modelFile,
        modelBytes: modelBytes,
        trainingCount: trainingTexts.values.reduce(0) { $0 + $1.count },
        calibrationCount: calibration.count,
        thresholdCalibrationCount: thresholdObservations.count,
        acceptanceCount: acceptance.count,
        threshold: globalThreshold,
        thresholdsByLanguage: thresholdsByLanguage,
        calibrationMetrics: globalThreshold.metrics,
        acceptanceMetrics: finalAcceptanceMetrics,
        acceptanceByLanguage: acceptanceByLanguage,
        gate: gate
    )
    let configuration: [String: Any] = [
        "id": verifier.rawValue,
        "modelFile": verifier.modelFile,
        "labels": verifier.labels,
        "confidenceThreshold": globalThreshold.confidenceThreshold,
        "confidenceThresholdsByLanguage": thresholdsByLanguage.mapValues {
            $0.confidenceThreshold
        },
        "minimumMargin": globalThreshold.minimumMargin,
        "minimumMarginsByLanguage": thresholdsByLanguage.mapValues {
            $0.minimumMargin
        },
        "acceptedForAutomaticRouting": gate.accepted,
        "deploymentMode": gate.accepted ? "automatic" : "shadow"
    ]
    return (report, configuration)
}

private func main() throws {
    try fileManager.createDirectory(
        at: outputDirectoryURL,
        withIntermediateDirectories: true
    )
    let baseRecords = try readJSONLines(BaseRecord.self, from: baseCorpusURL)
    let silverURLs = [
        silverDirectoryURL.appendingPathComponent("silver-train.jsonl"),
        silverDirectoryURL.appendingPathComponent("silver-calibration.jsonl"),
        silverDirectoryURL.appendingPathComponent("silver-acceptance.jsonl")
    ]
    let silverRecords = try silverURLs.flatMap {
        try readJSONLines(SilverRecord.self, from: $0)
    }
    var reports: [VerifierReport] = []
    var configurations: [[String: Any]] = []
    for verifier in VerifierID.allCases {
        let (report, configuration) = try train(
            verifier: verifier,
            baseRecords: baseRecords,
            silverRecords: silverRecords
        )
        reports.append(report)
        configurations.append(configuration)
        print(
            "VERIFIER_SELECTED id=\(verifier.rawValue) "
                + "accepted=\(report.gate.accepted) "
                + "precision=\(report.acceptanceMetrics.precision)"
        )
    }

    let generatedAt = ISO8601DateFormatter().string(from: Date())
    let report = TrainingReport(
        generatedAt: generatedAt,
        baseCorpusPath: baseCorpusURL.path,
        silverDirectoryPath: silverDirectoryURL.path,
        selectionPolicy:
            "Thresholds and top-1/top-2 margins are calibrated without acceptance data. "
            + "Automatic routing requires >=95% exact-route precision and >=100 routed "
            + "positives for every specialized label in both English and Simplified Chinese. "
            + "Without human gold, passing values remain consensus-relative.",
        verifiers: reports
    )
    try writeJSON(report, to: reportURL)

    let manifestData = try Data(contentsOf: baseManifestURL)
    guard var manifest = try JSONSerialization.jsonObject(
        with: manifestData
    ) as? [String: Any] else {
        throw NSError(
            domain: "VerifierTraining",
            code: 1,
            userInfo: [NSLocalizedDescriptionKey: "Invalid base manifest"]
        )
    }
    for classifier in manifest["classifiers"] as? [[String: Any]] ?? [] {
        guard let modelFile = classifier["modelFile"] as? String else { continue }
        let source = baseManifestURL.deletingLastPathComponent()
            .appendingPathComponent(modelFile)
        let destination = outputDirectoryURL.appendingPathComponent(modelFile)
        if fileManager.fileExists(atPath: destination.path) {
            try fileManager.removeItem(at: destination)
        }
        try fileManager.copyItem(at: source, to: destination)
    }
    manifest["schemaVersion"] = 3
    manifest["verifiers"] = configurations
    manifest["verifierGeneratedAt"] = generatedAt
    manifest["verifierLabelPolicy"] = "multi-model-consensus-without-human-gold"
    let outputManifestURL = outputDirectoryURL.appendingPathComponent(
        "clipboard-semantic-models.json"
    )
    let outputManifestData = try JSONSerialization.data(
        withJSONObject: manifest,
        options: [.prettyPrinted, .sortedKeys, .withoutEscapingSlashes]
    )
    try outputManifestData.write(to: outputManifestURL, options: .atomic)
    print("VERIFIER_REPORT \(reportURL.path)")
}

do {
    try main()
} catch {
    fputs("Verifier training failed: \(error)\n", stderr)
    exit(1)
}