feat(typing): wire PersonalDictionary into Chinese Pinyin via Rime sidecar

Redeploy an osg_personal import table on dictionary add/delete/sync so Chinese, English typing, and ASR share one curated lexicon (next keyboard open).
2026-08-05 22:12:30 +08:00
parent 31f5937a7f
commit 717902716a
11 changed files with 665 additions and 14 deletions
@@ -0,0 +1,72 @@
+// PersonalDictionaryRimeSync.swift
+// OSGKeyboard · Shared
+//
+// Debounces PersonalDictionary mutations on the iOS host and redeploys
+// Rime so the osg_personal sidecar matches. The keyboard extension only
+// picks this up the next time it opens a typing session.
+
+import Foundation
+
+@MainActor
+public enum PersonalDictionaryRimeSync {
+    private static var pending: Task<Void, Never>?
+    private static let debounceNanoseconds: UInt64 = 750_000_000
+    private static let retryNanoseconds: UInt64 = 5_000_000_000
+
+    /// Call after App Group personal-dictionary writes (add / delete / sync).
+    /// Safe from any executor — work is hoppped onto the main actor.
+    public nonisolated static func scheduleAfterDictionaryChange() {
+        Task { @MainActor in
+            scheduleOnMainActor()
+        }
+    }
+
+    public static func deployNow() async {
+        pending?.cancel()
+        pending = nil
+        await deploy(retryOnMemoryPressure: false)
+    }
+
+    private static func scheduleOnMainActor() {
+        pending?.cancel()
+        pending = Task {
+            try? await Task.sleep(nanoseconds: debounceNanoseconds)
+            guard !Task.isCancelled else { return }
+            await deploy(retryOnMemoryPressure: true)
+        }
+    }
+
+    private static func deploy(retryOnMemoryPressure: Bool) async {
+        guard HostMemoryBudget.gate("rime.personalDictionary") else {
+            OSGDiag.log("rime.personalDictionary deferred by memory gate", category: "boot")
+            if retryOnMemoryPressure {
+                pending?.cancel()
+                pending = Task {
+                    try? await Task.sleep(nanoseconds: retryNanoseconds)
+                    guard !Task.isCancelled else { return }
+                    await deploy(retryOnMemoryPressure: true)
+                }
+            }
+            return
+        }
+
+        FlowSessionBridge.setHostHeavy(true)
+        defer { FlowSessionBridge.setHostHeavy(false) }
+
+        let typingConfig = TypingInputConfiguration.shared.snapshot
+        let dictionary = AppGroupStore().personalDictionary
+        do {
+            try await RimeResourceInstaller.shared.installIfNeeded(
+                configuration: typingConfig,
+                personalDictionary: dictionary,
+                force: false
+            )
+            OSGDiag.log("rime.personalDictionary deploy done", category: "boot")
+        } catch {
+            OSGDiag.log(
+                "rime.personalDictionary deploy failed error=\(error.localizedDescription)",
+                category: "boot"
+            )
+        }
+    }
+}
@@ -0,0 +1,131 @@
+// RimePersonalDictionaryExporter.swift
+// OSGKeyboard · Shared
+//
+// Turns PersonalDictionary into an osg_personal Rime table that the
+// main osg_pinyin dictionary imports. High weight keeps same-code
+// personal hits above the baseline lexicon.
+
+import CryptoKit
+import Foundation
+
+public enum RimePersonalDictionaryExporter {
+    public static let dictionaryName = "osg_personal"
+    /// Above any baseline `osg_pinyin` weight so same-code hits pin to top.
+    public static let pinWeight = 50_000_000
+
+    public struct Entry: Equatable, Sendable {
+        public let text: String
+        public let code: String
+        public let weight: Int
+
+        public init(text: String, code: String, weight: Int = RimePersonalDictionaryExporter.pinWeight) {
+            self.text = text
+            self.code = code
+            self.weight = weight
+        }
+    }
+
+    /// Builds Rime rows for Chinese (pinyin-coded), English (latin-coded),
+    /// and Latin aliases that should surface the canonical term.
+    public static func entries(
+        from dictionary: PersonalDictionary,
+        annotator: RimePinyinAnnotator
+    ) -> [Entry] {
+        var seen = Set<String>()
+        var rows: [Entry] = []
+
+        func append(text: String, code: String) {
+            let key = "\(text)\t\(code)"
+            guard seen.insert(key).inserted else { return }
+            rows.append(Entry(text: text, code: code))
+        }
+
+        for entry in dictionary.effectiveEntries {
+            let term = entry.term.trimmingCharacters(in: .whitespacesAndNewlines)
+            guard !term.isEmpty else { continue }
+
+            if let code = annotator.code(for: term) {
+                append(text: term, code: code)
+            }
+
+            // Latin aliases → alternate codes for the canonical term
+            // (e.g. brand English → 中文专名). Chinese ASR aliases stay out.
+            for alias in entry.aliases {
+                let trimmed = alias.trimmingCharacters(in: .whitespacesAndNewlines)
+                guard !trimmed.isEmpty else { continue }
+                guard !RimePinyinAnnotator.containsCJK(trimmed) else { continue }
+                let latin = RimePinyinAnnotator.latinSpellerCode(trimmed)
+                guard !latin.isEmpty else { continue }
+                append(text: term, code: latin)
+            }
+        }
+
+        return rows.sorted {
+            if $0.text != $1.text {
+                return $0.text.localizedStandardCompare($1.text) == .orderedAscending
+            }
+            return $0.code < $1.code
+        }
+    }
+
+    public static func yaml(
+        from dictionary: PersonalDictionary,
+        annotator: RimePinyinAnnotator
+    ) -> String {
+        yaml(entries: entries(from: dictionary, annotator: annotator))
+    }
+
+    public static func yaml(entries: [Entry]) -> String {
+        var lines: [String] = [
+            "# Generated by OSGKeyboard from PersonalDictionary. Do not edit.",
+            "---",
+            "name: \(dictionaryName)",
+            "version: \"1.0\"",
+            "sort: by_weight",
+            "columns:",
+            "  - text",
+            "  - code",
+            "  - weight",
+            "..."
+        ]
+        for entry in entries {
+            lines.append("\(entry.text)\t\(entry.code)\t\(entry.weight)")
+        }
+        lines.append("")
+        return lines.joined(separator: "\n")
+    }
+
+    public static func fingerprint(of yaml: String) -> String {
+        let digest = SHA256.hash(data: Data(yaml.utf8))
+        return digest.map { String(format: "%02x", $0) }.joined()
+    }
+
+    /// Injects `import_tables: [osg_personal]` into a baseline dict header.
+    public static func injectingImportTables(into baselineYAML: String) -> String {
+        if baselineYAML.contains("import_tables:") {
+            return baselineYAML
+        }
+        let needle = "use_preset_vocabulary: false\n"
+        let injection = """
+            use_preset_vocabulary: false
+            import_tables:
+              - \(dictionaryName)
+
+            """
+        if let range = baselineYAML.range(of: needle) {
+            return baselineYAML.replacingCharacters(in: range, with: injection)
+        }
+        // Fallback: insert after the `---` document start block's name line.
+        let nameNeedle = "name: osg_pinyin\n"
+        if let range = baselineYAML.range(of: nameNeedle) {
+            let injectionAfterName = """
+                name: osg_pinyin
+                import_tables:
+                  - \(dictionaryName)
+
+                """
+            return baselineYAML.replacingCharacters(in: range, with: injectionAfterName)
+        }
+        return baselineYAML
+    }
+}
@@ -0,0 +1,184 @@
+// RimePinyinAnnotator.swift
+// OSGKeyboard · Shared
+//
+// Builds phrase / character pinyin maps from the bundled osg_pinyin
+// dictionary so PersonalDictionary terms can be coded for Rime without
+// shipping a second pronunciation dataset.
+
+import Foundation
+
+public struct RimePinyinAnnotator: Sendable {
+    private let phraseCodes: [String: String]
+    private let characterCodes: [String: String]
+
+    public init(phraseCodes: [String: String], characterCodes: [String: String]) {
+        self.phraseCodes = phraseCodes
+        self.characterCodes = characterCodes
+    }
+
+    /// Parses `osg_pinyin.dict.yaml` (text / code / weight columns).
+    /// When duplicate texts exist, keeps the highest-weight code.
+    public static func load(from dictYAML: URL) throws -> RimePinyinAnnotator {
+        let raw = try String(contentsOf: dictYAML, encoding: .utf8)
+        var phraseCodes: [String: (code: String, weight: Int)] = [:]
+        var characterCodes: [String: (code: String, weight: Int)] = [:]
+        var inBody = false
+
+        for line in raw.split(separator: "\n", omittingEmptySubsequences: false) {
+            let trimmed = line.trimmingCharacters(in: .whitespaces)
+            if trimmed == "..." {
+                inBody = true
+                continue
+            }
+            guard inBody, !trimmed.isEmpty, !trimmed.hasPrefix("#") else { continue }
+
+            let parts = trimmed.split(separator: "\t", omittingEmptySubsequences: false)
+            guard parts.count >= 2 else { continue }
+            let text = String(parts[0])
+            let code = String(parts[1]).trimmingCharacters(in: .whitespacesAndNewlines)
+            guard !text.isEmpty, !code.isEmpty else { continue }
+            let weight = parts.count >= 3 ? Int(parts[2]) ?? 0 : 0
+
+            if let existing = phraseCodes[text] {
+                if weight >= existing.weight {
+                    phraseCodes[text] = (code, weight)
+                }
+            } else {
+                phraseCodes[text] = (code, weight)
+            }
+
+            if text.count == 1, Self.isCJKIdeograph(text.unicodeScalars.first!) {
+                if let existing = characterCodes[text] {
+                    if weight >= existing.weight {
+                        characterCodes[text] = (code, weight)
+                    }
+                } else {
+                    characterCodes[text] = (code, weight)
+                }
+            }
+        }
+
+        return RimePinyinAnnotator(
+            phraseCodes: phraseCodes.mapValues(\.code),
+            characterCodes: characterCodes.mapValues(\.code)
+        )
+    }
+
+    /// Returns a Rime speller code (space-separated syllables / Latin tokens),
+    /// or `nil` when any CJK character cannot be annotated.
+    public func code(for term: String) -> String? {
+        let trimmed = term.trimmingCharacters(in: .whitespacesAndNewlines)
+        guard !trimmed.isEmpty else { return nil }
+
+        if let exact = phraseCodes[trimmed] {
+            return exact
+        }
+
+        var parts: [String] = []
+        for run in Self.scriptRuns(in: trimmed) {
+            switch run.kind {
+            case .cjk:
+                if let phrase = phraseCodes[run.text] {
+                    parts.append(phrase)
+                    continue
+                }
+                var syllables: [String] = []
+                for character in run.text {
+                    let key = String(character)
+                    guard let syllable = characterCodes[key] else { return nil }
+                    syllables.append(syllable)
+                }
+                parts.append(syllables.joined(separator: " "))
+            case .latin:
+                let latin = Self.latinSpellerCode(run.text)
+                guard !latin.isEmpty else { continue }
+                parts.append(latin)
+            case .other:
+                continue
+            }
+        }
+
+        let joined = parts.joined(separator: " ")
+            .split(separator: " ", omittingEmptySubsequences: true)
+            .joined(separator: " ")
+        return joined.isEmpty ? nil : joined
+    }
+
+    // MARK: - Script helpers
+
+    private enum RunKind {
+        case cjk
+        case latin
+        case other
+    }
+
+    private struct ScriptRun {
+        let kind: RunKind
+        let text: String
+    }
+
+    private static func scriptRuns(in term: String) -> [ScriptRun] {
+        var runs: [ScriptRun] = []
+        var currentKind: RunKind?
+        var buffer = ""
+
+        func flush() {
+            guard let kind = currentKind, !buffer.isEmpty else { return }
+            runs.append(ScriptRun(kind: kind, text: buffer))
+            buffer = ""
+            currentKind = nil
+        }
+
+        for scalar in term.unicodeScalars {
+            let kind: RunKind
+            if isCJKIdeograph(scalar) {
+                kind = .cjk
+            } else if scalar.isASCII, CharacterSet.letters.contains(scalar)
+                || CharacterSet.decimalDigits.contains(scalar)
+                || scalar == "-" || scalar == "'" || scalar == "_" {
+                kind = .latin
+            } else if scalar == " " || scalar == "\u{3000}" {
+                flush()
+                continue
+            } else {
+                kind = .other
+            }
+
+            if currentKind == nil {
+                currentKind = kind
+                buffer = String(scalar)
+            } else if currentKind == kind {
+                buffer.append(Character(scalar))
+            } else {
+                flush()
+                currentKind = kind
+                buffer = String(scalar)
+            }
+        }
+        flush()
+        return runs
+    }
+
+    /// Speller alphabet is a–z only; strip everything else and lowercase.
+    public static func latinSpellerCode(_ raw: String) -> String {
+        var output = ""
+        for scalar in raw.lowercased().unicodeScalars {
+            guard scalar.isASCII, CharacterSet.lowercaseLetters.contains(scalar) else { continue }
+            output.append(Character(scalar))
+        }
+        return output
+    }
+
+    public static func isCJKIdeograph(_ scalar: Unicode.Scalar) -> Bool {
+        switch scalar.value {
+        case 0x3400...0x4DBF, 0x4E00...0x9FFF, 0xF900...0xFAFF:
+            return true
+        default:
+            return false
+        }
+    }
+
+    public static func containsCJK(_ text: String) -> Bool {
+        text.unicodeScalars.contains(where: isCJKIdeograph)
+    }
+}
@@ -55,7 +55,8 @@ public struct RimeResourcePaths: Sendable {

 public actor RimeResourceInstaller {
    public static let shared = RimeResourceInstaller()
-    public static let resourceVersion = "2.2.0"
+    /// Bump when SharedSupport layout / schema / import_tables contract changes.
+    public static let resourceVersion = "2.3.0"

    public init() {}

@@ -71,11 +72,21 @@ public actor RimeResourceInstaller {

    /// Installs source data and asks librime to prebuild schemas. Call only
    /// from the host app, never from the keyboard extension.
+    ///
+    /// Redeploys when `force` is set, the resource version is stale, or the
+    /// PersonalDictionary sidecar fingerprint changed.
    public func installIfNeeded(
        configuration: TypingInputConfigurationSnapshot,
+        personalDictionary: PersonalDictionary? = nil,
        force: Bool = false
    ) throws {
-        if !force, Self.isReady { return }
+        let dictionary = personalDictionary ?? AppGroupStore().personalDictionary
+        let personalYAML = try Self.makePersonalDictionaryYAML(from: dictionary)
+        let personalFingerprint = RimePersonalDictionaryExporter.fingerprint(of: personalYAML)
+        let personalChanged =
+            TypingInputConfiguration.installedPersonalDictionaryFingerprint() != personalFingerprint
+
+        if !force, Self.isReady, !personalChanged { return }

        let paths = try RimeResourcePaths.resolve()
        let fileManager = FileManager.default
@@ -99,16 +110,32 @@ public actor RimeResourceInstaller {
        defer { try? fileManager.removeItem(at: staging) }
        try fileManager.createDirectory(at: staging, withIntermediateDirectories: true)

-        for resource in ["osg_pinyin.dict", "manifest"] {
-            let ext = resource == "manifest" ? "json" : "yaml"
-            guard let source = Self.bundledURL(forResource: resource, withExtension: ext) else {
-                throw RimeResourceError.bundledResourceMissing("\(resource).\(ext)")
-            }
-            try fileManager.copyItem(
-                at: source,
-                to: staging.appendingPathComponent("\(resource).\(ext)")
-            )
+        guard let pinyinSource = Self.bundledURL(forResource: "osg_pinyin.dict", withExtension: "yaml") else {
+            throw RimeResourceError.bundledResourceMissing("osg_pinyin.dict.yaml")
        }
+        let baseline = try String(contentsOf: pinyinSource, encoding: .utf8)
+        let patched = RimePersonalDictionaryExporter.injectingImportTables(into: baseline)
+        try patched.write(
+            to: staging.appendingPathComponent("osg_pinyin.dict.yaml"),
+            atomically: true,
+            encoding: .utf8
+        )
+
+        guard let manifestSource = Self.bundledURL(forResource: "manifest", withExtension: "json") else {
+            throw RimeResourceError.bundledResourceMissing("manifest.json")
+        }
+        try fileManager.copyItem(
+            at: manifestSource,
+            to: staging.appendingPathComponent("manifest.json")
+        )
+
+        try personalYAML.write(
+            to: staging.appendingPathComponent(
+                "\(RimePersonalDictionaryExporter.dictionaryName).dict.yaml"
+            ),
+            atomically: true,
+            encoding: .utf8
+        )

        try RimeSchemaGenerator.defaultConfiguration().write(
            to: staging.appendingPathComponent("default.yaml"),
@@ -147,9 +174,20 @@ public actor RimeResourceInstaller {
        bridge.finalizeRuntime()

        TypingInputConfiguration.setInstalledResourceVersion(Self.resourceVersion)
+        TypingInputConfiguration.setInstalledPersonalDictionaryFingerprint(personalFingerprint)
        AppGroupConfigDarwin.postConfigChanged()
    }

+    private static func makePersonalDictionaryYAML(
+        from dictionary: PersonalDictionary
+    ) throws -> String {
+        guard let pinyinSource = bundledURL(forResource: "osg_pinyin.dict", withExtension: "yaml") else {
+            throw RimeResourceError.bundledResourceMissing("osg_pinyin.dict.yaml")
+        }
+        let annotator = try RimePinyinAnnotator.load(from: pinyinSource)
+        return RimePersonalDictionaryExporter.yaml(from: dictionary, annotator: annotator)
+    }
+
    public func syncUserData() throws {
        let paths = try RimeResourcePaths.resolve()
        let bridge = OSGRimeBridge(