fix(asr): restore computer terminology bias
Rebuild the bundled language model and runtime source filtering around a dedicated computer-term corpus so technical dictation keeps domain coverage.
This commit is contained in:
Binary file not shown.
@@ -1,13 +1,14 @@
|
|||||||
{
|
{
|
||||||
"bin_bytes" : 41456,
|
"bin_bytes" : 198494,
|
||||||
"bin_file" : "OSGKeyboardCLM.bin",
|
"bin_file" : "OSGKeyboardCLM.bin",
|
||||||
"export_seconds" : 0.030717015266418457,
|
"export_seconds" : 0.035165071487426758,
|
||||||
"generated_at" : "2026-08-14T15:07:23Z",
|
"generated_at" : "2026-07-06T13:16:27Z",
|
||||||
"identifier" : "com.osgkeyboard.custom-lm.v1",
|
"identifier" : "com.osgkeyboard.custom-lm.v1",
|
||||||
"locale" : "zh_CN",
|
"locale" : "zh_CN",
|
||||||
"phrase_count" : 3040,
|
"phrase_count" : 13329,
|
||||||
"sources" : {
|
"sources" : {
|
||||||
"ai_tech_seed" : 3040
|
"ai_tech_seed" : 3040,
|
||||||
|
"computer_terms" : 10300
|
||||||
},
|
},
|
||||||
"version" : "1.0.1"
|
"version" : "1.0.0"
|
||||||
}
|
}
|
||||||
@@ -1,22 +1,22 @@
|
|||||||
{
|
{
|
||||||
"entry_count" : 3040,
|
"version": "v1",
|
||||||
"files" : {
|
"generated_at": "2026-07-05T11:27:48.823095+00:00",
|
||||||
"phrases" : "phrases.tsv"
|
|
||||||
},
|
|
||||||
"generated_at" : "2026-08-14T15:07:23Z",
|
|
||||||
"locale": "zh-Hans",
|
"locale": "zh-Hans",
|
||||||
"notes" : [
|
"entry_count": 10300,
|
||||||
"Project-curated bilingual AI brands, technology terms, companies, and names.",
|
|
||||||
"No third-party cell dictionaries or Sogou-derived data.",
|
|
||||||
"PhraseCount weights map to SFCustomLanguageModelData relative frequencies."
|
|
||||||
],
|
|
||||||
"sources": [
|
"sources": [
|
||||||
{
|
{
|
||||||
"key" : "ai_tech_seed",
|
"key": "computer_terms",
|
||||||
"label" : "OSGKeyboard curated AI\/tech lexicon",
|
"label": "计算机词汇大全【官方推荐】",
|
||||||
"license" : "MIT (curated seed; OSGKeyboard contributors)",
|
"weight": 5,
|
||||||
"raw_count" : 3040
|
"raw_count": 10300
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"version" : "v1"
|
"notes": [
|
||||||
|
"Domain-specific computer/IT vocabulary only; casual network slang and Sogou popular words removed.",
|
||||||
|
"PhraseCount weights map to SFCustomLanguageModelData relative frequencies.",
|
||||||
|
"Merged with ai-tech-brands seed at export time for the final .bin asset."
|
||||||
|
],
|
||||||
|
"files": {
|
||||||
|
"phrases": "phrases.tsv"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
Binary file not shown.
@@ -1,13 +1,14 @@
|
|||||||
{
|
{
|
||||||
"bin_bytes" : 41456,
|
"bin_bytes" : 198494,
|
||||||
"bin_file" : "OSGKeyboardCLM.bin",
|
"bin_file" : "OSGKeyboardCLM.bin",
|
||||||
"export_seconds" : 0.030717015266418457,
|
"export_seconds" : 0.035165071487426758,
|
||||||
"generated_at" : "2026-08-14T15:07:23Z",
|
"generated_at" : "2026-07-06T13:16:27Z",
|
||||||
"identifier" : "com.osgkeyboard.custom-lm.v1",
|
"identifier" : "com.osgkeyboard.custom-lm.v1",
|
||||||
"locale" : "zh_CN",
|
"locale" : "zh_CN",
|
||||||
"phrase_count" : 3040,
|
"phrase_count" : 13329,
|
||||||
"sources" : {
|
"sources" : {
|
||||||
"ai_tech_seed" : 3040
|
"ai_tech_seed" : 3040,
|
||||||
|
"computer_terms" : 10300
|
||||||
},
|
},
|
||||||
"version" : "1.0.1"
|
"version" : "1.0.0"
|
||||||
}
|
}
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
// BuiltinLexiconIndex.swift
|
// BuiltinLexiconIndex.swift
|
||||||
// OSGKeyboard · Shared
|
// OSGKeyboard · Shared
|
||||||
//
|
//
|
||||||
// In-memory index over the bundled project-curated AI/technology `phrases.tsv`.
|
// In-memory index over bundled `phrases.tsv` (~10k computer terms).
|
||||||
// macOS local ASR consumes a Top-N subset; the full index also backs
|
// macOS local ASR consumes a Top-N subset; the full index also backs
|
||||||
// polish supplements and future retrieval.
|
// polish supplements and future retrieval.
|
||||||
|
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ import Foundation
|
|||||||
|
|
||||||
public enum LocalASRBiasAdapter {
|
public enum LocalASRBiasAdapter {
|
||||||
|
|
||||||
/// Bundle IDs where the curated AI/technology vocabulary is especially likely.
|
/// Bundle IDs where computer-science vocabulary is especially likely.
|
||||||
private static let codeEditorBundleIDs: Set<String> = [
|
private static let codeEditorBundleIDs: Set<String> = [
|
||||||
"com.apple.dt.Xcode",
|
"com.apple.dt.Xcode",
|
||||||
"com.microsoft.VSCode",
|
"com.microsoft.VSCode",
|
||||||
@@ -119,7 +119,7 @@ public enum LocalASRBiasAdapter {
|
|||||||
|
|
||||||
private static func preferredLexiconSources(for bundleId: String?) -> Set<String>? {
|
private static func preferredLexiconSources(for bundleId: String?) -> Set<String>? {
|
||||||
guard let bundleId, codeEditorBundleIDs.contains(bundleId) else { return nil }
|
guard let bundleId, codeEditorBundleIDs.contains(bundleId) else { return nil }
|
||||||
return ["ai_tech_seed"]
|
return ["computer_terms"]
|
||||||
}
|
}
|
||||||
|
|
||||||
private static func hardHotwordList(from terms: [String], maxCount: Int) -> [String] {
|
private static func hardHotwordList(from terms: [String], maxCount: Int) -> [String] {
|
||||||
|
|||||||
@@ -13,9 +13,9 @@ final class LocalASRBiasAdapterTests: XCTestCase {
|
|||||||
let url = dir.appendingPathComponent("phrases.tsv")
|
let url = dir.appendingPathComponent("phrases.tsv")
|
||||||
let tsv = """
|
let tsv = """
|
||||||
word\tpinyin\tsource\tweight
|
word\tpinyin\tsource\tweight
|
||||||
SwiftUI\tswift ui\tai_tech_seed\t90
|
SwiftUI\tswift ui\tcomputer_terms\t5
|
||||||
Kubernetes\tku bo ne si\tai_tech_seed\t90
|
Kubernetes\tku bo ne si\tcomputer_terms\t5
|
||||||
一致性\tyi zhi xing\tai_tech_seed\t80
|
一致性\tyi zhi xing\tcomputer_terms\t5
|
||||||
"""
|
"""
|
||||||
try tsv.write(to: url, atomically: true, encoding: .utf8)
|
try tsv.write(to: url, atomically: true, encoding: .utf8)
|
||||||
addTeardownBlock {
|
addTeardownBlock {
|
||||||
@@ -103,12 +103,11 @@ final class LocalASRBiasAdapterTests: XCTestCase {
|
|||||||
|
|
||||||
func testBuiltinLexiconParsesTSV() {
|
func testBuiltinLexiconParsesTSV() {
|
||||||
let terms = BuiltinLexiconIndex.parseTSV(
|
let terms = BuiltinLexiconIndex.parseTSV(
|
||||||
"word\tpinyin\tsource\tweight\nFoo\tfoo\tai_tech_seed\t80\n"
|
"word\tpinyin\tsource\tweight\nFoo\tfoo\tcomputer_terms\t5\n"
|
||||||
)
|
)
|
||||||
XCTAssertEqual(terms.count, 1)
|
XCTAssertEqual(terms.count, 1)
|
||||||
XCTAssertEqual(terms[0].word, "Foo")
|
XCTAssertEqual(terms[0].word, "Foo")
|
||||||
XCTAssertEqual(terms[0].source, "ai_tech_seed")
|
XCTAssertEqual(terms[0].weight, 5)
|
||||||
XCTAssertEqual(terms[0].weight, 80)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func testPolishingServiceMergesDictionarySupplement() {
|
func testPolishingServiceMergesDictionarySupplement() {
|
||||||
|
|||||||
@@ -106,7 +106,7 @@ final class LocalASRModelCatalogTests: XCTestCase {
|
|||||||
func testMLXAdapterProducesPromptBiasNotHardHotwords() throws {
|
func testMLXAdapterProducesPromptBiasNotHardHotwords() throws {
|
||||||
let fixtureURL = FileManager.default.temporaryDirectory
|
let fixtureURL = FileManager.default.temporaryDirectory
|
||||||
.appendingPathComponent("phrases-\(UUID().uuidString).tsv")
|
.appendingPathComponent("phrases-\(UUID().uuidString).tsv")
|
||||||
try "word\tpinyin\tsource\tweight\nSwiftUI\tswift ui\tai_tech_seed\t90\n"
|
try "word\tpinyin\tsource\tweight\nSwiftUI\tswift ui\tcomputer_terms\t5\n"
|
||||||
.write(to: fixtureURL, atomically: true, encoding: .utf8)
|
.write(to: fixtureURL, atomically: true, encoding: .utf8)
|
||||||
defer { try? FileManager.default.removeItem(at: fixtureURL) }
|
defer { try? FileManager.default.removeItem(at: fixtureURL) }
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,130 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""Build OSGKeyboard domain ASR lexicon v1 from the computer-terms scel source.
|
||||||
|
|
||||||
|
Sources:
|
||||||
|
Local 计算机词汇大全【官方推荐】.scel (IT / computer vocabulary only)
|
||||||
|
|
||||||
|
Casual network slang and Sogou popular-word dumps are intentionally excluded —
|
||||||
|
they dilute custom LM phrase biasing without improving domain ASR accuracy.
|
||||||
|
|
||||||
|
Output:
|
||||||
|
OSGKeyboard/Resources/CustomLanguageModel/v1/phrases.tsv
|
||||||
|
OSGKeyboard/Resources/CustomLanguageModel/v1/manifest.json
|
||||||
|
|
||||||
|
The compiled .bin asset is exported separately to
|
||||||
|
OSGKeyboard/Resources/CustomLanguageModel/v1/ via export_clm.swift.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from scel_parser import get_scel_info, parse_scel_file
|
||||||
|
|
||||||
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
DEFAULT_OUTPUT_DIR = REPO_ROOT / "OSGKeyboard/Resources/CustomLanguageModel/v1"
|
||||||
|
DEFAULT_COMPUTER_SCEL = Path("/Users/rocky/Downloads/计算机词汇大全【官方推荐】.scel")
|
||||||
|
|
||||||
|
SOURCE_KEY = "computer_terms"
|
||||||
|
SOURCE_LABEL = "计算机词汇大全【官方推荐】"
|
||||||
|
SOURCE_WEIGHT = 5
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class LexiconEntry:
|
||||||
|
word: str
|
||||||
|
pinyin: str
|
||||||
|
source: str
|
||||||
|
weight: int
|
||||||
|
|
||||||
|
|
||||||
|
def merge_entries(entries: list[tuple[str, str]]) -> list[LexiconEntry]:
|
||||||
|
merged: dict[str, LexiconEntry] = {}
|
||||||
|
|
||||||
|
for word, pinyin in entries:
|
||||||
|
current = merged.get(word)
|
||||||
|
candidate = LexiconEntry(
|
||||||
|
word=word,
|
||||||
|
pinyin=pinyin,
|
||||||
|
source=SOURCE_KEY,
|
||||||
|
weight=SOURCE_WEIGHT,
|
||||||
|
)
|
||||||
|
if current is None:
|
||||||
|
merged[word] = candidate
|
||||||
|
elif not current.pinyin and pinyin:
|
||||||
|
merged[word] = candidate
|
||||||
|
|
||||||
|
return sorted(merged.values(), key=lambda item: item.word)
|
||||||
|
|
||||||
|
|
||||||
|
def write_outputs(entries: list[LexiconEntry], output_dir: Path, raw_count: int) -> None:
|
||||||
|
output_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
phrases_path = output_dir / "phrases.tsv"
|
||||||
|
with phrases_path.open("w", encoding="utf-8") as handle:
|
||||||
|
handle.write("word\tpinyin\tsource\tweight\n")
|
||||||
|
for entry in entries:
|
||||||
|
handle.write(f"{entry.word}\t{entry.pinyin}\t{entry.source}\t{entry.weight}\n")
|
||||||
|
|
||||||
|
manifest = {
|
||||||
|
"version": "v1",
|
||||||
|
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||||
|
"locale": "zh-Hans",
|
||||||
|
"entry_count": len(entries),
|
||||||
|
"sources": [
|
||||||
|
{
|
||||||
|
"key": SOURCE_KEY,
|
||||||
|
"label": SOURCE_LABEL,
|
||||||
|
"weight": SOURCE_WEIGHT,
|
||||||
|
"raw_count": raw_count,
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"notes": [
|
||||||
|
"Domain-specific computer/IT vocabulary only; casual network slang removed.",
|
||||||
|
"PhraseCount weights map to SFCustomLanguageModelData relative frequencies.",
|
||||||
|
"Merged with ai-tech-brands seed at export time for the final .bin asset.",
|
||||||
|
],
|
||||||
|
"files": {
|
||||||
|
"phrases": phrases_path.name,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
manifest_path = output_dir / "manifest.json"
|
||||||
|
manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def build(*, computer_scel: Path, output_dir: Path) -> int:
|
||||||
|
if not computer_scel.exists():
|
||||||
|
print(f"Missing computer scel: {computer_scel}", file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
computer_info = get_scel_info(computer_scel)
|
||||||
|
print(f"Computer dict: {computer_info.name} ({computer_info.word_count} header count)")
|
||||||
|
|
||||||
|
raw_entries = parse_scel_file(computer_scel)
|
||||||
|
merged = merge_entries(raw_entries)
|
||||||
|
write_outputs(merged, output_dir, raw_count=len(raw_entries))
|
||||||
|
|
||||||
|
print(f"Raw count: {len(raw_entries)}")
|
||||||
|
print(f"Merged unique entries: {len(merged)}")
|
||||||
|
print(f"Wrote {output_dir / 'phrases.tsv'}")
|
||||||
|
print(f"Wrote {output_dir / 'manifest.json'}")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser(description="Build OSGKeyboard domain ASR lexicon v1")
|
||||||
|
parser.add_argument("--computer-scel", type=Path, default=DEFAULT_COMPUTER_SCEL)
|
||||||
|
parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT_DIR)
|
||||||
|
args = parser.parse_args()
|
||||||
|
return build(computer_scel=args.computer_scel, output_dir=args.output_dir)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -3,9 +3,7 @@
|
|||||||
// export_clm.swift
|
// export_clm.swift
|
||||||
// OSGKeyboard · offline SFCustomLanguageModelData exporter (macOS 14+)
|
// OSGKeyboard · offline SFCustomLanguageModelData exporter (macOS 14+)
|
||||||
//
|
//
|
||||||
// Reads the project-curated AI/tech TSV and writes a .bin training asset via
|
// Reads merged phrase TSVs and writes a .bin training asset via Speech framework.
|
||||||
// Speech framework. The normalized four-column TSV beside the binary also
|
|
||||||
// powers the Mac runtime bias index.
|
|
||||||
// Usage:
|
// Usage:
|
||||||
// swift Scripts/lexicon/export_clm.swift
|
// swift Scripts/lexicon/export_clm.swift
|
||||||
// swift Scripts/lexicon/export_clm.swift --max-entries 30000
|
// swift Scripts/lexicon/export_clm.swift --max-entries 30000
|
||||||
@@ -17,6 +15,7 @@ import Speech
|
|||||||
// MARK: - CLI
|
// MARK: - CLI
|
||||||
|
|
||||||
struct CLIOptions {
|
struct CLIOptions {
|
||||||
|
var domainTSV: URL
|
||||||
var aiTechTSV: URL
|
var aiTechTSV: URL
|
||||||
var outputBin: URL
|
var outputBin: URL
|
||||||
var localeID: String
|
var localeID: String
|
||||||
@@ -30,6 +29,9 @@ struct CLIOptions {
|
|||||||
.deletingLastPathComponent()
|
.deletingLastPathComponent()
|
||||||
.deletingLastPathComponent()
|
.deletingLastPathComponent()
|
||||||
|
|
||||||
|
var domain = repoRoot.appendingPathComponent(
|
||||||
|
"OSGKeyboard/Resources/CustomLanguageModel/v1/phrases.tsv"
|
||||||
|
)
|
||||||
var aiTech = repoRoot.appendingPathComponent(
|
var aiTech = repoRoot.appendingPathComponent(
|
||||||
"OSGKeyboard/Resources/CustomLanguageModel/ai-tech-brands/v1/phrases.tsv"
|
"OSGKeyboard/Resources/CustomLanguageModel/ai-tech-brands/v1/phrases.tsv"
|
||||||
)
|
)
|
||||||
@@ -38,12 +40,14 @@ struct CLIOptions {
|
|||||||
)
|
)
|
||||||
var localeID = "zh_CN"
|
var localeID = "zh_CN"
|
||||||
var modelID = "com.osgkeyboard.custom-lm.v1"
|
var modelID = "com.osgkeyboard.custom-lm.v1"
|
||||||
var modelVersion = "1.0.1"
|
var modelVersion = "1.0.0"
|
||||||
var maxEntries: Int?
|
var maxEntries: Int?
|
||||||
|
|
||||||
var iterator = CommandLine.arguments.dropFirst().makeIterator()
|
var iterator = CommandLine.arguments.dropFirst().makeIterator()
|
||||||
while let flag = iterator.next() {
|
while let flag = iterator.next() {
|
||||||
switch flag {
|
switch flag {
|
||||||
|
case "--domain-tsv", "--sogou-tsv":
|
||||||
|
domain = URL(fileURLWithPath: iterator.next() ?? "")
|
||||||
case "--ai-tech-tsv":
|
case "--ai-tech-tsv":
|
||||||
aiTech = URL(fileURLWithPath: iterator.next() ?? "")
|
aiTech = URL(fileURLWithPath: iterator.next() ?? "")
|
||||||
case "--output":
|
case "--output":
|
||||||
@@ -67,6 +71,7 @@ struct CLIOptions {
|
|||||||
}
|
}
|
||||||
|
|
||||||
return CLIOptions(
|
return CLIOptions(
|
||||||
|
domainTSV: domain,
|
||||||
aiTechTSV: aiTech,
|
aiTechTSV: aiTech,
|
||||||
outputBin: output,
|
outputBin: output,
|
||||||
localeID: localeID,
|
localeID: localeID,
|
||||||
@@ -81,7 +86,8 @@ struct CLIOptions {
|
|||||||
export_clm.swift — build SFCustomLanguageModelData .bin on macOS
|
export_clm.swift — build SFCustomLanguageModelData .bin on macOS
|
||||||
|
|
||||||
Options:
|
Options:
|
||||||
--ai-tech-tsv <path> Project-curated AI/tech phrases TSV
|
--domain-tsv <path> Domain phrases TSV (computer/IT terms)
|
||||||
|
--ai-tech-tsv <path> AI/tech seed phrases TSV
|
||||||
--output <path> Output .bin path
|
--output <path> Output .bin path
|
||||||
--locale <id> Locale identifier (default: zh_CN)
|
--locale <id> Locale identifier (default: zh_CN)
|
||||||
--identifier <id> Custom LM identifier
|
--identifier <id> Custom LM identifier
|
||||||
@@ -96,7 +102,6 @@ struct CLIOptions {
|
|||||||
|
|
||||||
struct PhraseEntry: Hashable {
|
struct PhraseEntry: Hashable {
|
||||||
let phrase: String
|
let phrase: String
|
||||||
let pinyin: String
|
|
||||||
let weight: Int
|
let weight: Int
|
||||||
let source: String
|
let source: String
|
||||||
}
|
}
|
||||||
@@ -121,7 +126,7 @@ enum TSVLoader {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Formats:
|
// Formats:
|
||||||
// normalized: word, pinyin, source, weight
|
// domain: word, pinyin, source, weight
|
||||||
// ai-tech: word, pinyin, source, category, weight, canonical
|
// ai-tech: word, pinyin, source, category, weight, canonical
|
||||||
let weight: Int
|
let weight: Int
|
||||||
if parts.count >= 6, let parsed = Int(parts[4]) {
|
if parts.count >= 6, let parsed = Int(parts[4]) {
|
||||||
@@ -132,16 +137,8 @@ enum TSVLoader {
|
|||||||
weight = 1
|
weight = 1
|
||||||
}
|
}
|
||||||
|
|
||||||
let pinyin = parts.count >= 2 ? parts[1] : ""
|
|
||||||
let source = parts.count >= 3 ? parts[2] : sourceLabel
|
let source = parts.count >= 3 ? parts[2] : sourceLabel
|
||||||
entries.append(
|
entries.append(PhraseEntry(phrase: word, weight: max(1, weight), source: source))
|
||||||
PhraseEntry(
|
|
||||||
phrase: word,
|
|
||||||
pinyin: pinyin,
|
|
||||||
weight: max(1, weight),
|
|
||||||
source: source
|
|
||||||
)
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return entries
|
return entries
|
||||||
@@ -174,13 +171,17 @@ enum ExportCLM {
|
|||||||
let options = CLIOptions.parse()
|
let options = CLIOptions.parse()
|
||||||
let fm = FileManager.default
|
let fm = FileManager.default
|
||||||
|
|
||||||
|
guard fm.fileExists(atPath: options.domainTSV.path) else {
|
||||||
|
throw ExportError.missingInput(options.domainTSV.path)
|
||||||
|
}
|
||||||
guard fm.fileExists(atPath: options.aiTechTSV.path) else {
|
guard fm.fileExists(atPath: options.aiTechTSV.path) else {
|
||||||
throw ExportError.missingInput(options.aiTechTSV.path)
|
throw ExportError.missingInput(options.aiTechTSV.path)
|
||||||
}
|
}
|
||||||
|
|
||||||
fputs("Loading phrases…\n", stderr)
|
fputs("Loading phrases…\n", stderr)
|
||||||
|
let domain = try TSVLoader.load(from: options.domainTSV, sourceLabel: "computer_terms")
|
||||||
let aiTech = try TSVLoader.load(from: options.aiTechTSV, sourceLabel: "ai_tech_seed")
|
let aiTech = try TSVLoader.load(from: options.aiTechTSV, sourceLabel: "ai_tech_seed")
|
||||||
var merged = TSVLoader.merge([aiTech])
|
var merged = TSVLoader.merge([domain, aiTech])
|
||||||
|
|
||||||
if let cap = options.maxEntries, merged.count > cap {
|
if let cap = options.maxEntries, merged.count > cap {
|
||||||
merged = Array(merged.prefix(cap))
|
merged = Array(merged.prefix(cap))
|
||||||
@@ -188,7 +189,7 @@ enum ExportCLM {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fputs(
|
fputs(
|
||||||
"Loaded \(merged.count) unique project-curated AI/tech phrases\n",
|
"Merged \(merged.count) unique phrases (domain=\(domain.count), ai-tech=\(aiTech.count))\n",
|
||||||
stderr
|
stderr
|
||||||
)
|
)
|
||||||
fputs("Locale=\(options.localeID) identifier=\(options.modelID) version=\(options.modelVersion)\n", stderr)
|
fputs("Locale=\(options.localeID) identifier=\(options.modelID) version=\(options.modelVersion)\n", stderr)
|
||||||
@@ -217,15 +218,6 @@ enum ExportCLM {
|
|||||||
try fm.removeItem(at: outputURL)
|
try fm.removeItem(at: outputURL)
|
||||||
}
|
}
|
||||||
|
|
||||||
let phrasesURL = parent.appendingPathComponent("phrases.tsv")
|
|
||||||
let normalizedLines = merged.map { entry in
|
|
||||||
"\(entry.phrase)\t\(entry.pinyin)\t\(entry.source)\t\(entry.weight)"
|
|
||||||
}
|
|
||||||
let normalizedTSV = (["word\tpinyin\tsource\tweight"] + normalizedLines)
|
|
||||||
.joined(separator: "\n") + "\n"
|
|
||||||
try normalizedTSV.write(to: phrasesURL, atomically: true, encoding: .utf8)
|
|
||||||
fputs("Wrote \(phrasesURL.path)\n", stderr)
|
|
||||||
|
|
||||||
fputs("Exporting to \(outputURL.path)…\n", stderr)
|
fputs("Exporting to \(outputURL.path)…\n", stderr)
|
||||||
try await data.export(to: outputURL)
|
try await data.export(to: outputURL)
|
||||||
|
|
||||||
@@ -244,6 +236,7 @@ enum ExportCLM {
|
|||||||
"version": options.modelVersion,
|
"version": options.modelVersion,
|
||||||
"phrase_count": merged.count,
|
"phrase_count": merged.count,
|
||||||
"sources": [
|
"sources": [
|
||||||
|
"computer_terms": domain.count,
|
||||||
"ai_tech_seed": aiTech.count,
|
"ai_tech_seed": aiTech.count,
|
||||||
],
|
],
|
||||||
"bin_file": outputURL.lastPathComponent,
|
"bin_file": outputURL.lastPathComponent,
|
||||||
@@ -253,32 +246,6 @@ enum ExportCLM {
|
|||||||
let manifestData = try JSONSerialization.data(withJSONObject: manifest, options: [.prettyPrinted, .sortedKeys])
|
let manifestData = try JSONSerialization.data(withJSONObject: manifest, options: [.prettyPrinted, .sortedKeys])
|
||||||
try manifestData.write(to: manifestURL)
|
try manifestData.write(to: manifestURL)
|
||||||
fputs("Wrote \(manifestURL.path)\n", stderr)
|
fputs("Wrote \(manifestURL.path)\n", stderr)
|
||||||
|
|
||||||
let sourceManifestURL = parent.appendingPathComponent("manifest.json")
|
|
||||||
let sourceManifest: [String: Any] = [
|
|
||||||
"version": "v1",
|
|
||||||
"generated_at": ISO8601DateFormatter().string(from: Date()),
|
|
||||||
"locale": "zh-Hans",
|
|
||||||
"entry_count": merged.count,
|
|
||||||
"sources": [[
|
|
||||||
"key": "ai_tech_seed",
|
|
||||||
"label": "OSGKeyboard curated AI/tech lexicon",
|
|
||||||
"license": "MIT (curated seed; OSGKeyboard contributors)",
|
|
||||||
"raw_count": aiTech.count,
|
|
||||||
]],
|
|
||||||
"notes": [
|
|
||||||
"Project-curated bilingual AI brands, technology terms, companies, and names.",
|
|
||||||
"No third-party cell dictionaries or Sogou-derived data.",
|
|
||||||
"PhraseCount weights map to SFCustomLanguageModelData relative frequencies.",
|
|
||||||
],
|
|
||||||
"files": ["phrases": "phrases.tsv"],
|
|
||||||
]
|
|
||||||
let sourceManifestData = try JSONSerialization.data(
|
|
||||||
withJSONObject: sourceManifest,
|
|
||||||
options: [.prettyPrinted, .sortedKeys]
|
|
||||||
)
|
|
||||||
try sourceManifestData.write(to: sourceManifestURL)
|
|
||||||
fputs("Wrote \(sourceManifestURL.path)\n", stderr)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,133 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""Parse Sogou .scel cell dictionaries into (word, pinyin) entries.
|
||||||
|
|
||||||
|
Layout follows the classic SCEL format used by imewlconverter / SogouPopularDict.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import struct
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class ScelInfo:
|
||||||
|
word_count: int
|
||||||
|
name: str
|
||||||
|
type_name: str
|
||||||
|
description: str
|
||||||
|
|
||||||
|
|
||||||
|
def _read_uint16(handle) -> int:
|
||||||
|
data = handle.read(2)
|
||||||
|
if not data or len(data) < 2:
|
||||||
|
return 0
|
||||||
|
return struct.unpack("<H", data)[0]
|
||||||
|
|
||||||
|
|
||||||
|
def _read_uint32(handle) -> int:
|
||||||
|
data = handle.read(4)
|
||||||
|
if not data or len(data) < 4:
|
||||||
|
return 0
|
||||||
|
return struct.unpack("<I", data)[0]
|
||||||
|
|
||||||
|
|
||||||
|
def _read_utf16_str(handle, *, offset: int = -1, length: int = 0) -> str:
|
||||||
|
if offset >= 0:
|
||||||
|
handle.seek(offset)
|
||||||
|
if length > 0:
|
||||||
|
data = handle.read(length)
|
||||||
|
end = 0
|
||||||
|
for index in range(0, len(data), 2):
|
||||||
|
if index + 1 < len(data) and data[index] == 0 and data[index + 1] == 0:
|
||||||
|
end = index
|
||||||
|
break
|
||||||
|
if end > 0:
|
||||||
|
data = data[:end]
|
||||||
|
return data.decode("utf-16le", errors="ignore")
|
||||||
|
|
||||||
|
result = bytearray()
|
||||||
|
while True:
|
||||||
|
char = handle.read(2)
|
||||||
|
if not char or len(char) < 2 or (char[0] == 0 and char[1] == 0):
|
||||||
|
break
|
||||||
|
result.extend(char)
|
||||||
|
return result.decode("utf-16le", errors="ignore")
|
||||||
|
|
||||||
|
|
||||||
|
def is_valid_word(word: str) -> bool:
|
||||||
|
if not word or not (1 <= len(word) <= 10):
|
||||||
|
return False
|
||||||
|
allowed_punct = ",。:;?!()【】《》""''、"
|
||||||
|
return all("\u4e00" <= char <= "\u9fff" or char.isdigit() or char in allowed_punct for char in word)
|
||||||
|
|
||||||
|
|
||||||
|
def get_scel_info(scel_path: Path) -> ScelInfo:
|
||||||
|
with scel_path.open("rb") as handle:
|
||||||
|
handle.seek(0x124)
|
||||||
|
word_count = _read_uint32(handle)
|
||||||
|
handle.seek(0x130)
|
||||||
|
name = _read_utf16_str(handle, length=64)
|
||||||
|
handle.seek(0x338)
|
||||||
|
type_name = _read_utf16_str(handle, length=64)
|
||||||
|
handle.seek(0x540)
|
||||||
|
description = _read_utf16_str(handle, length=1024)
|
||||||
|
return ScelInfo(word_count=word_count, name=name, type_name=type_name, description=description)
|
||||||
|
|
||||||
|
|
||||||
|
def parse_scel_file(scel_path: Path) -> list[tuple[str, str]]:
|
||||||
|
"""Return ordered (word, pinyin) pairs from a .scel file."""
|
||||||
|
entries: list[tuple[str, str]] = []
|
||||||
|
|
||||||
|
with scel_path.open("rb") as handle:
|
||||||
|
handle.seek(0x1540)
|
||||||
|
pinyin_count = _read_uint32(handle)
|
||||||
|
pinyin_dict: dict[int, str] = {}
|
||||||
|
for _ in range(pinyin_count):
|
||||||
|
pinyin_idx = _read_uint16(handle)
|
||||||
|
pinyin_len = _read_uint16(handle)
|
||||||
|
pinyin = handle.read(pinyin_len).decode("utf-16le", errors="ignore").strip().lower()
|
||||||
|
pinyin_dict[pinyin_idx] = pinyin
|
||||||
|
|
||||||
|
try:
|
||||||
|
while True:
|
||||||
|
same_pinyin_count = _read_uint16(handle)
|
||||||
|
pinyin_index_len = _read_uint16(handle)
|
||||||
|
if pinyin_index_len <= 0 or same_pinyin_count <= 0:
|
||||||
|
break
|
||||||
|
|
||||||
|
pinyin_parts: list[str] = []
|
||||||
|
for _ in range(pinyin_index_len // 2):
|
||||||
|
idx = _read_uint16(handle)
|
||||||
|
part = pinyin_dict.get(idx, "")
|
||||||
|
if part:
|
||||||
|
pinyin_parts.append(part)
|
||||||
|
joined_pinyin = " ".join(pinyin_parts).strip()
|
||||||
|
|
||||||
|
for _ in range(same_pinyin_count):
|
||||||
|
word_len = _read_uint16(handle)
|
||||||
|
word = handle.read(word_len).decode("utf-16le", errors="ignore")
|
||||||
|
_ = _read_uint16(handle)
|
||||||
|
_ = _read_uint32(handle)
|
||||||
|
_ = handle.read(6)
|
||||||
|
if is_valid_word(word):
|
||||||
|
entries.append((word, joined_pinyin))
|
||||||
|
except (struct.error, OSError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
return entries
|
||||||
|
|
||||||
|
|
||||||
|
def load_pinyin_tsv(tsv_path: Path) -> list[tuple[str, str]]:
|
||||||
|
entries: list[tuple[str, str]] = []
|
||||||
|
with tsv_path.open(encoding="utf-8") as handle:
|
||||||
|
for raw_line in handle:
|
||||||
|
line = raw_line.strip()
|
||||||
|
if not line:
|
||||||
|
continue
|
||||||
|
word, _, pinyin = line.partition("\t")
|
||||||
|
if word and pinyin:
|
||||||
|
entries.append((word, pinyin.strip()))
|
||||||
|
return entries
|
||||||
@@ -83,7 +83,7 @@ LocalASRBiasAdapter.adapt(
|
|||||||
| `hardHotwords` | 为具备 hard-hotword capability 的 backend 保留;当前 Qwen3 MLX 不使用 |
|
| `hardHotwords` | 为具备 hard-hotword capability 的 backend 保留;当前 Qwen3 MLX 不使用 |
|
||||||
|
|
||||||
个人词优先;`BuiltinLexiconIndex` 从 `phrases.tsv` 选择 `weight >= 4` 的 Top-N,
|
个人词优先;`BuiltinLexiconIndex` 从 `phrases.tsv` 选择 `weight >= 4` 的 Top-N,
|
||||||
代码编辑器/终端前台场景优先项目维护的 `ai_tech_seed`。默认最多考虑 300 个内置 ASR 词,
|
代码编辑器/终端前台场景优先 `computer_terms`。默认最多考虑 300 个内置 ASR 词,
|
||||||
Qwen3 soft prompt 最长 800 字符,润色补充最多 40 个内置词。
|
Qwen3 soft prompt 最长 800 字符,润色补充最多 40 个内置词。
|
||||||
|
|
||||||
### 2.3 Apple Speech fallback
|
### 2.3 Apple Speech fallback
|
||||||
|
|||||||
Reference in New Issue
Block a user