Rocky
13214e6601
Rebuild the bundled language model and runtime source filtering around a dedicated computer-term corpus so technical dictation keeps domain coverage.
131 lines
4.2 KiB
Python
131 lines
4.2 KiB
Python
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
"""Build OSGKeyboard domain ASR lexicon v1 from the computer-terms scel source.
|
|
|
|
Sources:
|
|
Local 计算机词汇大全【官方推荐】.scel (IT / computer vocabulary only)
|
|
|
|
Casual network slang and Sogou popular-word dumps are intentionally excluded —
|
|
they dilute custom LM phrase biasing without improving domain ASR accuracy.
|
|
|
|
Output:
|
|
OSGKeyboard/Resources/CustomLanguageModel/v1/phrases.tsv
|
|
OSGKeyboard/Resources/CustomLanguageModel/v1/manifest.json
|
|
|
|
The compiled .bin asset is exported separately to
|
|
OSGKeyboard/Resources/CustomLanguageModel/v1/ via export_clm.swift.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import sys
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
from scel_parser import get_scel_info, parse_scel_file
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
DEFAULT_OUTPUT_DIR = REPO_ROOT / "OSGKeyboard/Resources/CustomLanguageModel/v1"
|
|
DEFAULT_COMPUTER_SCEL = Path("/Users/rocky/Downloads/计算机词汇大全【官方推荐】.scel")
|
|
|
|
SOURCE_KEY = "computer_terms"
|
|
SOURCE_LABEL = "计算机词汇大全【官方推荐】"
|
|
SOURCE_WEIGHT = 5
|
|
|
|
|
|
@dataclass
|
|
class LexiconEntry:
|
|
word: str
|
|
pinyin: str
|
|
source: str
|
|
weight: int
|
|
|
|
|
|
def merge_entries(entries: list[tuple[str, str]]) -> list[LexiconEntry]:
|
|
merged: dict[str, LexiconEntry] = {}
|
|
|
|
for word, pinyin in entries:
|
|
current = merged.get(word)
|
|
candidate = LexiconEntry(
|
|
word=word,
|
|
pinyin=pinyin,
|
|
source=SOURCE_KEY,
|
|
weight=SOURCE_WEIGHT,
|
|
)
|
|
if current is None:
|
|
merged[word] = candidate
|
|
elif not current.pinyin and pinyin:
|
|
merged[word] = candidate
|
|
|
|
return sorted(merged.values(), key=lambda item: item.word)
|
|
|
|
|
|
def write_outputs(entries: list[LexiconEntry], output_dir: Path, raw_count: int) -> None:
|
|
output_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
phrases_path = output_dir / "phrases.tsv"
|
|
with phrases_path.open("w", encoding="utf-8") as handle:
|
|
handle.write("word\tpinyin\tsource\tweight\n")
|
|
for entry in entries:
|
|
handle.write(f"{entry.word}\t{entry.pinyin}\t{entry.source}\t{entry.weight}\n")
|
|
|
|
manifest = {
|
|
"version": "v1",
|
|
"generated_at": datetime.now(timezone.utc).isoformat(),
|
|
"locale": "zh-Hans",
|
|
"entry_count": len(entries),
|
|
"sources": [
|
|
{
|
|
"key": SOURCE_KEY,
|
|
"label": SOURCE_LABEL,
|
|
"weight": SOURCE_WEIGHT,
|
|
"raw_count": raw_count,
|
|
}
|
|
],
|
|
"notes": [
|
|
"Domain-specific computer/IT vocabulary only; casual network slang removed.",
|
|
"PhraseCount weights map to SFCustomLanguageModelData relative frequencies.",
|
|
"Merged with ai-tech-brands seed at export time for the final .bin asset.",
|
|
],
|
|
"files": {
|
|
"phrases": phrases_path.name,
|
|
},
|
|
}
|
|
|
|
manifest_path = output_dir / "manifest.json"
|
|
manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
|
|
|
|
def build(*, computer_scel: Path, output_dir: Path) -> int:
|
|
if not computer_scel.exists():
|
|
print(f"Missing computer scel: {computer_scel}", file=sys.stderr)
|
|
return 1
|
|
|
|
computer_info = get_scel_info(computer_scel)
|
|
print(f"Computer dict: {computer_info.name} ({computer_info.word_count} header count)")
|
|
|
|
raw_entries = parse_scel_file(computer_scel)
|
|
merged = merge_entries(raw_entries)
|
|
write_outputs(merged, output_dir, raw_count=len(raw_entries))
|
|
|
|
print(f"Raw count: {len(raw_entries)}")
|
|
print(f"Merged unique entries: {len(merged)}")
|
|
print(f"Wrote {output_dir / 'phrases.tsv'}")
|
|
print(f"Wrote {output_dir / 'manifest.json'}")
|
|
return 0
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description="Build OSGKeyboard domain ASR lexicon v1")
|
|
parser.add_argument("--computer-scel", type=Path, default=DEFAULT_COMPUTER_SCEL)
|
|
parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT_DIR)
|
|
args = parser.parse_args()
|
|
return build(computer_scel=args.computer_scel, output_dir=args.output_dir)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|