fix(release): prepare 1.8 disclosures and licensed resources

Replace the unclear CLM source, align release and privacy disclosures, complete third-party notices, and keep the expanded test suites deterministic for release validation.
2026-08-14 23:28:11 +08:00
parent 7364659027
commit f42a812256
36 changed files with 3950 additions and 10851 deletions
@@ -1,130 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""Build OSGKeyboard domain ASR lexicon v1 from the computer-terms scel source.
-
-Sources:
-  Local 计算机词汇大全【官方推荐】.scel (IT / computer vocabulary only)
-
-Casual network slang and Sogou popular-word dumps are intentionally excluded —
-they dilute custom LM phrase biasing without improving domain ASR accuracy.
-
-Output:
-  OSGKeyboard/Resources/CustomLanguageModel/v1/phrases.tsv
-  OSGKeyboard/Resources/CustomLanguageModel/v1/manifest.json
-
-The compiled .bin asset is exported separately to
-OSGKeyboard/Resources/CustomLanguageModel/v1/ via export_clm.swift.
-"""
-
-from __future__ import annotations
-
-import argparse
-import json
-import sys
-from dataclasses import dataclass
-from datetime import datetime, timezone
-from pathlib import Path
-
-from scel_parser import get_scel_info, parse_scel_file
-
-REPO_ROOT = Path(__file__).resolve().parents[2]
-DEFAULT_OUTPUT_DIR = REPO_ROOT / "OSGKeyboard/Resources/CustomLanguageModel/v1"
-DEFAULT_COMPUTER_SCEL = Path("/Users/rocky/Downloads/计算机词汇大全【官方推荐】.scel")
-
-SOURCE_KEY = "computer_terms"
-SOURCE_LABEL = "计算机词汇大全【官方推荐】"
-SOURCE_WEIGHT = 5
-
-
-@dataclass
-class LexiconEntry:
-    word: str
-    pinyin: str
-    source: str
-    weight: int
-
-
-def merge_entries(entries: list[tuple[str, str]]) -> list[LexiconEntry]:
-    merged: dict[str, LexiconEntry] = {}
-
-    for word, pinyin in entries:
-        current = merged.get(word)
-        candidate = LexiconEntry(
-            word=word,
-            pinyin=pinyin,
-            source=SOURCE_KEY,
-            weight=SOURCE_WEIGHT,
-        )
-        if current is None:
-            merged[word] = candidate
-        elif not current.pinyin and pinyin:
-            merged[word] = candidate
-
-    return sorted(merged.values(), key=lambda item: item.word)
-
-
-def write_outputs(entries: list[LexiconEntry], output_dir: Path, raw_count: int) -> None:
-    output_dir.mkdir(parents=True, exist_ok=True)
-
-    phrases_path = output_dir / "phrases.tsv"
-    with phrases_path.open("w", encoding="utf-8") as handle:
-        handle.write("word\tpinyin\tsource\tweight\n")
-        for entry in entries:
-            handle.write(f"{entry.word}\t{entry.pinyin}\t{entry.source}\t{entry.weight}\n")
-
-    manifest = {
-        "version": "v1",
-        "generated_at": datetime.now(timezone.utc).isoformat(),
-        "locale": "zh-Hans",
-        "entry_count": len(entries),
-        "sources": [
-            {
-                "key": SOURCE_KEY,
-                "label": SOURCE_LABEL,
-                "weight": SOURCE_WEIGHT,
-                "raw_count": raw_count,
-            }
-        ],
-        "notes": [
-            "Domain-specific computer/IT vocabulary only; casual network slang removed.",
-            "PhraseCount weights map to SFCustomLanguageModelData relative frequencies.",
-            "Merged with ai-tech-brands seed at export time for the final .bin asset.",
-        ],
-        "files": {
-            "phrases": phrases_path.name,
-        },
-    }
-
-    manifest_path = output_dir / "manifest.json"
-    manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
-
-
-def build(*, computer_scel: Path, output_dir: Path) -> int:
-    if not computer_scel.exists():
-        print(f"Missing computer scel: {computer_scel}", file=sys.stderr)
-        return 1
-
-    computer_info = get_scel_info(computer_scel)
-    print(f"Computer dict: {computer_info.name} ({computer_info.word_count} header count)")
-
-    raw_entries = parse_scel_file(computer_scel)
-    merged = merge_entries(raw_entries)
-    write_outputs(merged, output_dir, raw_count=len(raw_entries))
-
-    print(f"Raw count: {len(raw_entries)}")
-    print(f"Merged unique entries: {len(merged)}")
-    print(f"Wrote {output_dir / 'phrases.tsv'}")
-    print(f"Wrote {output_dir / 'manifest.json'}")
-    return 0
-
-
-def main() -> int:
-    parser = argparse.ArgumentParser(description="Build OSGKeyboard domain ASR lexicon v1")
-    parser.add_argument("--computer-scel", type=Path, default=DEFAULT_COMPUTER_SCEL)
-    parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT_DIR)
-    args = parser.parse_args()
-    return build(computer_scel=args.computer_scel, output_dir=args.output_dir)
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())