feat(lexicon): add custom ASR language model v1 (Sogou + AI/tech brands)
Build a first-version SFCustomLanguageModelData for on-device ASR customization, generated offline on macOS (no app code dependency). - Scripts/lexicon: Python builders for Sogou-derived phrases (scel + SogouPopularDict) and a curated bilingual AI/tech/brand seed lexicon - export_clm.swift / prepare_clm.swift: macOS CLI tools that export the .bin training asset and compile it into LM + Vocab via the Speech framework - Resources/CustomLanguageModel: generated phrases.tsv, manifests, the 129k-phrase .bin, and compiled LM/Vocab assets (zh_CN) - .gitignore: ignore lexicon build cache and Python bytecode Note: Sogou-derived data is for internal experimentation only; the curated AI/tech seed is MIT and safe to ship.
This commit is contained in:
Rocky
Executable
+9
@@ -0,0 +1,9 @@
|
||||
#!/usr/bin/env bash
|
||||
# Full offline CLM pipeline on macOS:
|
||||
# 1) export .bin from TSVs
|
||||
# 2) prepare compiled LM + Vocab
|
||||
set -euo pipefail
|
||||
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||
cd "$ROOT"
|
||||
"$ROOT/Scripts/lexicon/build-clm-bin.sh" "$@"
|
||||
"$ROOT/Scripts/lexicon/prepare-clm.sh" "$@"
|
||||
Executable
+6
@@ -0,0 +1,6 @@
|
||||
#!/usr/bin/env bash
|
||||
# Build SFCustomLanguageModelData .bin on macOS (requires Speech framework).
|
||||
set -euo pipefail
|
||||
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||
cd "$ROOT"
|
||||
exec swift "$ROOT/Scripts/lexicon/export_clm.swift" "$@"
|
||||
@@ -0,0 +1,177 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Build the curated AI / tech / brand seed lexicon for OSGKeyboard ASR.
|
||||
|
||||
Reads Scripts/lexicon/seeds/ai_tech_brands_seed.tsv and emits:
|
||||
OSGKeyboard/Resources/CustomLanguageModel/ai-tech-brands/v1/phrases.tsv
|
||||
OSGKeyboard/Resources/CustomLanguageModel/ai-tech-brands/v1/manifest.json
|
||||
|
||||
Each seed row may declare pipe-separated aliases; aliases are expanded into
|
||||
additional phrase rows sharing the same category and weight.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from collections import Counter
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
DEFAULT_SEED = REPO_ROOT / "Scripts/lexicon/seeds/ai_tech_brands_seed.tsv"
|
||||
DEFAULT_OUTPUT = REPO_ROOT / "OSGKeyboard/Resources/CustomLanguageModel/ai-tech-brands/v1"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SeedRow:
|
||||
word: str
|
||||
pinyin: str
|
||||
aliases: tuple[str, ...]
|
||||
category: str
|
||||
weight: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PhraseRow:
|
||||
word: str
|
||||
pinyin: str
|
||||
source: str
|
||||
category: str
|
||||
weight: int
|
||||
canonical: str
|
||||
|
||||
|
||||
def parse_seed_file(seed_path: Path) -> list[SeedRow]:
|
||||
rows: list[SeedRow] = []
|
||||
with seed_path.open(encoding="utf-8") as handle:
|
||||
for line_number, raw_line in enumerate(handle, start=1):
|
||||
line = raw_line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
parts = line.split("\t")
|
||||
if len(parts) < 4:
|
||||
print(f"Warning: skip malformed line {line_number}: {line}", file=sys.stderr)
|
||||
continue
|
||||
word = parts[0].strip()
|
||||
pinyin = parts[1].strip() if len(parts) > 1 else ""
|
||||
aliases_raw = parts[2].strip() if len(parts) > 2 else ""
|
||||
category = parts[3].strip() if len(parts) > 3 else "misc"
|
||||
weight_raw = parts[4].strip() if len(parts) > 4 else "80"
|
||||
if not word:
|
||||
continue
|
||||
aliases = tuple(
|
||||
alias.strip()
|
||||
for alias in aliases_raw.split("|")
|
||||
if alias.strip() and alias.strip() != word
|
||||
)
|
||||
try:
|
||||
weight = int(weight_raw)
|
||||
except ValueError:
|
||||
weight = 80
|
||||
rows.append(
|
||||
SeedRow(
|
||||
word=word,
|
||||
pinyin=pinyin,
|
||||
aliases=aliases,
|
||||
category=category,
|
||||
weight=weight,
|
||||
)
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def expand_rows(seeds: list[SeedRow]) -> list[PhraseRow]:
|
||||
"""Expand canonical + aliases; dedupe by word keeping highest weight."""
|
||||
merged: dict[str, PhraseRow] = {}
|
||||
|
||||
for seed in seeds:
|
||||
candidates = [(seed.word, seed.pinyin, seed.category, seed.weight, seed.word)]
|
||||
for alias in seed.aliases:
|
||||
# Aliases inherit canonical pinyin only when alias is Chinese.
|
||||
alias_pinyin = seed.pinyin if _contains_cjk(alias) else ""
|
||||
candidates.append((alias, alias_pinyin, seed.category, seed.weight, seed.word))
|
||||
|
||||
for word, pinyin, category, weight, canonical in candidates:
|
||||
if not word:
|
||||
continue
|
||||
row = PhraseRow(
|
||||
word=word,
|
||||
pinyin=pinyin,
|
||||
source="ai_tech_seed",
|
||||
category=category,
|
||||
weight=weight,
|
||||
canonical=canonical,
|
||||
)
|
||||
current = merged.get(word)
|
||||
if current is None or row.weight > current.weight:
|
||||
merged[word] = row
|
||||
|
||||
return sorted(merged.values(), key=lambda item: (-item.weight, item.word.lower()))
|
||||
|
||||
|
||||
def _contains_cjk(text: str) -> bool:
|
||||
return any("\u4e00" <= char <= "\u9fff" for char in text)
|
||||
|
||||
|
||||
def write_outputs(phrases: list[PhraseRow], output_dir: Path, seed_path: Path) -> None:
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
phrases_path = output_dir / "phrases.tsv"
|
||||
with phrases_path.open("w", encoding="utf-8") as handle:
|
||||
handle.write("word\tpinyin\tsource\tcategory\tweight\tcanonical\n")
|
||||
for row in phrases:
|
||||
handle.write(
|
||||
f"{row.word}\t{row.pinyin}\t{row.source}\t{row.category}\t{row.weight}\t{row.canonical}\n"
|
||||
)
|
||||
|
||||
category_counts = Counter(row.category for row in phrases)
|
||||
manifest = {
|
||||
"version": "v1",
|
||||
"name": "ai-tech-brands",
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
"locale": "zh-Hans",
|
||||
"entry_count": len(phrases),
|
||||
"seed_file": str(seed_path.relative_to(REPO_ROOT)),
|
||||
"license": "MIT (curated seed; OSGKeyboard contributors)",
|
||||
"categories": dict(sorted(category_counts.items())),
|
||||
"notes": [
|
||||
"Curated bilingual AI brands, tech companies, terminology, and hot words.",
|
||||
"English canonical forms + Chinese aliases for ASR PhraseCount weighting.",
|
||||
"Aliases expanded at build time; canonical column tracks the primary form.",
|
||||
"English post-processing (casing) remains LLM polish responsibility.",
|
||||
],
|
||||
"files": {"phrases": phrases_path.name},
|
||||
}
|
||||
manifest_path = output_dir / "manifest.json"
|
||||
manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def build(seed_path: Path, output_dir: Path) -> int:
|
||||
if not seed_path.exists():
|
||||
print(f"Missing seed file: {seed_path}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
seeds = parse_seed_file(seed_path)
|
||||
phrases = expand_rows(seeds)
|
||||
write_outputs(phrases, output_dir, seed_path)
|
||||
|
||||
print(f"Seed rows: {len(seeds)}")
|
||||
print(f"Expanded unique phrases: {len(phrases)}")
|
||||
print(f"Wrote {output_dir / 'phrases.tsv'}")
|
||||
print(f"Wrote {output_dir / 'manifest.json'}")
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Build AI/tech brand seed lexicon")
|
||||
parser.add_argument("--seed", type=Path, default=DEFAULT_SEED)
|
||||
parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT)
|
||||
args = parser.parse_args()
|
||||
return build(args.seed, args.output_dir)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,184 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Build OSGKeyboard custom ASR lexicon v1 from Sogou-derived sources.
|
||||
|
||||
Sources (experimentation only — Sogou data is non-commercial):
|
||||
1. ASC8384/SogouPopularDict accumulated pinyin TSV
|
||||
2. Local 计算机词汇大全【官方推荐】.scel
|
||||
3. Local 网络流行新词.scel
|
||||
|
||||
Output:
|
||||
OSGKeyboard/Resources/CustomLanguageModel/v1/phrases.tsv
|
||||
OSGKeyboard/Resources/CustomLanguageModel/v1/manifest.json
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import urllib.request
|
||||
from collections import Counter
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from scel_parser import get_scel_info, load_pinyin_tsv, parse_scel_file
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
DEFAULT_OUTPUT_DIR = REPO_ROOT / "OSGKeyboard/Resources/CustomLanguageModel/v1"
|
||||
SOGOU_ACCUMULATED_URL = (
|
||||
"https://raw.githubusercontent.com/ASC8384/SogouPopularDict/main/"
|
||||
"data/sogou_network_words_accumulated_pinyin.tsv"
|
||||
)
|
||||
|
||||
DEFAULT_COMPUTER_SCEL = Path("/Users/rocky/Downloads/计算机词汇大全【官方推荐】.scel")
|
||||
DEFAULT_NETWORK_SCEL = Path("/Users/rocky/Downloads/网络流行新词.scel")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SourceSpec:
|
||||
key: str
|
||||
label: str
|
||||
weight: int
|
||||
|
||||
|
||||
SOURCES = [
|
||||
SourceSpec("computer_terms", "计算机词汇大全【官方推荐】", weight=5),
|
||||
SourceSpec("network_slang_local", "网络流行新词.scel", weight=3),
|
||||
SourceSpec("sogou_popular_accumulated", "SogouPopularDict accumulated", weight=1),
|
||||
]
|
||||
|
||||
|
||||
def download_accumulated_tsv(destination: Path) -> None:
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
with urllib.request.urlopen(SOGOU_ACCUMULATED_URL, timeout=120) as response:
|
||||
destination.write_bytes(response.read())
|
||||
|
||||
|
||||
@dataclass
|
||||
class LexiconEntry:
|
||||
word: str
|
||||
pinyin: str
|
||||
source: str
|
||||
weight: int
|
||||
|
||||
|
||||
def merge_entries(sources: list[tuple[SourceSpec, list[tuple[str, str]]]]) -> list[LexiconEntry]:
|
||||
merged: dict[str, LexiconEntry] = {}
|
||||
source_counts: Counter[str] = Counter()
|
||||
|
||||
for spec, entries in sources:
|
||||
for word, pinyin in entries:
|
||||
source_counts[spec.key] += 1
|
||||
current = merged.get(word)
|
||||
candidate = LexiconEntry(word=word, pinyin=pinyin, source=spec.key, weight=spec.weight)
|
||||
if current is None or candidate.weight > current.weight:
|
||||
merged[word] = candidate
|
||||
elif current.weight == candidate.weight and not current.pinyin and pinyin:
|
||||
merged[word] = candidate
|
||||
|
||||
return sorted(merged.values(), key=lambda item: (item.weight * -1, item.word))
|
||||
|
||||
|
||||
def write_outputs(entries: list[LexiconEntry], output_dir: Path, source_stats: dict[str, int]) -> None:
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
phrases_path = output_dir / "phrases.tsv"
|
||||
with phrases_path.open("w", encoding="utf-8") as handle:
|
||||
handle.write("word\tpinyin\tsource\tweight\n")
|
||||
for entry in entries:
|
||||
handle.write(f"{entry.word}\t{entry.pinyin}\t{entry.source}\t{entry.weight}\n")
|
||||
|
||||
manifest = {
|
||||
"version": "v1",
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
"locale": "zh-Hans",
|
||||
"entry_count": len(entries),
|
||||
"sources": [
|
||||
{
|
||||
"key": spec.key,
|
||||
"label": spec.label,
|
||||
"weight": spec.weight,
|
||||
"raw_count": source_stats.get(spec.key, 0),
|
||||
}
|
||||
for spec in SOURCES
|
||||
],
|
||||
"notes": [
|
||||
"Sogou-derived data is for internal ASR experimentation only.",
|
||||
"PhraseCount weights map to SFCustomLanguageModelData relative frequencies.",
|
||||
"Higher source weight wins on duplicate words.",
|
||||
],
|
||||
"files": {
|
||||
"phrases": phrases_path.name,
|
||||
},
|
||||
}
|
||||
|
||||
manifest_path = output_dir / "manifest.json"
|
||||
manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def build(
|
||||
*,
|
||||
computer_scel: Path,
|
||||
network_scel: Path,
|
||||
output_dir: Path,
|
||||
skip_download: bool,
|
||||
) -> int:
|
||||
cache_dir = REPO_ROOT / ".cache/lexicon"
|
||||
cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
accumulated_tsv = cache_dir / "sogou_network_words_accumulated_pinyin.tsv"
|
||||
|
||||
if not skip_download and not accumulated_tsv.exists():
|
||||
print(f"Downloading {SOGOU_ACCUMULATED_URL} …")
|
||||
download_accumulated_tsv(accumulated_tsv)
|
||||
elif not accumulated_tsv.exists():
|
||||
print(f"Missing accumulated TSV: {accumulated_tsv}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
if not computer_scel.exists():
|
||||
print(f"Missing computer scel: {computer_scel}", file=sys.stderr)
|
||||
return 1
|
||||
if not network_scel.exists():
|
||||
print(f"Missing network scel: {network_scel}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
computer_info = get_scel_info(computer_scel)
|
||||
network_info = get_scel_info(network_scel)
|
||||
print(f"Computer dict: {computer_info.name} ({computer_info.word_count} header count)")
|
||||
print(f"Network dict: {network_info.name} ({network_info.word_count} header count)")
|
||||
|
||||
loaded_sources: list[tuple[SourceSpec, list[tuple[str, str]]]] = [
|
||||
(SOURCES[0], parse_scel_file(computer_scel)),
|
||||
(SOURCES[1], parse_scel_file(network_scel)),
|
||||
(SOURCES[2], load_pinyin_tsv(accumulated_tsv)),
|
||||
]
|
||||
|
||||
source_stats = {spec.key: len(entries) for spec, entries in loaded_sources}
|
||||
merged = merge_entries(loaded_sources)
|
||||
write_outputs(merged, output_dir, source_stats)
|
||||
|
||||
print(f"Raw counts: {source_stats}")
|
||||
print(f"Merged unique entries: {len(merged)}")
|
||||
print(f"Wrote {output_dir / 'phrases.tsv'}")
|
||||
print(f"Wrote {output_dir / 'manifest.json'}")
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Build OSGKeyboard custom ASR lexicon v1")
|
||||
parser.add_argument("--computer-scel", type=Path, default=DEFAULT_COMPUTER_SCEL)
|
||||
parser.add_argument("--network-scel", type=Path, default=DEFAULT_NETWORK_SCEL)
|
||||
parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT_DIR)
|
||||
parser.add_argument("--skip-download", action="store_true")
|
||||
args = parser.parse_args()
|
||||
return build(
|
||||
computer_scel=args.computer_scel,
|
||||
network_scel=args.network_scel,
|
||||
output_dir=args.output_dir,
|
||||
skip_download=args.skip_download,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,272 @@
|
||||
#!/usr/bin/env swift
|
||||
//
|
||||
// export_clm.swift
|
||||
// OSGKeyboard · offline SFCustomLanguageModelData exporter (macOS 14+)
|
||||
//
|
||||
// Reads merged phrase TSVs and writes a .bin training asset via Speech framework.
|
||||
// Usage:
|
||||
// swift Scripts/lexicon/export_clm.swift
|
||||
// swift Scripts/lexicon/export_clm.swift --max-entries 30000
|
||||
//
|
||||
|
||||
import Foundation
|
||||
import Speech
|
||||
|
||||
// MARK: - CLI
|
||||
|
||||
struct CLIOptions {
|
||||
var sogouTSV: URL
|
||||
var aiTechTSV: URL
|
||||
var outputBin: URL
|
||||
var localeID: String
|
||||
var modelID: String
|
||||
var modelVersion: String
|
||||
var maxEntries: Int?
|
||||
|
||||
static func parse() -> CLIOptions {
|
||||
let repoRoot = URL(fileURLWithPath: #filePath)
|
||||
.deletingLastPathComponent()
|
||||
.deletingLastPathComponent()
|
||||
.deletingLastPathComponent()
|
||||
|
||||
var sogou = repoRoot.appendingPathComponent(
|
||||
"OSGKeyboard/Resources/CustomLanguageModel/v1/phrases.tsv"
|
||||
)
|
||||
var aiTech = repoRoot.appendingPathComponent(
|
||||
"OSGKeyboard/Resources/CustomLanguageModel/ai-tech-brands/v1/phrases.tsv"
|
||||
)
|
||||
var output = repoRoot.appendingPathComponent(
|
||||
"OSGKeyboard/Resources/CustomLanguageModel/v1/OSGKeyboardCLM.bin"
|
||||
)
|
||||
var localeID = "zh_CN"
|
||||
var modelID = "com.osgkeyboard.custom-lm.v1"
|
||||
var modelVersion = "1.0.0"
|
||||
var maxEntries: Int?
|
||||
|
||||
var iterator = CommandLine.arguments.dropFirst().makeIterator()
|
||||
while let flag = iterator.next() {
|
||||
switch flag {
|
||||
case "--sogou-tsv":
|
||||
sogou = URL(fileURLWithPath: iterator.next() ?? "")
|
||||
case "--ai-tech-tsv":
|
||||
aiTech = URL(fileURLWithPath: iterator.next() ?? "")
|
||||
case "--output":
|
||||
output = URL(fileURLWithPath: iterator.next() ?? "")
|
||||
case "--locale":
|
||||
localeID = iterator.next() ?? localeID
|
||||
case "--identifier":
|
||||
modelID = iterator.next() ?? modelID
|
||||
case "--version":
|
||||
modelVersion = iterator.next() ?? modelVersion
|
||||
case "--max-entries":
|
||||
maxEntries = Int(iterator.next() ?? "")
|
||||
case "-h", "--help":
|
||||
printUsage()
|
||||
exit(0)
|
||||
default:
|
||||
fputs("Unknown flag: \(flag)\n", stderr)
|
||||
printUsage()
|
||||
exit(2)
|
||||
}
|
||||
}
|
||||
|
||||
return CLIOptions(
|
||||
sogouTSV: sogou,
|
||||
aiTechTSV: aiTech,
|
||||
outputBin: output,
|
||||
localeID: localeID,
|
||||
modelID: modelID,
|
||||
modelVersion: modelVersion,
|
||||
maxEntries: maxEntries
|
||||
)
|
||||
}
|
||||
|
||||
static func printUsage() {
|
||||
print("""
|
||||
export_clm.swift — build SFCustomLanguageModelData .bin on macOS
|
||||
|
||||
Options:
|
||||
--sogou-tsv <path> Sogou merged phrases TSV
|
||||
--ai-tech-tsv <path> AI/tech seed phrases TSV
|
||||
--output <path> Output .bin path
|
||||
--locale <id> Locale identifier (default: zh_CN)
|
||||
--identifier <id> Custom LM identifier
|
||||
--version <ver> Custom LM version string
|
||||
--max-entries <n> Optional cap for smoke tests
|
||||
-h, --help Show help
|
||||
""")
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - TSV parsing
|
||||
|
||||
struct PhraseEntry: Hashable {
|
||||
let phrase: String
|
||||
let weight: Int
|
||||
let source: String
|
||||
}
|
||||
|
||||
enum TSVLoader {
|
||||
static func load(from url: URL, sourceLabel: String) throws -> [PhraseEntry] {
|
||||
let text = try String(contentsOf: url, encoding: .utf8)
|
||||
var entries: [PhraseEntry] = []
|
||||
|
||||
for (index, rawLine) in text.split(whereSeparator: \.isNewline).enumerated() {
|
||||
let line = String(rawLine)
|
||||
if index == 0, line.lowercased().hasPrefix("word\t") {
|
||||
continue
|
||||
}
|
||||
if line.isEmpty || line.hasPrefix("#") {
|
||||
continue
|
||||
}
|
||||
|
||||
let parts = line.split(separator: "\t", omittingEmptySubsequences: false).map(String.init)
|
||||
guard let word = parts.first?.trimmingCharacters(in: .whitespacesAndNewlines), !word.isEmpty else {
|
||||
continue
|
||||
}
|
||||
|
||||
// Formats:
|
||||
// sogou: word, pinyin, source, weight
|
||||
// ai-tech: word, pinyin, source, category, weight, canonical
|
||||
let weight: Int
|
||||
if parts.count >= 6, let parsed = Int(parts[4]) {
|
||||
weight = parsed
|
||||
} else if parts.count >= 4, let parsed = Int(parts[3]) {
|
||||
weight = parsed
|
||||
} else {
|
||||
weight = 1
|
||||
}
|
||||
|
||||
let source = parts.count >= 3 ? parts[2] : sourceLabel
|
||||
entries.append(PhraseEntry(phrase: word, weight: max(1, weight), source: source))
|
||||
}
|
||||
|
||||
return entries
|
||||
}
|
||||
|
||||
static func merge(_ batches: [[PhraseEntry]]) -> [PhraseEntry] {
|
||||
var merged: [String: PhraseEntry] = [:]
|
||||
for batch in batches {
|
||||
for entry in batch {
|
||||
if let current = merged[entry.phrase] {
|
||||
if entry.weight >= current.weight {
|
||||
merged[entry.phrase] = entry
|
||||
}
|
||||
} else {
|
||||
merged[entry.phrase] = entry
|
||||
}
|
||||
}
|
||||
}
|
||||
return merged.values.sorted {
|
||||
if $0.weight != $1.weight { return $0.weight > $1.weight }
|
||||
return $0.phrase < $1.phrase
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Export
|
||||
|
||||
enum ExportCLM {
|
||||
static func run() async throws {
|
||||
let options = CLIOptions.parse()
|
||||
let fm = FileManager.default
|
||||
|
||||
guard fm.fileExists(atPath: options.sogouTSV.path) else {
|
||||
throw ExportError.missingInput(options.sogouTSV.path)
|
||||
}
|
||||
guard fm.fileExists(atPath: options.aiTechTSV.path) else {
|
||||
throw ExportError.missingInput(options.aiTechTSV.path)
|
||||
}
|
||||
|
||||
fputs("Loading phrases…\n", stderr)
|
||||
let sogou = try TSVLoader.load(from: options.sogouTSV, sourceLabel: "sogou_v1")
|
||||
let aiTech = try TSVLoader.load(from: options.aiTechTSV, sourceLabel: "ai_tech_seed")
|
||||
var merged = TSVLoader.merge([sogou, aiTech])
|
||||
|
||||
if let cap = options.maxEntries, merged.count > cap {
|
||||
merged = Array(merged.prefix(cap))
|
||||
fputs("Capped to \(cap) entries (--max-entries)\n", stderr)
|
||||
}
|
||||
|
||||
fputs(
|
||||
"Merged \(merged.count) unique phrases (sogou=\(sogou.count), ai-tech=\(aiTech.count))\n",
|
||||
stderr
|
||||
)
|
||||
fputs("Locale=\(options.localeID) identifier=\(options.modelID) version=\(options.modelVersion)\n", stderr)
|
||||
|
||||
let locale = Locale(identifier: options.localeID)
|
||||
let started = Date()
|
||||
|
||||
fputs("Building SFCustomLanguageModelData…\n", stderr)
|
||||
let data = SFCustomLanguageModelData(
|
||||
locale: locale,
|
||||
identifier: options.modelID,
|
||||
version: options.modelVersion
|
||||
) {
|
||||
for entry in merged {
|
||||
SFCustomLanguageModelData.PhraseCount(
|
||||
phrase: entry.phrase,
|
||||
count: entry.weight
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
let outputURL = options.outputBin
|
||||
let parent = outputURL.deletingLastPathComponent()
|
||||
try fm.createDirectory(at: parent, withIntermediateDirectories: true)
|
||||
if fm.fileExists(atPath: outputURL.path) {
|
||||
try fm.removeItem(at: outputURL)
|
||||
}
|
||||
|
||||
fputs("Exporting to \(outputURL.path)…\n", stderr)
|
||||
try await data.export(to: outputURL)
|
||||
|
||||
let elapsed = Date().timeIntervalSince(started)
|
||||
let bytes = (try? fm.attributesOfItem(atPath: outputURL.path)[.size] as? NSNumber)?.intValue ?? 0
|
||||
fputs(
|
||||
"Done in \(String(format: "%.1f", elapsed))s — \(outputURL.lastPathComponent) (\(bytes) bytes)\n",
|
||||
stderr
|
||||
)
|
||||
|
||||
let manifestURL = parent.appendingPathComponent("compiled-manifest.json")
|
||||
let manifest: [String: Any] = [
|
||||
"generated_at": ISO8601DateFormatter().string(from: Date()),
|
||||
"locale": options.localeID,
|
||||
"identifier": options.modelID,
|
||||
"version": options.modelVersion,
|
||||
"phrase_count": merged.count,
|
||||
"sources": [
|
||||
"sogou_v1": sogou.count,
|
||||
"ai_tech_seed": aiTech.count,
|
||||
],
|
||||
"bin_file": outputURL.lastPathComponent,
|
||||
"bin_bytes": bytes,
|
||||
"export_seconds": elapsed,
|
||||
]
|
||||
let manifestData = try JSONSerialization.data(withJSONObject: manifest, options: [.prettyPrinted, .sortedKeys])
|
||||
try manifestData.write(to: manifestURL)
|
||||
fputs("Wrote \(manifestURL.path)\n", stderr)
|
||||
}
|
||||
}
|
||||
|
||||
enum ExportError: LocalizedError {
|
||||
case missingInput(String)
|
||||
|
||||
var errorDescription: String? {
|
||||
switch self {
|
||||
case .missingInput(let path):
|
||||
return "Missing input file: \(path)"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Task {
|
||||
do {
|
||||
try await ExportCLM.run()
|
||||
exit(0)
|
||||
} catch {
|
||||
fputs("export_clm failed: \(error)\n", stderr)
|
||||
exit(1)
|
||||
}
|
||||
}
|
||||
dispatchMain()
|
||||
Executable
+6
@@ -0,0 +1,6 @@
|
||||
#!/usr/bin/env bash
|
||||
# Compile SFCustomLanguageModelData .bin into LM + Vocab on macOS.
|
||||
set -euo pipefail
|
||||
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||
cd "$ROOT"
|
||||
exec swift "$ROOT/Scripts/lexicon/prepare_clm.swift" "$@"
|
||||
@@ -0,0 +1,196 @@
|
||||
#!/usr/bin/env swift
|
||||
//
|
||||
// prepare_clm.swift
|
||||
// OSGKeyboard · offline custom language model compiler (macOS 14+)
|
||||
//
|
||||
// Takes a SFCustomLanguageModelData .bin and runs:
|
||||
// SFSpeechLanguageModel.prepareCustomLanguageModel(...)
|
||||
//
|
||||
// Usage:
|
||||
// swift Scripts/lexicon/prepare_clm.swift
|
||||
// swift Scripts/lexicon/prepare_clm.swift --input path/to/OSGKeyboardCLM.bin
|
||||
//
|
||||
|
||||
import Foundation
|
||||
import Speech
|
||||
|
||||
// MARK: - CLI
|
||||
|
||||
struct PrepareOptions {
|
||||
var inputBin: URL
|
||||
var outputDir: URL
|
||||
var clientIdentifier: String
|
||||
var weight: Double?
|
||||
|
||||
static func parse() -> PrepareOptions {
|
||||
let repoRoot = URL(fileURLWithPath: #filePath)
|
||||
.deletingLastPathComponent()
|
||||
.deletingLastPathComponent()
|
||||
.deletingLastPathComponent()
|
||||
|
||||
var input = repoRoot.appendingPathComponent(
|
||||
"OSGKeyboard/Resources/CustomLanguageModel/v1/OSGKeyboardCLM.bin"
|
||||
)
|
||||
var output = repoRoot.appendingPathComponent(
|
||||
"OSGKeyboard/Resources/CustomLanguageModel/v1/compiled"
|
||||
)
|
||||
var clientID = "com.osgkeyboard.custom-lm.v1"
|
||||
var weight: Double?
|
||||
|
||||
var iterator = CommandLine.arguments.dropFirst().makeIterator()
|
||||
while let flag = iterator.next() {
|
||||
switch flag {
|
||||
case "--input":
|
||||
input = URL(fileURLWithPath: iterator.next() ?? "")
|
||||
case "--output-dir":
|
||||
output = URL(fileURLWithPath: iterator.next() ?? "")
|
||||
case "--client-identifier":
|
||||
clientID = iterator.next() ?? clientID
|
||||
case "--weight":
|
||||
weight = Double(iterator.next() ?? "")
|
||||
case "-h", "--help":
|
||||
printUsage()
|
||||
exit(0)
|
||||
default:
|
||||
fputs("Unknown flag: \(flag)\n", stderr)
|
||||
printUsage()
|
||||
exit(2)
|
||||
}
|
||||
}
|
||||
|
||||
return PrepareOptions(
|
||||
inputBin: input,
|
||||
outputDir: output,
|
||||
clientIdentifier: clientID,
|
||||
weight: weight
|
||||
)
|
||||
}
|
||||
|
||||
static func printUsage() {
|
||||
print("""
|
||||
prepare_clm.swift — compile SFCustomLanguageModelData .bin on macOS
|
||||
|
||||
Options:
|
||||
--input <path> Training .bin (default: OSGKeyboardCLM.bin)
|
||||
--output-dir <path> Directory for compiled LM + Vocab
|
||||
--client-identifier <id> Client identifier (default: com.osgkeyboard.custom-lm.v1)
|
||||
--weight <0.0-1.0> Optional customization weight
|
||||
-h, --help Show help
|
||||
""")
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Runner
|
||||
|
||||
enum PrepareCLM {
|
||||
static func run() async throws {
|
||||
let options = PrepareOptions.parse()
|
||||
let fm = FileManager.default
|
||||
|
||||
guard fm.fileExists(atPath: options.inputBin.path) else {
|
||||
throw PrepareError.missingInput(options.inputBin.path)
|
||||
}
|
||||
|
||||
try fm.createDirectory(at: options.outputDir, withIntermediateDirectories: true)
|
||||
|
||||
let languageModelURL = options.outputDir.appendingPathComponent("LM")
|
||||
let vocabularyURL = options.outputDir.appendingPathComponent("Vocab")
|
||||
|
||||
// Remove stale outputs so prepare always starts clean.
|
||||
for url in [languageModelURL, vocabularyURL] {
|
||||
if fm.fileExists(atPath: url.path) {
|
||||
try fm.removeItem(at: url)
|
||||
}
|
||||
}
|
||||
|
||||
let configuration: SFSpeechLanguageModel.Configuration
|
||||
if let weight = options.weight {
|
||||
configuration = SFSpeechLanguageModel.Configuration(
|
||||
languageModel: languageModelURL,
|
||||
vocabulary: vocabularyURL,
|
||||
weight: NSNumber(value: weight)
|
||||
)
|
||||
} else {
|
||||
configuration = SFSpeechLanguageModel.Configuration(
|
||||
languageModel: languageModelURL,
|
||||
vocabulary: vocabularyURL
|
||||
)
|
||||
}
|
||||
|
||||
fputs("Input asset: \(options.inputBin.path)\n", stderr)
|
||||
fputs("Output dir: \(options.outputDir.path)\n", stderr)
|
||||
fputs("Client ID: \(options.clientIdentifier)\n", stderr)
|
||||
|
||||
let inputBytes = (try? fm.attributesOfItem(atPath: options.inputBin.path)[.size] as? NSNumber)?.intValue ?? 0
|
||||
fputs("Preparing custom language model (\(inputBytes) byte asset)…\n", stderr)
|
||||
fputs("This may take several minutes for large lexicons.\n", stderr)
|
||||
|
||||
let started = Date()
|
||||
try await SFSpeechLanguageModel.prepareCustomLanguageModel(
|
||||
for: options.inputBin,
|
||||
clientIdentifier: options.clientIdentifier,
|
||||
configuration: configuration
|
||||
)
|
||||
let elapsed = Date().timeIntervalSince(started)
|
||||
|
||||
let lmBytes = fileSize(at: languageModelURL)
|
||||
let vocabBytes = fileSize(at: vocabularyURL)
|
||||
|
||||
fputs(
|
||||
"Done in \(String(format: "%.1f", elapsed))s — LM=\(lmBytes) bytes, Vocab=\(vocabBytes) bytes\n",
|
||||
stderr
|
||||
)
|
||||
|
||||
let manifestURL = options.outputDir.appendingPathComponent("prepared-manifest.json")
|
||||
let manifest: [String: Any] = [
|
||||
"generated_at": ISO8601DateFormatter().string(from: Date()),
|
||||
"client_identifier": options.clientIdentifier,
|
||||
"input_bin": options.inputBin.lastPathComponent,
|
||||
"input_bin_bytes": inputBytes,
|
||||
"language_model": languageModelURL.lastPathComponent,
|
||||
"language_model_bytes": lmBytes,
|
||||
"vocabulary": vocabularyURL.lastPathComponent,
|
||||
"vocabulary_bytes": vocabBytes,
|
||||
"prepare_seconds": elapsed,
|
||||
"configuration": [
|
||||
"language_model": languageModelURL.path,
|
||||
"vocabulary": vocabularyURL.path,
|
||||
"weight": options.weight as Any,
|
||||
],
|
||||
]
|
||||
let manifestData = try JSONSerialization.data(
|
||||
withJSONObject: manifest,
|
||||
options: [.prettyPrinted, .sortedKeys]
|
||||
)
|
||||
try manifestData.write(to: manifestURL)
|
||||
fputs("Wrote \(manifestURL.path)\n", stderr)
|
||||
}
|
||||
|
||||
private static func fileSize(at url: URL) -> Int {
|
||||
let fm = FileManager.default
|
||||
guard fm.fileExists(atPath: url.path) else { return 0 }
|
||||
return (try? fm.attributesOfItem(atPath: url.path)[.size] as? NSNumber)?.intValue ?? 0
|
||||
}
|
||||
}
|
||||
|
||||
enum PrepareError: LocalizedError {
|
||||
case missingInput(String)
|
||||
|
||||
var errorDescription: String? {
|
||||
switch self {
|
||||
case .missingInput(let path):
|
||||
return "Missing input .bin: \(path)"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Task {
|
||||
do {
|
||||
try await PrepareCLM.run()
|
||||
exit(0)
|
||||
} catch {
|
||||
fputs("prepare_clm failed: \(error)\n", stderr)
|
||||
exit(1)
|
||||
}
|
||||
}
|
||||
dispatchMain()
|
||||
@@ -0,0 +1,133 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Parse Sogou .scel cell dictionaries into (word, pinyin) entries.
|
||||
|
||||
Layout follows the classic SCEL format used by imewlconverter / SogouPopularDict.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import struct
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ScelInfo:
|
||||
word_count: int
|
||||
name: str
|
||||
type_name: str
|
||||
description: str
|
||||
|
||||
|
||||
def _read_uint16(handle) -> int:
|
||||
data = handle.read(2)
|
||||
if not data or len(data) < 2:
|
||||
return 0
|
||||
return struct.unpack("<H", data)[0]
|
||||
|
||||
|
||||
def _read_uint32(handle) -> int:
|
||||
data = handle.read(4)
|
||||
if not data or len(data) < 4:
|
||||
return 0
|
||||
return struct.unpack("<I", data)[0]
|
||||
|
||||
|
||||
def _read_utf16_str(handle, *, offset: int = -1, length: int = 0) -> str:
|
||||
if offset >= 0:
|
||||
handle.seek(offset)
|
||||
if length > 0:
|
||||
data = handle.read(length)
|
||||
end = 0
|
||||
for index in range(0, len(data), 2):
|
||||
if index + 1 < len(data) and data[index] == 0 and data[index + 1] == 0:
|
||||
end = index
|
||||
break
|
||||
if end > 0:
|
||||
data = data[:end]
|
||||
return data.decode("utf-16le", errors="ignore")
|
||||
|
||||
result = bytearray()
|
||||
while True:
|
||||
char = handle.read(2)
|
||||
if not char or len(char) < 2 or (char[0] == 0 and char[1] == 0):
|
||||
break
|
||||
result.extend(char)
|
||||
return result.decode("utf-16le", errors="ignore")
|
||||
|
||||
|
||||
def is_valid_word(word: str) -> bool:
|
||||
if not word or not (1 <= len(word) <= 10):
|
||||
return False
|
||||
allowed_punct = "，。：；？！（）【】《》""''、"
|
||||
return all("\u4e00" <= char <= "\u9fff" or char.isdigit() or char in allowed_punct for char in word)
|
||||
|
||||
|
||||
def get_scel_info(scel_path: Path) -> ScelInfo:
|
||||
with scel_path.open("rb") as handle:
|
||||
handle.seek(0x124)
|
||||
word_count = _read_uint32(handle)
|
||||
handle.seek(0x130)
|
||||
name = _read_utf16_str(handle, length=64)
|
||||
handle.seek(0x338)
|
||||
type_name = _read_utf16_str(handle, length=64)
|
||||
handle.seek(0x540)
|
||||
description = _read_utf16_str(handle, length=1024)
|
||||
return ScelInfo(word_count=word_count, name=name, type_name=type_name, description=description)
|
||||
|
||||
|
||||
def parse_scel_file(scel_path: Path) -> list[tuple[str, str]]:
|
||||
"""Return ordered (word, pinyin) pairs from a .scel file."""
|
||||
entries: list[tuple[str, str]] = []
|
||||
|
||||
with scel_path.open("rb") as handle:
|
||||
handle.seek(0x1540)
|
||||
pinyin_count = _read_uint32(handle)
|
||||
pinyin_dict: dict[int, str] = {}
|
||||
for _ in range(pinyin_count):
|
||||
pinyin_idx = _read_uint16(handle)
|
||||
pinyin_len = _read_uint16(handle)
|
||||
pinyin = handle.read(pinyin_len).decode("utf-16le", errors="ignore").strip().lower()
|
||||
pinyin_dict[pinyin_idx] = pinyin
|
||||
|
||||
try:
|
||||
while True:
|
||||
same_pinyin_count = _read_uint16(handle)
|
||||
pinyin_index_len = _read_uint16(handle)
|
||||
if pinyin_index_len <= 0 or same_pinyin_count <= 0:
|
||||
break
|
||||
|
||||
pinyin_parts: list[str] = []
|
||||
for _ in range(pinyin_index_len // 2):
|
||||
idx = _read_uint16(handle)
|
||||
part = pinyin_dict.get(idx, "")
|
||||
if part:
|
||||
pinyin_parts.append(part)
|
||||
joined_pinyin = " ".join(pinyin_parts).strip()
|
||||
|
||||
for _ in range(same_pinyin_count):
|
||||
word_len = _read_uint16(handle)
|
||||
word = handle.read(word_len).decode("utf-16le", errors="ignore")
|
||||
_ = _read_uint16(handle)
|
||||
_ = _read_uint32(handle)
|
||||
_ = handle.read(6)
|
||||
if is_valid_word(word):
|
||||
entries.append((word, joined_pinyin))
|
||||
except (struct.error, OSError):
|
||||
pass
|
||||
|
||||
return entries
|
||||
|
||||
|
||||
def load_pinyin_tsv(tsv_path: Path) -> list[tuple[str, str]]:
|
||||
entries: list[tuple[str, str]] = []
|
||||
with tsv_path.open(encoding="utf-8") as handle:
|
||||
for raw_line in handle:
|
||||
line = raw_line.strip()
|
||||
if not line:
|
||||
continue
|
||||
word, _, pinyin = line.partition("\t")
|
||||
if word and pinyin:
|
||||
entries.append((word, pinyin.strip()))
|
||||
return entries
|
||||
@@ -0,0 +1,381 @@
|
||||
# OSGKeyboard · AI / Tech / Brand seed lexicon (curated, permissive sources only)
|
||||
# Columns: word pinyin aliases category weight
|
||||
# aliases: pipe-separated alternate spellings / ASR confusions
|
||||
# License: curated by OSGKeyboard contributors (MIT). No third-party data bundled.
|
||||
#
|
||||
# --- AI brands & products ---
|
||||
DeepSeek deepseek|Deepseek|deep seek ai_brand 100
|
||||
深度求索 shen du qiu suo ai_brand 100
|
||||
OpenAI open ai|Open AI ai_brand 100
|
||||
ChatGPT chat gpt|Chat GPT|chatgpt ai_brand 100
|
||||
GPT gpt-4|GPT-4|GPT-4o|gpt4o ai_model 95
|
||||
Anthropic anthropic|Athropic|Anthropic ai_brand 100
|
||||
Claude claude|Claude Sonnet|Claude Opus ai_brand 100
|
||||
Google google|Alphabet tech_company 95
|
||||
Gemini gemini|Bard|Google Gemini ai_brand 95
|
||||
Meta meta|Facebook tech_company 95
|
||||
Llama llama|LLaMA|Llama 3|Llama 4 ai_model 90
|
||||
Microsoft microsoft|MSFT tech_company 95
|
||||
Copilot copilot|GitHub Copilot|Microsoft Copilot ai_brand 90
|
||||
GitHub Copilot github copilot ai_brand 88
|
||||
DeepMind deep mind|Google DeepMind ai_brand 88
|
||||
Mistral mistral ai|Mistral AI ai_brand 85
|
||||
Cohere cohere ai_brand 80
|
||||
Perplexity perplexity ai|Perplexity AI ai_brand 88
|
||||
Cursor cursor ai|Cursor AI dev_tool 90
|
||||
Kimi kimi|Kimi AI|Moonshot|Moonshot AI ai_brand 95
|
||||
月之暗面 yue zhi an mian ai_brand 90
|
||||
Qwen qwen|Qwen2|Qwen3|通义千问|千问 ai_brand 95
|
||||
通义千问 tong yi qian wen Qwen|qwen ai_brand 95
|
||||
GLM glm|GLM-4|GLM-5|智谱|Zhipu ai_brand 90
|
||||
智谱 zhi pu GLM|Zhipu AI ai_brand 88
|
||||
Zhipu AI zhipu|智谱 ai_brand 88
|
||||
文心一言 wen xin yi yan ERNIE|Ernie ai_brand 90
|
||||
ERNIE ernie|文心一言 ai_brand 88
|
||||
豆包 dou bao Doubao|doubao ai_brand 90
|
||||
Doubao dou bao|豆包 ai_brand 88
|
||||
混元 hun yuan Hunyuan|腾讯混元 ai_brand 85
|
||||
Hunyuan hun yuan|混元 ai_brand 85
|
||||
星火 xing huo Spark|讯飞星火 ai_brand 85
|
||||
讯飞 iFlytek|iflytek|科大讯飞 ai_brand 85
|
||||
iFlytek iflytek|讯飞 ai_brand 85
|
||||
Midjourney mid journey|Mid Journey ai_brand 88
|
||||
Stable Diffusion stable diffusion|SD ai_brand 85
|
||||
DALL-E dalle|DALL E|Dall-E ai_brand 85
|
||||
Sora sora ai|Sora AI ai_brand 88
|
||||
Hugging Face huggingface|HuggingFace|HF ai_platform 85
|
||||
Ollama ollama ai_platform 82
|
||||
LangChain lang chain|Langchain ai_platform 80
|
||||
vLLM vllm|VLLM ai_platform 80
|
||||
xAI x ai|Grok ai_brand 88
|
||||
Grok grok|xAI ai_brand 85
|
||||
Groq groq|GROQ ai_brand 82
|
||||
Replicate replicate ai_platform 78
|
||||
Runway runway ai|Runway ML ai_brand 80
|
||||
Pika pika ai|Pika Labs ai_brand 75
|
||||
ElevenLabs eleven labs|Eleven Labs ai_brand 80
|
||||
Character AI character.ai|Character.AI ai_brand 78
|
||||
Notion AI notion ai ai_brand 75
|
||||
Windsurf windsurf|Codeium Windsurf dev_tool 82
|
||||
Codeium codeium ai_brand 80
|
||||
Cline cline|Cline AI dev_tool 78
|
||||
Aider aider ai_brand 75
|
||||
Replit replit|Replit Agent dev_tool 78
|
||||
SiliconFlow silicon flow|Silicon Flow ai_platform 80
|
||||
OpenRouter open router|Open Router ai_platform 82
|
||||
DeepSeek-R1 deepseek r1|DeepSeek R1|R1 ai_model 92
|
||||
DeepSeek-V3 deepseek v3|DeepSeek V3 ai_model 90
|
||||
o1 openai o1|O1 ai_model 88
|
||||
o3 openai o3|O3 ai_model 88
|
||||
Sonnet claude sonnet|Sonnet 4 ai_model 85
|
||||
Opus claude opus|Opus 4 ai_model 85
|
||||
MiniMax minimax|MiniMax AI ai_brand 85
|
||||
StepFun step fun|阶跃星辰 ai_brand 82
|
||||
阶跃星辰 jie yue xing chen StepFun ai_brand 80
|
||||
百川 bai chuan|Baichuan ai_brand 82
|
||||
Baichuan baichuan|百川 ai_brand 82
|
||||
零一万物 ling yi wan wu|01.AI|Yi ai_brand 82
|
||||
01.AI 01 ai|零一万物 ai_brand 80
|
||||
面壁智能 mian bi zhi neng MiniCPM ai_brand 78
|
||||
MiniCPM minicpm|面壁 ai_model 75
|
||||
#
|
||||
# --- AI / dev terminology ---
|
||||
Transformer transformer|变换器 ai_term 95
|
||||
RAG rag|检索增强生成|retrieval augmented generation ai_term 95
|
||||
检索增强生成 jian suo zeng qiang sheng cheng RAG ai_term 92
|
||||
LoRA lora|低秩适配 ai_term 90
|
||||
微调 wei tiao fine-tuning|fine tuning ai_term 90
|
||||
fine-tuning fine tuning|微调 ai_term 88
|
||||
MoE moe|mixture of experts|混合专家 ai_term 88
|
||||
混合专家 hun he zhuan jia MoE ai_term 85
|
||||
embedding 嵌入|embeddings ai_term 90
|
||||
嵌入 qian ru embedding ai_term 88
|
||||
tokenizer tokenizer|分词器 ai_term 85
|
||||
幻觉 huan jue hallucination ai_term 88
|
||||
hallucination 幻觉 ai_term 85
|
||||
提示词 ti shi ci prompt|Prompt ai_term 92
|
||||
prompt prompt engineering|提示词 ai_term 90
|
||||
智能体 zhi neng ti agent|Agent|AI agent ai_term 92
|
||||
agent agentic|智能体|AI agent ai_term 90
|
||||
agentic agentic AI|智能体 ai_term 88
|
||||
vibe coding vibe code|Vibe Coding|氛围编程 ai_term 95
|
||||
氛围编程 fen wei bian cheng vibe coding ai_term 90
|
||||
MCP model context protocol|MCP server ai_term 92
|
||||
上下文窗口 shang xia wen chuang kou context window ai_term 88
|
||||
context window 上下文窗口 ai_term 85
|
||||
diffusion 扩散模型 ai_term 85
|
||||
扩散模型 kuo san mo xing diffusion ai_term 82
|
||||
推理 tui li inference|reasoning ai_term 88
|
||||
inference 推理 ai_term 85
|
||||
RLHF rlhf|人类反馈强化学习 ai_term 85
|
||||
chain of thought chain-of-thought|思维链 ai_term 88
|
||||
思维链 si wei lian chain of thought ai_term 85
|
||||
多模态 duo mo tai multimodal ai_term 88
|
||||
multimodal 多模态 ai_term 85
|
||||
AGI agi|通用人工智能 ai_term 90
|
||||
通用人工智能 tong yong ren gong zhi neng AGI ai_term 88
|
||||
LLM llm|大语言模型|large language model ai_term 92
|
||||
大语言模型 da yu yan mo xing LLM ai_term 90
|
||||
大模型 da mo xing LLM|large model ai_term 92
|
||||
SLM slm|小语言模型 ai_term 80
|
||||
量化 liang hua quantization ai_term 85
|
||||
quantization 量化 ai_term 82
|
||||
function calling tool calling|工具调用 ai_term 88
|
||||
工具调用 gong ju diao yong function calling ai_term 85
|
||||
流式 liu shi streaming ai_term 82
|
||||
streaming 流式 ai_term 80
|
||||
AIGC aigc|生成式人工智能 ai_term 88
|
||||
生成式人工智能 sheng cheng shi ren gong zhi neng GenAI|AIGC ai_term 85
|
||||
GenAI gen ai|生成式AI ai_term 88
|
||||
prompt engineering 提示工程 ai_term 85
|
||||
提示工程 ti shi gong cheng prompt engineering ai_term 82
|
||||
reasoning model 推理模型|thinking model ai_term 88
|
||||
推理模型 tui li mo xing reasoning model ai_term 85
|
||||
jailbreak 越狱 ai_term 75
|
||||
越狱 yue yu jailbreak ai_term 72
|
||||
SWE-bench swe bench|SWE bench ai_term 80
|
||||
benchmark 基准测试 ai_term 78
|
||||
open weight open weights|开放权重 ai_term 82
|
||||
开放权重 kai fang quan zhong open weight ai_term 80
|
||||
distillation 蒸馏|知识蒸馏 ai_term 82
|
||||
知识蒸馏 zhi shi zheng liu distillation ai_term 80
|
||||
pretraining pre-training|预训练 ai_term 82
|
||||
预训练 yu xun lian pretraining ai_term 80
|
||||
vector database 向量数据库 ai_term 82
|
||||
向量数据库 xiang liang shu ju ku vector database ai_term 80
|
||||
RAG pipeline rag pipeline ai_term 78
|
||||
AI native AI-native|AI原生 ai_term 82
|
||||
AI原生 AI yuan sheng AI native ai_term 80
|
||||
computer use computer-use|电脑使用 ai_term 80
|
||||
deep research deep research|深度研究 ai_term 82
|
||||
on-device AI on device ai|端侧AI ai_term 80
|
||||
端侧AI duan ce AI on-device AI ai_term 78
|
||||
#
|
||||
# --- US / global tech companies ---
|
||||
SpaceX space x|Space X tech_company 95
|
||||
Tesla tesla|Tesla Motors tech_company 95
|
||||
Apple apple|苹果公司 tech_company 95
|
||||
Microsoft microsoft|微软 tech_company 95
|
||||
Google google|谷歌 tech_company 95
|
||||
Alphabet alphabet|Google tech_company 90
|
||||
Meta meta|Facebook|脸书 tech_company 95
|
||||
Amazon amazon|AWS|亚马逊 tech_company 95
|
||||
AWS aws|Amazon Web Services tech_company 92
|
||||
Nvidia nvidia|NVDA|英伟达 tech_company 98
|
||||
英伟达 ying wei da Nvidia|NVDA tech_company 95
|
||||
AMD amd tech_company 88
|
||||
Intel intel|英特尔 tech_company 88
|
||||
Netflix netflix tech_company 85
|
||||
Uber uber tech_company 85
|
||||
Airbnb airbnb tech_company 82
|
||||
Stripe stripe tech_company 88
|
||||
Shopify shopify tech_company 82
|
||||
Salesforce salesforce tech_company 85
|
||||
Oracle oracle tech_company 82
|
||||
IBM ibm tech_company 82
|
||||
Adobe adobe tech_company 85
|
||||
Spotify spotify tech_company 82
|
||||
Reddit reddit tech_company 80
|
||||
Discord discord tech_company 80
|
||||
Slack slack tech_company 80
|
||||
Zoom zoom tech_company 80
|
||||
Palantir palantir tech_company 82
|
||||
Neuralink neural link|Neural Link tech_company 85
|
||||
Waymo waymo tech_company 85
|
||||
Rivian rivian tech_company 80
|
||||
Lucid lucid motors|Lucid Motors tech_company 78
|
||||
Coinbase coinbase tech_company 82
|
||||
Robinhood robin hood|Robin Hood fintech 78
|
||||
PayPal paypal tech_company 82
|
||||
Block block|Square|square fintech 78
|
||||
Visa visa tech_company 78
|
||||
Samsung samsung|三星 tech_company 88
|
||||
Sony sony|索尼 tech_company 85
|
||||
Nintendo nintendo|任天堂 tech_company 82
|
||||
TSMC tsmc|台积电 tech_company 90
|
||||
台积电 tai ji dian TSMC tech_company 88
|
||||
ASML asml tech_company 85
|
||||
Broadcom broadcom tech_company 80
|
||||
Qualcomm qualcomm|高通 tech_company 85
|
||||
高通 gao tong Qualcomm tech_company 82
|
||||
Arm arm|ARM Holdings tech_company 85
|
||||
Snowflake snowflake tech_company 80
|
||||
Databricks databricks tech_company 85
|
||||
Cloudflare cloudflare tech_company 82
|
||||
Twilio twilio tech_company 78
|
||||
Datadog datadog tech_company 78
|
||||
ServiceNow service now|ServiceNow tech_company 78
|
||||
Atlassian atlassian|Jira|Confluence tech_company 80
|
||||
Canva canva tech_company 80
|
||||
Figma figma tech_company 85
|
||||
Notion notion tech_company 82
|
||||
Linear linear app|Linear tech_company 78
|
||||
Vercel vercel|Next.js tech_company 82
|
||||
Next.js nextjs|NextJS|next js dev_tool 85
|
||||
Vercel vercel tech_company 80
|
||||
Supabase supabase tech_company 80
|
||||
Firebase firebase tech_company 80
|
||||
MongoDB mongodb|Mongo DB tech_company 82
|
||||
Redis redis tech_company 82
|
||||
Elastic elastic|Elasticsearch tech_company 78
|
||||
Docker docker tech_company 88
|
||||
Kubernetes kubernetes|k8s|K8s dev_tool 90
|
||||
k8s kubernetes|Kubernetes dev_tool 88
|
||||
Terraform terraform tech_company 80
|
||||
GitLab gitlab tech_company 80
|
||||
Bitbucket bitbucket tech_company 75
|
||||
Jenkins jenkins dev_tool 75
|
||||
CircleCI circle ci|Circle CI dev_tool 75
|
||||
#
|
||||
# --- Chinese tech companies (English names) ---
|
||||
Huawei huawei|华为|HW tech_company 95
|
||||
华为 hua wei Huawei tech_company 95
|
||||
Xiaomi xiaomi|小米|MI tech_company 95
|
||||
小米 xiao mi Xiaomi tech_company 95
|
||||
ByteDance byte dance|字节跳动|Bytedance tech_company 95
|
||||
字节跳动 zi jie tiao dong ByteDance tech_company 95
|
||||
TikTok tik tok|Tik Tok|抖音海外 tech_company 92
|
||||
Douyin dou yin|抖音 tech_company 90
|
||||
抖音 dou yin Douyin|TikTok tech_company 90
|
||||
Alibaba alibaba|阿里巴巴|阿里 tech_company 95
|
||||
阿里巴巴 a li ba ba Alibaba tech_company 95
|
||||
Taobao taobao|淘宝 tech_company 88
|
||||
淘宝 tao bao Taobao tech_company 88
|
||||
Tmall tmall|天猫 tech_company 85
|
||||
天猫 tian mao Tmall tech_company 85
|
||||
Tencent tencent|腾讯 tech_company 95
|
||||
腾讯 teng xun Tencent tech_company 95
|
||||
WeChat we chat|微信 tech_company 92
|
||||
微信 wei xin WeChat tech_company 92
|
||||
Baidu baidu|百度 tech_company 92
|
||||
百度 bai du Baidu tech_company 92
|
||||
JD.com jd.com|京东|JD tech_company 88
|
||||
京东 jing dong JD.com tech_company 88
|
||||
Meituan meituan|美团 tech_company 88
|
||||
美团 mei tuan Meituan tech_company 88
|
||||
Pinduoduo pinduoduo|拼多多|PDD tech_company 88
|
||||
拼多多 pin duo duo Pinduoduo tech_company 88
|
||||
BYD byd|比亚迪 tech_company 95
|
||||
比亚迪 bi ya di BYD tech_company 95
|
||||
NIO nio|蔚来 tech_company 90
|
||||
蔚来 wei lai NIO tech_company 90
|
||||
XPeng xpeng|小鹏汽车|X Peng tech_company 90
|
||||
小鹏汽车 xiao peng qi che XPeng tech_company 90
|
||||
Li Auto li auto|理想汽车|理想 tech_company 90
|
||||
理想汽车 li xiang qi che Li Auto tech_company 90
|
||||
Zeekr zeekr|极氪 tech_company 85
|
||||
极氪 ji ke Zeekr tech_company 85
|
||||
CATL catl|宁德时代 tech_company 92
|
||||
宁德时代 ning de shi dai CATL tech_company 92
|
||||
DJI dji|大疆 tech_company 90
|
||||
大疆 da jiang DJI tech_company 90
|
||||
SMIC smic|中芯国际 tech_company 88
|
||||
中芯国际 zhong xin guo ji SMIC tech_company 88
|
||||
Lenovo lenovo|联想 tech_company 88
|
||||
联想 lian xiang Lenovo tech_company 88
|
||||
Oppo oppo|OPPO tech_company 85
|
||||
Vivo vivo|VIVO tech_company 85
|
||||
Honor honor|荣耀 tech_company 85
|
||||
荣耀 rong yao Honor tech_company 85
|
||||
Shein shein|SHEIN tech_company 85
|
||||
Temu temu tech_company 85
|
||||
Ant Group ant group|蚂蚁集团|Alipay tech_company 88
|
||||
蚂蚁集团 ma yi ji tuan Ant Group tech_company 88
|
||||
Alipay alipay|支付宝 tech_company 88
|
||||
支付宝 zhi fu bao Alipay tech_company 88
|
||||
Weibo weibo|微博 tech_company 82
|
||||
微博 wei bo Weibo tech_company 82
|
||||
Bilibili bilibili|B站|哔哩哔哩 tech_company 88
|
||||
哔哩哔哩 bi li bi li Bilibili|B站 tech_company 88
|
||||
B站 B zhan Bilibili|哔哩哔哩 tech_company 85
|
||||
NetEase netease|网易 tech_company 85
|
||||
网易 wang yi NetEase tech_company 85
|
||||
Kuaishou kuaishou|快手 tech_company 85
|
||||
快手 kuai shou Kuaishou tech_company 85
|
||||
SenseTime sensetime|商汤 tech_company 82
|
||||
商汤 shang tang SenseTime tech_company 82
|
||||
Megvii megvii|旷视 tech_company 80
|
||||
旷视 kuang shi Megvii tech_company 80
|
||||
Horizon Robotics horizon|地平线 tech_company 82
|
||||
地平线 di ping xian Horizon tech_company 80
|
||||
Geely geely|吉利 tech_company 82
|
||||
吉利 ji li Geely tech_company 80
|
||||
Great Wall great wall|长城汽车 tech_company 78
|
||||
长城汽车 chang cheng qi che Great Wall tech_company 78
|
||||
#
|
||||
# --- Tech leaders (bilingual where useful) ---
|
||||
Elon Musk 马斯克|elon musk tech_leader 92
|
||||
马斯克 ma si ke Elon Musk tech_leader 92
|
||||
Sam Altman sam altman|山姆奥特曼 tech_leader 88
|
||||
Jensen Huang 黄仁勋|jensen huang tech_leader 90
|
||||
黄仁勋 huang ren xun Jensen Huang tech_leader 90
|
||||
Tim Cook 库克|tim cook tech_leader 85
|
||||
库克 ku ke Tim Cook tech_leader 85
|
||||
Satya Nadella satya nadella|纳德拉 tech_leader 82
|
||||
Sundar Pichai sundar pichai|皮查伊 tech_leader 82
|
||||
Mark Zuckerberg mark zuckerberg|扎克伯格 tech_leader 85
|
||||
扎克伯格 zhai ke bo ge Mark Zuckerberg tech_leader 85
|
||||
Jeff Bezos jeff bezos|贝索斯 tech_leader 82
|
||||
雷军 lei jun Lei Jun tech_leader 90
|
||||
Lei Jun lei jun|雷军 tech_leader 90
|
||||
何小鹏 he xiao peng He Xiaopeng|XPeng tech_leader 88
|
||||
He Xiaopeng he xiao peng|何小鹏 tech_leader 88
|
||||
李斌 li bin William Li|NIO tech_leader 85
|
||||
余承东 yu cheng dong tech_leader 82
|
||||
梁文锋 liang wen feng DeepSeek tech_leader 85
|
||||
乔布斯 qiao bu si Steve Jobs tech_leader 90
|
||||
Steve Jobs steve jobs|乔布斯 tech_leader 90
|
||||
#
|
||||
# --- Dev tools & platforms ---
|
||||
GitHub github|GitHub tech_company 92
|
||||
GitLab gitlab tech_company 80
|
||||
VS Code vs code|VSCode|Visual Studio Code dev_tool 88
|
||||
Visual Studio Code vscode|VS Code dev_tool 85
|
||||
Xcode xcode dev_tool 85
|
||||
SwiftUI swift ui|Swift UI dev_tool 82
|
||||
React react|ReactJS dev_tool 85
|
||||
Vue vue|Vue.js|Vue3 dev_tool 82
|
||||
TypeScript typescript|TS dev_tool 85
|
||||
Python python dev_tool 88
|
||||
Rust rust dev_tool 82
|
||||
Go golang|Golang dev_tool 82
|
||||
Node.js nodejs|NodeJS|node js dev_tool 82
|
||||
PyTorch pytorch|Py Torch dev_tool 88
|
||||
TensorFlow tensorflow|Tensor Flow dev_tool 85
|
||||
Jupyter jupyter|Jupyter Notebook dev_tool 78
|
||||
Postman postman dev_tool 78
|
||||
Figma figma dev_tool 85
|
||||
Notion notion dev_tool 82
|
||||
Linear linear dev_tool 78
|
||||
Obsidian obsidian dev_tool 75
|
||||
Raycast raycast dev_tool 75
|
||||
Warp warp terminal|Warp dev_tool 75
|
||||
#
|
||||
# --- Hot internet / product terms ---
|
||||
SaaS saas|SaaS tech_term 82
|
||||
API api|API tech_term 85
|
||||
SDK sdk|SDK tech_term 82
|
||||
GPU gpu|GPU tech_term 88
|
||||
CUDA cuda|CUDA tech_term 85
|
||||
NPU npu|NPU tech_term 82
|
||||
TPU tpu|TPU tech_term 80
|
||||
Web3 web3|Web 3 tech_term 78
|
||||
区块链 qu kuai lian blockchain tech_term 82
|
||||
blockchain 区块链 tech_term 80
|
||||
元宇宙 yuan yu zhou metaverse tech_term 78
|
||||
metaverse 元宇宙 tech_term 75
|
||||
自动驾驶 zi dong jia shi autonomous driving|FSD tech_term 85
|
||||
FSD full self driving|全自动驾驶 tech_term 82
|
||||
人形机器人 ren xing ji qi ren humanoid robot|Optimus tech_term 82
|
||||
Optimus optimus|擎天柱 tech_term 80
|
||||
星链 xing lian Starlink tech_term 85
|
||||
Starlink star link|星链 tech_term 85
|
||||
低空经济 di kong jing ji low altitude economy tech_term 80
|
||||
具身智能 ju shen zhi neng embodied AI tech_term 82
|
||||
embodied AI 具身智能 tech_term 80
|
||||
出海 chu hai go global|全球化 tech_term 78
|
||||
内卷 nei juan involution tech_term 75
|
||||
躺平 tang ping lying flat tech_term 72
|
||||
数字游民 shu zi you min digital nomad tech_term 75
|
||||
远程办公 yuan cheng ban gong remote work tech_term 78
|
||||
副业 fu ye side hustle tech_term 72
|
||||
|
Reference in New Issue
Block a user