feat(app): add crash visibility, memory relief, and clipboard reply work
Squashes the working tree that had accumulated on cloud-agent. Two strands of work, kept in one commit at the maintainer's request. Crash and stability (this session): - Subscribe to MetricKit so crash, hang, CPU, and disk-write diagnostics are recorded on device. Keyboard-extension memory kills are the most likely way a user loses the keyboard mid-sentence and iOS does not classify them as crashes, so they never reach App Store Connect. - Upload those reports to POST /v1/analytics/diagnostics from internal builds only (local Debug and TestFlight), reusing the analytics installation identifier and skipping entirely when analytics are off. App Store builds keep everything on device. - Shed the Chinese engine, English lexicon, and in-flight pipeline work at the 40 MiB and 48 MiB budget thresholds instead of waiting for a system memory warning that jetsam often preempts. - Install audio taps with the input node's live format through one shared adaptive resampler, so an audio-route change mid-recording no longer raises an uncatchable format-mismatch exception. Covers the dictation preview and the Mac recorder, which still used the pattern the Flow capture path had already moved away from. - Harden the librime bridge: never pass a nil string across a nonnull boundary when a user dictionary holds malformed UTF-8, null-check the engine API vector, and distrust a candidate count without its array. - Guard the expanded candidate grid against a list that shrinks while the collection view is still laying out against the previous count. - Resolve localized text without the App Group so the screen shown when that container is missing renders its message instead of crashing on launch, which is the one situation it exists to handle. - Gate CI on the App Group and Keychain wiring that the keyboard extension hard-depends on, with a self-test proving the check fails. Clipboard, reply, and semantics (pre-existing working tree): - Unify the reply center, refresh keyboard AI features, and add native event and reminder exporters. - Expand clipboard semantic routing with authored corpora and a product-blind holdout, plus the scripts that build and validate them. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
@@ -0,0 +1,219 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Split authored batches into train and evaluation corpora by scenario family.
|
||||
|
||||
Whole families move together. A family is one scenario (for example
|
||||
"invitation to a meal" or "received thanks for a blessing"), so a family-level
|
||||
split forces the evaluation set to test scenarios the model never trained on
|
||||
instead of paraphrases of them. Text overlap between the two sides is therefore
|
||||
impossible by construction, and the script verifies it anyway.
|
||||
|
||||
Positive and negative families are assigned separately so both sides keep a
|
||||
usable label balance.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import unicodedata
|
||||
from collections import Counter, defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
SEED = 20260903
|
||||
AUTHORED_DIRECTORY = Path("ModelTraining/ClipboardSemantics/Authored")
|
||||
INTENT_FIELDS = (
|
||||
"task",
|
||||
"question",
|
||||
"invitation",
|
||||
"complaint",
|
||||
"scheduleNegotiation",
|
||||
"confirmationDecision",
|
||||
"followUpReminder",
|
||||
"blessing",
|
||||
"replyable",
|
||||
)
|
||||
SOURCE_DATASET = "authored-v1"
|
||||
SOURCE_LICENSE = "OSGKeyboard project license"
|
||||
|
||||
|
||||
def parse_arguments() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("batches", nargs="*", type=Path)
|
||||
parser.add_argument("--seed", type=int, default=SEED)
|
||||
parser.add_argument(
|
||||
"--evaluation-family-fraction",
|
||||
type=float,
|
||||
default=0.30,
|
||||
help="Share of families in each polarity group reserved for evaluation.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--sample-weight",
|
||||
type=float,
|
||||
default=1.0,
|
||||
help="Training sample weight recorded on every authored train record.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--train-output",
|
||||
type=Path,
|
||||
default=AUTHORED_DIRECTORY / "authored-train-corpus.jsonl",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--evaluation-output",
|
||||
type=Path,
|
||||
default=AUTHORED_DIRECTORY / "authored-eval-corpus.jsonl",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--report",
|
||||
type=Path,
|
||||
default=AUTHORED_DIRECTORY / "authored-split-report.json",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def normalized_text(value: str) -> str:
|
||||
return " ".join(unicodedata.normalize("NFKC", value).split()).strip()
|
||||
|
||||
|
||||
def fingerprint(value: str) -> str:
|
||||
return normalized_text(value).casefold()
|
||||
|
||||
|
||||
def load_records(path: Path) -> list[dict]:
|
||||
return [
|
||||
json.loads(line)
|
||||
for line in path.read_text(encoding="utf-8").splitlines()
|
||||
if line.strip()
|
||||
]
|
||||
|
||||
|
||||
def polarity_group(family: str) -> str:
|
||||
"""Negative families carry a `neg` marker in their leading token."""
|
||||
|
||||
return "negative" if "neg_" in family else "positive"
|
||||
|
||||
|
||||
def intent_group(family: str) -> str:
|
||||
return family.split("_", 1)[0].removesuffix("neg")
|
||||
|
||||
|
||||
def family_priority(family: str, seed: int) -> str:
|
||||
return hashlib.sha256(f"{seed}|{family}".encode()).hexdigest()
|
||||
|
||||
|
||||
def assign_families(families: set[str], seed: int, fraction: float) -> dict[str, str]:
|
||||
"""Reserve a deterministic share of each intent/polarity cell for evaluation."""
|
||||
|
||||
cells: dict[tuple[str, str], list[str]] = defaultdict(list)
|
||||
for family in families:
|
||||
cells[(intent_group(family), polarity_group(family))].append(family)
|
||||
|
||||
assignment: dict[str, str] = {}
|
||||
for cell, values in sorted(cells.items()):
|
||||
ordered = sorted(values, key=lambda value: family_priority(value, seed))
|
||||
reserved = max(1, round(len(ordered) * fraction))
|
||||
for index, family in enumerate(ordered):
|
||||
assignment[family] = "evaluation" if index < reserved else "train"
|
||||
return assignment
|
||||
|
||||
|
||||
def main() -> None:
|
||||
arguments = parse_arguments()
|
||||
batches = arguments.batches or sorted(
|
||||
path
|
||||
for path in AUTHORED_DIRECTORY.glob("*.jsonl")
|
||||
if not path.name.startswith("authored-")
|
||||
)
|
||||
records: list[dict] = []
|
||||
for path in batches:
|
||||
records.extend(load_records(path))
|
||||
|
||||
assignment = assign_families(
|
||||
{record["family"] for record in records},
|
||||
arguments.seed,
|
||||
arguments.evaluation_family_fraction,
|
||||
)
|
||||
|
||||
train: list[dict] = []
|
||||
evaluation: list[dict] = []
|
||||
for record in records:
|
||||
target = assignment[record["family"]]
|
||||
emitted = {
|
||||
"id": record["id"],
|
||||
"text": normalized_text(record["text"]),
|
||||
"language": record["language"],
|
||||
"family": record["family"],
|
||||
"sentiment": record["sentiment"],
|
||||
**{field: record[field] for field in INTENT_FIELDS},
|
||||
"sourceDataset": SOURCE_DATASET,
|
||||
"sourceLicense": SOURCE_LICENSE,
|
||||
"provenance": "llm-authored",
|
||||
"knownLabels": list(INTENT_FIELDS),
|
||||
}
|
||||
if target == "train":
|
||||
emitted["split"] = "train"
|
||||
emitted["sampleWeight"] = arguments.sample_weight
|
||||
train.append(emitted)
|
||||
else:
|
||||
emitted["split"] = "authoredEvaluation"
|
||||
evaluation.append(emitted)
|
||||
|
||||
train_texts = {fingerprint(record["text"]) for record in train}
|
||||
evaluation_texts = {fingerprint(record["text"]) for record in evaluation}
|
||||
leaked = sorted(train_texts & evaluation_texts)
|
||||
|
||||
for path, values in (
|
||||
(arguments.train_output, train),
|
||||
(arguments.evaluation_output, evaluation),
|
||||
):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(
|
||||
"\n".join(
|
||||
json.dumps(value, ensure_ascii=False, sort_keys=True)
|
||||
for value in values
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
def positives(values: list[dict]) -> dict:
|
||||
return {
|
||||
field: sum(1 for value in values if value[field] is True)
|
||||
for field in INTENT_FIELDS
|
||||
}
|
||||
|
||||
report = {
|
||||
"schemaVersion": 1,
|
||||
"seed": arguments.seed,
|
||||
"batches": [str(path) for path in batches],
|
||||
"evaluationFamilyFraction": arguments.evaluation_family_fraction,
|
||||
"sampleWeight": arguments.sample_weight,
|
||||
"families": len(assignment),
|
||||
"familyAssignment": dict(sorted(assignment.items())),
|
||||
"train": {
|
||||
"records": len(train),
|
||||
"families": len({value["family"] for value in train}),
|
||||
"languages": dict(Counter(value["language"] for value in train)),
|
||||
"positives": positives(train),
|
||||
},
|
||||
"evaluation": {
|
||||
"records": len(evaluation),
|
||||
"families": len({value["family"] for value in evaluation}),
|
||||
"languages": dict(Counter(value["language"] for value in evaluation)),
|
||||
"positives": positives(evaluation),
|
||||
},
|
||||
"validation": {
|
||||
"trainEvaluationTextOverlap": len(leaked),
|
||||
"leakedExamples": leaked[:5],
|
||||
},
|
||||
}
|
||||
arguments.report.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,387 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Prepare the v2 product blind holdout with stratified, IPW-correctable sampling.
|
||||
|
||||
The v1 blind holdout has 120 records and as few as two positives per intent, so
|
||||
it cannot separate a real quality regression from sampling noise. This script
|
||||
builds a larger queue from real, clipboard-shaped licensed text.
|
||||
|
||||
Every eligible pool record is assigned to exactly one stratum by a rarest-first
|
||||
intent priority, so the design is a clean stratified sample: each emitted record
|
||||
carries `inclusionProbability` = selected/available for its stratum, and
|
||||
prevalence-corrected metrics follow from Horvitz-Thompson weighting. Rare
|
||||
intents are oversampled for statistical power without biasing reported rates.
|
||||
|
||||
Weak source labels drive stratification only. They are written to the sealed
|
||||
provenance file and never to the blind queue.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import unicodedata
|
||||
from collections import Counter, defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
SEED = 20260902
|
||||
MODEL_TRAINING = Path("ModelTraining/ClipboardSemantics")
|
||||
OUTPUT_DIRECTORY = MODEL_TRAINING / "ProductBlindHoldoutV2"
|
||||
DEFAULT_SOURCES = (
|
||||
MODEL_TRAINING / "comprehensive-online-holdout-corpus.jsonl",
|
||||
MODEL_TRAINING / "online-real-holdout-corpus.jsonl",
|
||||
)
|
||||
|
||||
# Intents in rarest-first order. A record joins the first stratum it matches,
|
||||
# which is what gives the scarce intents their own guaranteed sampling cells.
|
||||
INTENT_PRIORITY = (
|
||||
"blessing",
|
||||
"confirmationDecision",
|
||||
"scheduleNegotiation",
|
||||
"invitation",
|
||||
"followUpReminder",
|
||||
"complaint",
|
||||
"task",
|
||||
"question",
|
||||
)
|
||||
REPLYABLE_STRATUM = "replyableOnly"
|
||||
NEGATIVE_STRATUM = "noWeakIntent"
|
||||
LANGUAGES = ("en", "zh-Hans")
|
||||
|
||||
# Families that are real text but not clipboard-shaped. CPED is television
|
||||
# dialogue and GoEmotions is short Reddit reaction text; both are dominated by
|
||||
# sub-15-character conversational turns that no user copies to a clipboard.
|
||||
EXCLUDED_FAMILY_PREFIXES = ("online_cped_", "online_goemotions_")
|
||||
|
||||
PII_PATTERN = re.compile(
|
||||
r"(?:[\w.+-]+@[\w.-]+\.\w+)|(?:\+?\d[\d ()-]{8,}\d)|"
|
||||
r"(?:\b\d{3}-\d{2}-\d{4}\b)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
INTENT_FIELDS = (*INTENT_PRIORITY, "replyable")
|
||||
|
||||
|
||||
def parse_arguments() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--source", action="append", type=Path, default=[])
|
||||
parser.add_argument("--output-directory", type=Path, default=OUTPUT_DIRECTORY)
|
||||
parser.add_argument("--seed", type=int, default=SEED)
|
||||
parser.add_argument(
|
||||
"--minimum-characters",
|
||||
type=int,
|
||||
default=25,
|
||||
help="Clipboard-shape floor. Shorter real text is conversational turns.",
|
||||
)
|
||||
parser.add_argument("--maximum-characters", type=int, default=1_500)
|
||||
parser.add_argument(
|
||||
"--rare-stratum-target",
|
||||
type=int,
|
||||
default=50,
|
||||
help="Reviewed records to draw from each scarce intent stratum.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--common-stratum-target",
|
||||
type=int,
|
||||
default=120,
|
||||
help="Reviewed records to draw from each abundant stratum.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--common-strata",
|
||||
default="question,replyableOnly,noWeakIntent",
|
||||
help="Comma-separated strata that use --common-stratum-target.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--training-corpus",
|
||||
action="append",
|
||||
default=[],
|
||||
type=Path,
|
||||
help="Additional JSONL whose text must not appear in the queue.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def normalized_text(value: str) -> str:
|
||||
value = unicodedata.normalize("NFKC", value.replace("\u0000", " "))
|
||||
return " ".join(value.split()).strip()
|
||||
|
||||
|
||||
def fingerprint(value: str) -> str:
|
||||
return normalized_text(value).casefold()
|
||||
|
||||
|
||||
def file_sha256(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def load_records(path: Path) -> list[dict]:
|
||||
return [
|
||||
json.loads(line)
|
||||
for line in path.read_text(encoding="utf-8").splitlines()
|
||||
if line.strip()
|
||||
]
|
||||
|
||||
|
||||
def protected_fingerprints(paths: list[Path]) -> set[str]:
|
||||
values: set[str] = set()
|
||||
for path in paths:
|
||||
for record in load_records(path):
|
||||
values.add(fingerprint(record["text"]))
|
||||
return values
|
||||
|
||||
|
||||
def is_real_source(record: dict) -> bool:
|
||||
"""Reject template-generated families, which are not real user text."""
|
||||
|
||||
family = record.get("family") or ""
|
||||
return family.startswith("online_") and not family.startswith(
|
||||
EXCLUDED_FAMILY_PREFIXES
|
||||
)
|
||||
|
||||
|
||||
def stratum_of(record: dict) -> str:
|
||||
for intent in INTENT_PRIORITY:
|
||||
if record.get(intent) is True:
|
||||
return intent
|
||||
if record.get("replyable") is True:
|
||||
return REPLYABLE_STRATUM
|
||||
return NEGATIVE_STRATUM
|
||||
|
||||
|
||||
def stable_priority(record_id: str, seed: int, salt: str) -> bytes:
|
||||
return hashlib.sha256(f"{seed}|{salt}|{record_id}".encode()).digest()
|
||||
|
||||
|
||||
def eligible_pool(
|
||||
sources: list[Path],
|
||||
*,
|
||||
minimum_characters: int,
|
||||
maximum_characters: int,
|
||||
protected: set[str],
|
||||
) -> tuple[list[dict], Counter]:
|
||||
rejected: Counter = Counter()
|
||||
seen: set[str] = set()
|
||||
pool: list[dict] = []
|
||||
for path in sources:
|
||||
for record in load_records(path):
|
||||
if not is_real_source(record):
|
||||
rejected["notRealClipboardShapedSource"] += 1
|
||||
continue
|
||||
if record.get("language") not in LANGUAGES:
|
||||
rejected["unsupportedLanguage"] += 1
|
||||
continue
|
||||
text = normalized_text(record.get("text") or "")
|
||||
if not minimum_characters <= len(text) <= maximum_characters:
|
||||
rejected["lengthOutsideClipboardShape"] += 1
|
||||
continue
|
||||
key = fingerprint(text)
|
||||
if key in protected:
|
||||
rejected["overlapsProtectedCorpus"] += 1
|
||||
continue
|
||||
if key in seen:
|
||||
rejected["duplicateText"] += 1
|
||||
continue
|
||||
if PII_PATTERN.search(text):
|
||||
rejected["detectedPII"] += 1
|
||||
continue
|
||||
seen.add(key)
|
||||
candidate = dict(record)
|
||||
candidate["_text"] = text
|
||||
candidate["_stratum"] = stratum_of(record)
|
||||
pool.append(candidate)
|
||||
return pool, rejected
|
||||
|
||||
|
||||
def select(
|
||||
pool: list[dict],
|
||||
*,
|
||||
seed: int,
|
||||
rare_target: int,
|
||||
common_target: int,
|
||||
common_strata: set[str],
|
||||
) -> tuple[list[dict], list[dict]]:
|
||||
"""Draw each stratum independently and report unfilled cells."""
|
||||
|
||||
by_cell: dict[tuple[str, str], list[dict]] = defaultdict(list)
|
||||
for record in pool:
|
||||
by_cell[(record["language"], record["_stratum"])].append(record)
|
||||
|
||||
all_strata = (*INTENT_PRIORITY, REPLYABLE_STRATUM, NEGATIVE_STRATUM)
|
||||
selected: list[dict] = []
|
||||
gaps: list[dict] = []
|
||||
for language in LANGUAGES:
|
||||
for stratum in all_strata:
|
||||
target = common_target if stratum in common_strata else rare_target
|
||||
available = by_cell.get((language, stratum), [])
|
||||
drawn = sorted(
|
||||
available,
|
||||
key=lambda value: stable_priority(value["id"], seed, stratum),
|
||||
)[:target]
|
||||
probability = len(drawn) / len(available) if available else 0.0
|
||||
for record in drawn:
|
||||
record["_inclusionProbability"] = probability
|
||||
record["_stratumAvailable"] = len(available)
|
||||
record["_stratumSelected"] = len(drawn)
|
||||
selected.append(record)
|
||||
if len(drawn) < target:
|
||||
gaps.append(
|
||||
{
|
||||
"language": language,
|
||||
"stratum": stratum,
|
||||
"target": target,
|
||||
"available": len(available),
|
||||
"selected": len(drawn),
|
||||
"mustAuthor": target - len(drawn),
|
||||
}
|
||||
)
|
||||
return selected, gaps
|
||||
|
||||
|
||||
def write_jsonl(path: Path, records: list[dict]) -> None:
|
||||
path.write_text(
|
||||
"\n".join(
|
||||
json.dumps(record, ensure_ascii=False, sort_keys=True)
|
||||
for record in records
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
arguments = parse_arguments()
|
||||
sources = arguments.source or list(DEFAULT_SOURCES)
|
||||
common_strata = {
|
||||
value.strip() for value in arguments.common_strata.split(",") if value.strip()
|
||||
}
|
||||
|
||||
protected = protected_fingerprints(arguments.training_corpus)
|
||||
pool, rejected = eligible_pool(
|
||||
sources,
|
||||
minimum_characters=arguments.minimum_characters,
|
||||
maximum_characters=arguments.maximum_characters,
|
||||
protected=protected,
|
||||
)
|
||||
selected, gaps = select(
|
||||
pool,
|
||||
seed=arguments.seed,
|
||||
rare_target=arguments.rare_stratum_target,
|
||||
common_target=arguments.common_stratum_target,
|
||||
common_strata=common_strata,
|
||||
)
|
||||
|
||||
output_directory = arguments.output_directory
|
||||
output_directory.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
queue: list[dict] = []
|
||||
provenance: list[dict] = []
|
||||
template: list[dict] = []
|
||||
for index, source in enumerate(
|
||||
sorted(
|
||||
selected,
|
||||
key=lambda value: stable_priority(value["id"], arguments.seed, "queue"),
|
||||
),
|
||||
start=1,
|
||||
):
|
||||
review_id = f"product-blind-v2-{index:05d}"
|
||||
queue.append(
|
||||
{
|
||||
"id": review_id,
|
||||
"text": source["_text"],
|
||||
"language": source["language"],
|
||||
"annotationStatus": "unreviewed",
|
||||
}
|
||||
)
|
||||
provenance.append(
|
||||
{
|
||||
"id": review_id,
|
||||
"language": source["language"],
|
||||
"sourceRecordID": source["id"],
|
||||
"sourceDataset": source.get("sourceDataset"),
|
||||
"sourceLicense": source.get("sourceLicense"),
|
||||
"sourceURL": source.get("sourceURL"),
|
||||
"family": source.get("family"),
|
||||
"samplingStratum": source["_stratum"],
|
||||
"inclusionProbability": source["_inclusionProbability"],
|
||||
"stratumAvailable": source["_stratumAvailable"],
|
||||
"stratumSelected": source["_stratumSelected"],
|
||||
"weakSourceLabels": {
|
||||
field: source.get(field) for field in INTENT_FIELDS
|
||||
},
|
||||
}
|
||||
)
|
||||
template.append(
|
||||
{
|
||||
"id": review_id,
|
||||
**{field: None for field in INTENT_FIELDS},
|
||||
"sentiment": None,
|
||||
"ambiguous": None,
|
||||
"confidence": None,
|
||||
"evidence": "",
|
||||
"notes": "",
|
||||
}
|
||||
)
|
||||
|
||||
queue_path = output_directory / "review-queue.jsonl"
|
||||
provenance_path = output_directory / "sealed-provenance.jsonl"
|
||||
write_jsonl(queue_path, queue)
|
||||
write_jsonl(provenance_path, provenance)
|
||||
write_jsonl(output_directory / "annotator-a.jsonl", template)
|
||||
write_jsonl(output_directory / "annotator-b.jsonl", template)
|
||||
|
||||
cell_counts: Counter = Counter(
|
||||
f"{value['language']}:{value['samplingStratum']}" for value in provenance
|
||||
)
|
||||
manifest = {
|
||||
"schemaVersion": 1,
|
||||
"seed": arguments.seed,
|
||||
"status": "awaiting-double-human-annotation",
|
||||
"humanReviewComplete": False,
|
||||
"policy": (
|
||||
"Evaluation-only stratified queue drawn from real licensed text. "
|
||||
"Weak source labels are stratification input only and are never "
|
||||
"gold. Never merge these records into training."
|
||||
),
|
||||
"design": (
|
||||
"Single-stage stratified sample. Each record belongs to exactly one "
|
||||
"(language, stratum) cell; divide by inclusionProbability for "
|
||||
"prevalence-corrected Horvitz-Thompson metrics."
|
||||
),
|
||||
"sources": [str(path) for path in sources],
|
||||
"clipboardShapeFilter": {
|
||||
"excludedFamilyPrefixes": list(EXCLUDED_FAMILY_PREFIXES),
|
||||
"minimumCharacters": arguments.minimum_characters,
|
||||
"maximumCharacters": arguments.maximum_characters,
|
||||
},
|
||||
"poolRecords": len(pool),
|
||||
"rejected": dict(rejected),
|
||||
"records": len(queue),
|
||||
"languages": dict(Counter(value["language"] for value in queue)),
|
||||
"cells": dict(sorted(cell_counts.items())),
|
||||
"unfilledCells": gaps,
|
||||
"authoringRequired": sum(gap["mustAuthor"] for gap in gaps),
|
||||
"validation": {
|
||||
"duplicateNormalizedTexts": len(queue)
|
||||
- len({fingerprint(value["text"]) for value in queue}),
|
||||
"configuredTrainingOverlap": sum(
|
||||
fingerprint(value["text"]) in protected for value in queue
|
||||
),
|
||||
"containsDetectedPII": any(
|
||||
PII_PATTERN.search(value["text"]) for value in queue
|
||||
),
|
||||
},
|
||||
"artifacts": {
|
||||
"reviewQueueSHA256": file_sha256(queue_path),
|
||||
"sealedProvenanceSHA256": file_sha256(provenance_path),
|
||||
},
|
||||
}
|
||||
(output_directory / "manifest.json").write_text(
|
||||
json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,257 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate LLM-authored clipboard corpus batches before they enter any split.
|
||||
|
||||
Authored records exist because four socially scarce intents (blessing,
|
||||
invitation, confirmationDecision, scheduleNegotiation) have no commercially
|
||||
licensed real-text source. Their whole value is that they are not
|
||||
template-generated, so this script measures template-ness directly instead of
|
||||
trusting the author.
|
||||
|
||||
Checks:
|
||||
* schema, label states, and duplicate ids/text
|
||||
* PII shapes and frozen-holdout overlap
|
||||
* distinct-n diversity and the most repeated 4-gram
|
||||
* prefix concentration, which is how slot-filled templates leak a fingerprint
|
||||
* per-family label balance, so a family is never a pure label proxy
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import unicodedata
|
||||
from collections import Counter, defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
MODEL_TRAINING = Path("ModelTraining/ClipboardSemantics")
|
||||
INTENT_FIELDS = (
|
||||
"task",
|
||||
"question",
|
||||
"invitation",
|
||||
"complaint",
|
||||
"scheduleNegotiation",
|
||||
"confirmationDecision",
|
||||
"followUpReminder",
|
||||
"blessing",
|
||||
"replyable",
|
||||
)
|
||||
SENTIMENTS = {"positive", "neutral", "negative"}
|
||||
REQUIRED_FIELDS = ("id", "text", "language", "family", "sentiment", *INTENT_FIELDS)
|
||||
LANGUAGES = {"en", "zh-Hans"}
|
||||
PII_PATTERN = re.compile(
|
||||
r"(?:[\w.+-]+@[\w.-]+\.\w+)|(?:\+?\d[\d ()-]{8,}\d)|"
|
||||
r"(?:\b\d{3}-\d{2}-\d{4}\b)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# Diversity floors. Template corpora sit far below these because slot filling
|
||||
# reuses one skeleton per family.
|
||||
MINIMUM_DISTINCT_4GRAM_RATIO = 0.80
|
||||
MAXIMUM_TOP_4GRAM_SHARE = 0.06
|
||||
MAXIMUM_PREFIX_SHARE = 0.10
|
||||
|
||||
|
||||
def parse_arguments() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("batches", nargs="+", type=Path)
|
||||
parser.add_argument(
|
||||
"--holdout",
|
||||
action="append",
|
||||
type=Path,
|
||||
default=[],
|
||||
help="Frozen holdout JSONL whose text must not be reused.",
|
||||
)
|
||||
parser.add_argument("--report", type=Path)
|
||||
parser.add_argument(
|
||||
"--compare-corpus",
|
||||
type=Path,
|
||||
default=MODEL_TRAINING / "clipboard_semantic_corpus.jsonl",
|
||||
help="Existing template corpus, scored with the same diversity metrics.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def normalized_text(value: str) -> str:
|
||||
value = unicodedata.normalize("NFKC", value)
|
||||
return " ".join(value.split()).strip()
|
||||
|
||||
|
||||
def fingerprint(value: str) -> str:
|
||||
return normalized_text(value).casefold()
|
||||
|
||||
|
||||
def load_records(path: Path) -> list[dict]:
|
||||
return [
|
||||
json.loads(line)
|
||||
for line in path.read_text(encoding="utf-8").splitlines()
|
||||
if line.strip()
|
||||
]
|
||||
|
||||
|
||||
def character_ngrams(value: str, size: int) -> list[str]:
|
||||
text = normalized_text(value)
|
||||
return [text[index : index + size] for index in range(len(text) - size + 1)]
|
||||
|
||||
|
||||
def diversity_metrics(texts: list[str]) -> dict:
|
||||
"""Distinct-n and concentration statistics over character 4-grams."""
|
||||
|
||||
grams: Counter = Counter()
|
||||
for text in texts:
|
||||
grams.update(character_ngrams(text, 4))
|
||||
total = sum(grams.values())
|
||||
if not total:
|
||||
return {"distinct4GramRatio": 0.0, "topGramShare": 1.0, "topGram": ""}
|
||||
top_gram, top_count = grams.most_common(1)[0]
|
||||
prefixes = Counter(normalized_text(text)[:6] for text in texts)
|
||||
top_prefix, prefix_count = prefixes.most_common(1)[0]
|
||||
return {
|
||||
"distinct4GramRatio": round(len(grams) / total, 4),
|
||||
"topGramShare": round(top_count / total, 4),
|
||||
"topGram": top_gram,
|
||||
"topPrefix": top_prefix,
|
||||
"topPrefixShare": round(prefix_count / len(texts), 4),
|
||||
"totalGrams": total,
|
||||
"distinctGrams": len(grams),
|
||||
}
|
||||
|
||||
|
||||
def validate(records: list[dict], protected: set[str]) -> list[str]:
|
||||
errors: list[str] = []
|
||||
seen_ids: set[str] = set()
|
||||
seen_text: set[str] = set()
|
||||
for record in records:
|
||||
identifier = record.get("id", "<missing>")
|
||||
for field in REQUIRED_FIELDS:
|
||||
if field not in record:
|
||||
errors.append(f"{identifier}: missing field {field}")
|
||||
if identifier in seen_ids:
|
||||
errors.append(f"{identifier}: duplicate id")
|
||||
seen_ids.add(identifier)
|
||||
|
||||
text = normalized_text(record.get("text") or "")
|
||||
if not text:
|
||||
errors.append(f"{identifier}: empty text")
|
||||
continue
|
||||
key = fingerprint(text)
|
||||
if key in seen_text:
|
||||
errors.append(f"{identifier}: duplicate text")
|
||||
seen_text.add(key)
|
||||
if key in protected:
|
||||
errors.append(f"{identifier}: text overlaps a frozen holdout")
|
||||
if PII_PATTERN.search(text):
|
||||
errors.append(f"{identifier}: detected PII shape")
|
||||
if record.get("language") not in LANGUAGES:
|
||||
errors.append(f"{identifier}: unsupported language {record.get('language')}")
|
||||
if record.get("sentiment") not in SENTIMENTS:
|
||||
errors.append(f"{identifier}: bad sentiment {record.get('sentiment')}")
|
||||
for field in INTENT_FIELDS:
|
||||
if not isinstance(record.get(field), bool):
|
||||
errors.append(f"{identifier}: {field} must be a boolean")
|
||||
return errors
|
||||
|
||||
|
||||
def family_balance(records: list[dict]) -> dict:
|
||||
by_family: dict[str, list[dict]] = defaultdict(list)
|
||||
for record in records:
|
||||
by_family[record["family"]].append(record)
|
||||
return {
|
||||
family: {
|
||||
"records": len(values),
|
||||
"positives": {
|
||||
field: sum(1 for value in values if value.get(field) is True)
|
||||
for field in INTENT_FIELDS
|
||||
if any(value.get(field) is True for value in values)
|
||||
},
|
||||
}
|
||||
for family, values in sorted(by_family.items())
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
arguments = parse_arguments()
|
||||
records: list[dict] = []
|
||||
for path in arguments.batches:
|
||||
records.extend(load_records(path))
|
||||
|
||||
protected: set[str] = set()
|
||||
for path in arguments.holdout:
|
||||
for record in load_records(path):
|
||||
protected.add(fingerprint(record["text"]))
|
||||
|
||||
errors = validate(records, protected)
|
||||
texts = [record["text"] for record in records if record.get("text")]
|
||||
metrics = diversity_metrics(texts)
|
||||
|
||||
by_language = {
|
||||
language: diversity_metrics(
|
||||
[record["text"] for record in records if record.get("language") == language]
|
||||
)
|
||||
for language in sorted({record.get("language") for record in records} - {None})
|
||||
}
|
||||
|
||||
comparison = {}
|
||||
if arguments.compare_corpus and arguments.compare_corpus.is_file():
|
||||
compare_records = load_records(arguments.compare_corpus)
|
||||
for language in sorted(by_language):
|
||||
subset = [
|
||||
record["text"]
|
||||
for record in compare_records
|
||||
if record.get("language") == language
|
||||
]
|
||||
if subset:
|
||||
comparison[language] = diversity_metrics(subset)
|
||||
|
||||
gates: dict[str, dict] = {}
|
||||
for language, values in by_language.items():
|
||||
gates[language] = {
|
||||
"distinct4GramRatio": {
|
||||
"actual": values["distinct4GramRatio"],
|
||||
"required": MINIMUM_DISTINCT_4GRAM_RATIO,
|
||||
"passed": values["distinct4GramRatio"] >= MINIMUM_DISTINCT_4GRAM_RATIO,
|
||||
},
|
||||
"topGramShare": {
|
||||
"actual": values["topGramShare"],
|
||||
"maximum": MAXIMUM_TOP_4GRAM_SHARE,
|
||||
"passed": values["topGramShare"] <= MAXIMUM_TOP_4GRAM_SHARE,
|
||||
},
|
||||
"topPrefixShare": {
|
||||
"actual": values["topPrefixShare"],
|
||||
"maximum": MAXIMUM_PREFIX_SHARE,
|
||||
"passed": values["topPrefixShare"] <= MAXIMUM_PREFIX_SHARE,
|
||||
},
|
||||
}
|
||||
|
||||
all_passed = not errors and all(
|
||||
check["passed"] for language in gates.values() for check in language.values()
|
||||
)
|
||||
report = {
|
||||
"schemaVersion": 1,
|
||||
"batches": [str(path) for path in arguments.batches],
|
||||
"records": len(records),
|
||||
"languages": dict(Counter(record.get("language") for record in records)),
|
||||
"families": len({record.get("family") for record in records}),
|
||||
"positivesByIntent": {
|
||||
field: sum(1 for record in records if record.get(field) is True)
|
||||
for field in INTENT_FIELDS
|
||||
},
|
||||
"diversity": metrics,
|
||||
"diversityByLanguage": by_language,
|
||||
"templateCorpusComparison": comparison,
|
||||
"gates": gates,
|
||||
"familyBalance": family_balance(records),
|
||||
"errors": errors,
|
||||
"allChecksPassed": all_passed,
|
||||
}
|
||||
output = json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True)
|
||||
if arguments.report:
|
||||
arguments.report.write_text(output + "\n", encoding="utf-8")
|
||||
print(output)
|
||||
return 0 if all_passed else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user