Rocky
aa37067f79
- Add reproducible v6 boundary, blessing, and consensus-adjudication corpora, plus the tiny-transformer trainer and v6 release-gate evaluator that gate every candidate on the deployed baselines. - Wire consensus-label merging, product-policy anchor evaluation, and sealed blessing benchmark review with their pytest coverage. - Refresh open-training corpus generation, iterative retraining runner, and random-holdout evaluation so v6 candidates can be benchmarked end-to-end.
1172 lines
38 KiB
Python
Executable File
1172 lines
38 KiB
Python
Executable File
#!/usr/bin/env python3
|
||
"""Run twenty deterministic weakly supervised clipboard-intent research rounds.
|
||
|
||
This Linux-compatible harness is a model-selection surrogate. It never replaces
|
||
the Apple Create ML artifacts: the selected data policy must be replayed by
|
||
`train_models.swift` on macOS before a deployable candidate exists.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import copy
|
||
import hashlib
|
||
import json
|
||
import math
|
||
import re
|
||
import time
|
||
from collections import defaultdict, deque
|
||
from dataclasses import asdict, dataclass
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
import numpy as np
|
||
from scipy import sparse
|
||
from sklearn.feature_extraction.text import TfidfVectorizer
|
||
from sklearn.linear_model import SGDClassifier
|
||
from sklearn.pipeline import FeatureUnion
|
||
|
||
SEED = 20260827
|
||
INTENTS = (
|
||
"task",
|
||
"question",
|
||
"invitation",
|
||
"complaint",
|
||
"scheduleNegotiation",
|
||
"confirmationDecision",
|
||
"followUpReminder",
|
||
"blessing",
|
||
"replyableMessage",
|
||
"assistantCommand",
|
||
"informationQuery",
|
||
"systemNotification",
|
||
)
|
||
LEGACY_IMPLICITLY_KNOWN_INTENTS = frozenset(INTENTS[:9])
|
||
OUTPUT_DIRECTORY = Path("ModelTraining/ClipboardSemantics/IterativeResearch")
|
||
BASE_CORPUS = Path(
|
||
"ModelTraining/ClipboardSemantics/clipboard_semantic_corpus.jsonl"
|
||
)
|
||
OPEN_CORPUS = Path("ModelTraining/ClipboardSemantics/open-training-corpus.jsonl")
|
||
RANDOM_HOLDOUT = Path(
|
||
"ModelTraining/ClipboardSemantics/random-holdout-corpus.jsonl"
|
||
)
|
||
TARGETED_HOLDOUT = Path(
|
||
"ModelTraining/ClipboardSemantics/targeted-release-holdout-corpus.jsonl"
|
||
)
|
||
COMPREHENSIVE_HOLDOUT = Path(
|
||
"ModelTraining/ClipboardSemantics/comprehensive-online-holdout-corpus.jsonl"
|
||
)
|
||
|
||
EXPLICIT_TASK_MARKERS = (
|
||
"请",
|
||
"麻烦",
|
||
"能否",
|
||
"可以请你",
|
||
"由你",
|
||
"交给你",
|
||
"需要你",
|
||
"你负责",
|
||
"下一步",
|
||
"行动项",
|
||
"please ",
|
||
"can you",
|
||
"could you",
|
||
"would you",
|
||
"assigned to you",
|
||
"you are responsible",
|
||
"we need you",
|
||
"would like you",
|
||
"counting on you",
|
||
"take ownership",
|
||
"your task",
|
||
"next action",
|
||
"complete the",
|
||
"finish the",
|
||
"send it to",
|
||
"deliver it to",
|
||
)
|
||
BLESSING_MARKERS = (
|
||
"生日快乐",
|
||
"新年快乐",
|
||
"春节快乐",
|
||
"节日快乐",
|
||
"圣诞快乐",
|
||
"中秋快乐",
|
||
"恭喜",
|
||
"预祝",
|
||
"祝你",
|
||
"祝您",
|
||
"祝大家",
|
||
"祝他",
|
||
"祝她",
|
||
"愿你",
|
||
"愿您",
|
||
"happy birthday",
|
||
"happy new year",
|
||
"merry christmas",
|
||
"happy holidays",
|
||
"congratulations",
|
||
"congrats",
|
||
"best wishes",
|
||
"good luck",
|
||
"wishing you",
|
||
"wish you",
|
||
"wish him",
|
||
"wish her",
|
||
"wish them",
|
||
"let us wish",
|
||
"let's wish",
|
||
"we wish",
|
||
"may you",
|
||
)
|
||
BLESSING_EXCLUSIONS = (
|
||
"祝福模板",
|
||
"祝福语模板",
|
||
"文章引用",
|
||
"搜索词",
|
||
"文档里收录",
|
||
"贺卡名单",
|
||
"收集祝福",
|
||
"greeting template",
|
||
"message template",
|
||
"the article quotes",
|
||
"search phrase",
|
||
"document contains",
|
||
"quotes the phrase",
|
||
"如何描述生日快乐",
|
||
"怎么说生日快乐",
|
||
"如何写生日祝福",
|
||
"how would you describe a happy birthday",
|
||
"how do you say happy birthday",
|
||
"what does happy birthday mean",
|
||
"宁愿你",
|
||
"祝你倒闭",
|
||
"祝你去死",
|
||
"祝你倒霉",
|
||
"祝你失败",
|
||
"祝你完蛋",
|
||
)
|
||
QUESTION_PATTERN = re.compile(
|
||
r"(?:[?？]|^(?:who|what|when|where|why|how|which|do|does|did|can|could|"
|
||
r"would|will|is|are|was|were|have|has|should)\b|"
|
||
r"(?:谁|什么|何时|哪里|为什么|怎么|如何|哪个|哪种|是否|能否|吗|么|呢|几))",
|
||
re.IGNORECASE,
|
||
)
|
||
INVITATION_PATTERN = re.compile(
|
||
r"(?:\b(?:join us|come (?:to|over)|you(?:'re| are) invited|"
|
||
r"would love (?:you|for you)|invitation|party|dinner together)\b|"
|
||
r"(?:邀请|一起来|一起去|来我家|参加.+(?:聚会|晚餐|活动)|聚个|吃饭吧))",
|
||
re.IGNORECASE,
|
||
)
|
||
COMPLAINT_PATTERN = re.compile(
|
||
r"(?:\b(?:broken|not working|doesn['’]?t work|failed|failure|crash(?:ed|es)?|"
|
||
r"refund|charged|unacceptable|terrible|worst|frustrat(?:ed|ing)|"
|
||
r"disappoint(?:ed|ing)|still can['’]?t|never received)\b|"
|
||
r"(?:坏了|不能用|无法|失败|崩溃|退款|扣费|太差|糟糕|失望|一直没有|"
|
||
r"仍然没有|延迟|投诉|没人回复|没有回复))",
|
||
re.IGNORECASE,
|
||
)
|
||
SCHEDULE_PATTERN = re.compile(
|
||
r"(?:\b(?:reschedule|move (?:it|the)|instead|work better|available|"
|
||
r"does .+ work|or .+(?:morning|afternoon|evening|am|pm))\b|"
|
||
r"(?:改到|改成|改期|还是|哪个时间|哪天|方便吗|可以吗|不行.+(?:周|点|号)))",
|
||
re.IGNORECASE,
|
||
)
|
||
CONFIRMATION_PATTERN = re.compile(
|
||
r"(?:\b(?:approve|approved|confirm|confirmed|accept|accepted|reject|"
|
||
r"rejected|choose|chosen|selected|agreed|proceed with|go with)\b|"
|
||
r"(?:确认|同意|批准|接受|拒绝|选择|采用|就按|决定用|没问题.+执行))",
|
||
re.IGNORECASE,
|
||
)
|
||
FOLLOW_UP_PATTERN = re.compile(
|
||
r"(?:\b(?:follow up|follow-up|check back|remind me|reminder|circle back|"
|
||
r"after .+(?:ships|arrives|responds)|if .+(?:no response|haven['’]?t heard))\b|"
|
||
r"(?:跟进|提醒我|提醒一下|稍后再|之后再|如果.+(?:没|没有).+再))",
|
||
re.IGNORECASE,
|
||
)
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class RoundConfiguration:
|
||
round: int
|
||
char_min: int
|
||
char_max: int
|
||
word_max: int
|
||
min_df: int
|
||
max_char_features: int
|
||
max_word_features: int
|
||
alpha: float
|
||
l1_ratio: float
|
||
hard_example_weight: float
|
||
augmentation: str
|
||
external_weight: float
|
||
pseudo_weight: float
|
||
|
||
|
||
@dataclass
|
||
class BinaryCounts:
|
||
true_positive: int = 0
|
||
true_negative: int = 0
|
||
false_positive: int = 0
|
||
false_negative: int = 0
|
||
|
||
def update(self, expected: np.ndarray, predicted: np.ndarray) -> None:
|
||
self.true_positive += int(np.sum(expected & predicted))
|
||
self.true_negative += int(np.sum(~expected & ~predicted))
|
||
self.false_positive += int(np.sum(~expected & predicted))
|
||
self.false_negative += int(np.sum(expected & ~predicted))
|
||
|
||
def metrics(self) -> dict[str, float | int]:
|
||
predicted_positive = self.true_positive + self.false_positive
|
||
actual_positive = self.true_positive + self.false_negative
|
||
total = predicted_positive + self.true_negative + self.false_negative
|
||
precision = (
|
||
self.true_positive / predicted_positive if predicted_positive else 0
|
||
)
|
||
recall = self.true_positive / actual_positive if actual_positive else 0
|
||
f1 = (
|
||
2 * precision * recall / (precision + recall)
|
||
if precision + recall
|
||
else 0
|
||
)
|
||
return {
|
||
"total": total,
|
||
"truePositive": self.true_positive,
|
||
"trueNegative": self.true_negative,
|
||
"falsePositive": self.false_positive,
|
||
"falseNegative": self.false_negative,
|
||
"predictedPositive": predicted_positive,
|
||
"precision": rounded(precision),
|
||
"recall": rounded(recall),
|
||
"f1": rounded(f1),
|
||
"wilsonPrecisionLower95": wilson_lower(
|
||
self.true_positive, predicted_positive
|
||
),
|
||
}
|
||
|
||
|
||
def rounded(value: float) -> float:
|
||
if not math.isfinite(value):
|
||
return 0
|
||
return round(float(value), 4)
|
||
|
||
|
||
def wilson_lower(successes: int, total: int) -> float:
|
||
if total == 0:
|
||
return 0
|
||
z = 1.959963984540054
|
||
proportion = successes / total
|
||
denominator = 1 + z * z / total
|
||
center = proportion + z * z / (2 * total)
|
||
adjustment = z * math.sqrt(
|
||
(proportion * (1 - proportion) + z * z / (4 * total)) / total
|
||
)
|
||
return rounded((center - adjustment) / denominator)
|
||
|
||
|
||
def read_json_lines(path: Path) -> list[dict[str, Any]]:
|
||
if not path.is_file():
|
||
return []
|
||
return [
|
||
json.loads(line)
|
||
for line in path.read_text(encoding="utf-8").splitlines()
|
||
if line.strip()
|
||
]
|
||
|
||
|
||
def stable_value(value: str) -> int:
|
||
return int.from_bytes(hashlib.sha256(value.encode()).digest()[:8], "big")
|
||
|
||
|
||
def record_label(record: dict[str, Any], intent: str) -> bool:
|
||
key = "replyable" if intent == "replyableMessage" else intent
|
||
return bool(record.get(key, False))
|
||
|
||
|
||
def is_known(record: dict[str, Any], intent: str) -> bool:
|
||
known = record.get("knownLabels")
|
||
if known is None:
|
||
return intent in LEGACY_IMPLICITLY_KNOWN_INTENTS
|
||
return intent in known or (
|
||
intent == "replyableMessage" and "replyable" in known
|
||
)
|
||
|
||
|
||
def has_explicit_blessing(text: str) -> bool:
|
||
normalized = text.casefold()
|
||
return not any(value in normalized for value in BLESSING_EXCLUSIONS) and any(
|
||
value in normalized for value in BLESSING_MARKERS
|
||
)
|
||
|
||
|
||
def has_explicit_task(text: str) -> bool:
|
||
normalized = text.casefold()
|
||
return any(value in normalized for value in EXPLICIT_TASK_MARKERS)
|
||
|
||
|
||
def has_explicit_evidence(intent: str, text: str) -> bool:
|
||
if intent == "task":
|
||
return has_explicit_task(text)
|
||
if intent == "question":
|
||
return bool(QUESTION_PATTERN.search(text.strip()))
|
||
if intent == "invitation":
|
||
return bool(INVITATION_PATTERN.search(text))
|
||
if intent == "complaint":
|
||
return bool(COMPLAINT_PATTERN.search(text))
|
||
if intent == "scheduleNegotiation":
|
||
return bool(SCHEDULE_PATTERN.search(text))
|
||
if intent == "confirmationDecision":
|
||
return bool(CONFIRMATION_PATTERN.search(text))
|
||
if intent == "followUpReminder":
|
||
return bool(FOLLOW_UP_PATTERN.search(text))
|
||
if intent == "blessing":
|
||
return has_explicit_blessing(text)
|
||
return True
|
||
|
||
|
||
def configurations() -> list[RoundConfiguration]:
|
||
axes = [
|
||
(2, 5, 2, 2, 45_000, 22_000, 2.0e-5, 0.00, 1.0, "none"),
|
||
(3, 5, 2, 2, 50_000, 25_000, 1.5e-5, 0.00, 1.0, "none"),
|
||
(2, 6, 2, 2, 55_000, 25_000, 1.0e-5, 0.05, 1.2, "none"),
|
||
(3, 6, 3, 2, 60_000, 30_000, 8.0e-6, 0.05, 1.2, "none"),
|
||
(2, 5, 3, 1, 60_000, 32_000, 1.2e-5, 0.10, 1.4, "none"),
|
||
(2, 6, 2, 2, 65_000, 28_000, 8.0e-6, 0.10, 1.5, "punctuation"),
|
||
(3, 6, 3, 2, 65_000, 32_000, 6.0e-6, 0.10, 1.5, "punctuation"),
|
||
(2, 7, 2, 2, 70_000, 28_000, 5.0e-6, 0.15, 1.7, "punctuation"),
|
||
(3, 7, 3, 2, 70_000, 35_000, 4.0e-6, 0.15, 1.7, "punctuation"),
|
||
(2, 6, 3, 1, 75_000, 38_000, 6.0e-6, 0.20, 1.8, "punctuation"),
|
||
(2, 6, 2, 2, 70_000, 30_000, 5.0e-6, 0.10, 1.8, "prefix"),
|
||
(3, 6, 3, 2, 75_000, 35_000, 4.0e-6, 0.15, 2.0, "prefix"),
|
||
(2, 7, 3, 2, 80_000, 38_000, 3.0e-6, 0.15, 2.0, "prefix"),
|
||
(3, 7, 3, 1, 80_000, 40_000, 2.5e-6, 0.20, 2.2, "prefix"),
|
||
(2, 6, 3, 1, 85_000, 42_000, 3.5e-6, 0.20, 2.2, "prefix"),
|
||
(2, 7, 3, 2, 80_000, 38_000, 3.0e-6, 0.15, 2.3, "numeric"),
|
||
(3, 7, 3, 1, 85_000, 42_000, 2.0e-6, 0.20, 2.5, "numeric"),
|
||
(2, 8, 3, 1, 90_000, 45_000, 1.5e-6, 0.20, 2.5, "numeric"),
|
||
(3, 8, 3, 1, 95_000, 45_000, 1.2e-6, 0.25, 2.7, "numeric"),
|
||
(2, 8, 3, 1, 100_000, 50_000, 1.0e-6, 0.25, 3.0, "numeric"),
|
||
]
|
||
return [
|
||
RoundConfiguration(
|
||
round=index,
|
||
char_min=values[0],
|
||
char_max=values[1],
|
||
word_max=values[2],
|
||
min_df=values[3],
|
||
max_char_features=values[4],
|
||
max_word_features=values[5],
|
||
alpha=values[6],
|
||
l1_ratio=values[7],
|
||
hard_example_weight=values[8],
|
||
augmentation=values[9],
|
||
external_weight=min(1.0, 0.55 + index * 0.02),
|
||
pseudo_weight=min(0.45, 0.15 + index * 0.015),
|
||
)
|
||
for index, values in enumerate(axes, start=1)
|
||
]
|
||
|
||
|
||
def augmented_text(text: str, mode: str) -> str:
|
||
if mode == "none":
|
||
return text
|
||
if mode == "punctuation":
|
||
return re.sub(r"[!?！？。，、；;:：]+", " ", text).strip()
|
||
if mode == "prefix":
|
||
return re.sub(
|
||
r"^(?:hey|hi|hello|please note that|顺便问下|你好|您好|那个|嗯)[,，:： ]*",
|
||
"",
|
||
text,
|
||
flags=re.IGNORECASE,
|
||
).strip()
|
||
if mode == "numeric":
|
||
return re.sub(r"\d+(?::\d+)?", "<num>", text).casefold().strip()
|
||
return text
|
||
|
||
|
||
def augmented_records(
|
||
records: list[dict[str, Any]],
|
||
configuration: RoundConfiguration,
|
||
) -> list[dict[str, Any]]:
|
||
if configuration.augmentation == "none":
|
||
return []
|
||
result = []
|
||
for record in records:
|
||
if stable_value(
|
||
f"{configuration.round}|augment|{record['id']}"
|
||
) % 4:
|
||
continue
|
||
transformed = augmented_text(
|
||
record["text"], configuration.augmentation
|
||
)
|
||
if not transformed or transformed == record["text"]:
|
||
continue
|
||
value = copy.copy(record)
|
||
value["id"] = f"round-{configuration.round}-aug-{record['id']}"
|
||
value["text"] = transformed
|
||
value["sourceDataset"] = "deterministic-label-preserving-augmentation"
|
||
value["_augmentation"] = True
|
||
result.append(value)
|
||
return result
|
||
|
||
|
||
def vectorizer(configuration: RoundConfiguration) -> FeatureUnion:
|
||
return FeatureUnion(
|
||
[
|
||
(
|
||
"char",
|
||
TfidfVectorizer(
|
||
analyzer="char_wb",
|
||
ngram_range=(
|
||
configuration.char_min,
|
||
configuration.char_max,
|
||
),
|
||
min_df=configuration.min_df,
|
||
max_features=configuration.max_char_features,
|
||
sublinear_tf=True,
|
||
lowercase=True,
|
||
dtype=np.float32,
|
||
),
|
||
),
|
||
(
|
||
"word",
|
||
TfidfVectorizer(
|
||
analyzer="word",
|
||
ngram_range=(1, configuration.word_max),
|
||
token_pattern=r"(?u)\b\w+\b",
|
||
min_df=configuration.min_df,
|
||
max_features=configuration.max_word_features,
|
||
sublinear_tf=True,
|
||
lowercase=True,
|
||
strip_accents="unicode",
|
||
dtype=np.float32,
|
||
),
|
||
),
|
||
]
|
||
)
|
||
|
||
|
||
def sample_weight(
|
||
record: dict[str, Any],
|
||
configuration: RoundConfiguration,
|
||
) -> float:
|
||
weight = 1.0
|
||
if record.get("knownLabels") is not None:
|
||
weight *= configuration.external_weight
|
||
weight *= float(record.get("sampleWeight", 1.0))
|
||
if record.get("_augmentation"):
|
||
weight *= 0.60
|
||
family = str(record.get("family", "")).casefold()
|
||
positive_count = sum(record_label(record, intent) for intent in INTENTS)
|
||
if "boundary" in family or positive_count > 1:
|
||
weight *= configuration.hard_example_weight
|
||
return weight
|
||
|
||
|
||
def select_threshold(
|
||
expected: np.ndarray,
|
||
probabilities: np.ndarray,
|
||
*,
|
||
minimum_precision: float = 0.95,
|
||
minimum_predictions: int = 10,
|
||
) -> dict[str, Any]:
|
||
options = []
|
||
for threshold in np.linspace(0.50, 0.995, 100):
|
||
predicted = probabilities >= threshold
|
||
counts = BinaryCounts()
|
||
counts.update(expected, predicted)
|
||
metrics = counts.metrics()
|
||
if (
|
||
metrics["precision"] >= minimum_precision
|
||
and metrics["predictedPositive"] >= minimum_predictions
|
||
):
|
||
options.append((metrics["recall"], metrics["precision"], -threshold, metrics))
|
||
if not options:
|
||
return {
|
||
"threshold": 1.0,
|
||
"metrics": BinaryCounts().metrics(),
|
||
"abstained": True,
|
||
}
|
||
recall, _, negative_threshold, metrics = max(options)
|
||
del recall
|
||
return {
|
||
"threshold": rounded(-negative_threshold),
|
||
"metrics": metrics,
|
||
"abstained": False,
|
||
}
|
||
|
||
|
||
def calibrated_thresholds(
|
||
records: list[dict[str, Any]],
|
||
probabilities: dict[str, np.ndarray],
|
||
) -> dict[str, dict[str, Any]]:
|
||
languages = np.array([record["language"] for record in records])
|
||
result: dict[str, dict[str, Any]] = {}
|
||
for intent in INTENTS:
|
||
known_mask = np.array(
|
||
[is_known(record, intent) for record in records],
|
||
dtype=bool,
|
||
)
|
||
expected = np.array(
|
||
[record_label(record, intent) for record in records], dtype=bool
|
||
)
|
||
selection = select_threshold(
|
||
expected[known_mask],
|
||
probabilities[intent][known_mask],
|
||
)
|
||
by_language = {}
|
||
for language in sorted(set(languages)):
|
||
mask = (languages == language) & known_mask
|
||
positives = int(np.sum(expected[mask]))
|
||
negatives = int(np.sum(~expected[mask]))
|
||
if positives < 20 or negatives < 20:
|
||
continue
|
||
by_language[language] = select_threshold(
|
||
expected[mask],
|
||
probabilities[intent][mask],
|
||
minimum_predictions=8,
|
||
)
|
||
result[intent] = {
|
||
**selection,
|
||
"byLanguage": by_language,
|
||
}
|
||
return result
|
||
|
||
|
||
def thresholds_for_records(
|
||
records: list[dict[str, Any]],
|
||
intent: str,
|
||
thresholds: dict[str, dict[str, Any]],
|
||
) -> np.ndarray:
|
||
global_threshold = thresholds[intent]["threshold"]
|
||
by_language = thresholds[intent]["byLanguage"]
|
||
return np.array(
|
||
[
|
||
by_language.get(record["language"], {}).get(
|
||
"threshold", global_threshold
|
||
)
|
||
for record in records
|
||
]
|
||
)
|
||
|
||
|
||
def runtime_predictions(
|
||
records: list[dict[str, Any]],
|
||
probabilities: dict[str, np.ndarray],
|
||
thresholds: dict[str, dict[str, Any]],
|
||
) -> dict[str, np.ndarray]:
|
||
predicted = {
|
||
intent: probabilities[intent]
|
||
>= thresholds_for_records(records, intent, thresholds)
|
||
for intent in INTENTS
|
||
}
|
||
for intent in INTENTS:
|
||
if intent == "replyableMessage":
|
||
continue
|
||
predicted[intent] &= np.array(
|
||
[
|
||
has_explicit_evidence(intent, record["text"])
|
||
for record in records
|
||
]
|
||
)
|
||
predicted["task"] &= ~(
|
||
(probabilities["complaint"] >= 0.60)
|
||
& np.array(
|
||
[not has_explicit_task(record["text"]) for record in records]
|
||
)
|
||
)
|
||
return predicted
|
||
|
||
|
||
def metrics_for_records(
|
||
records: list[dict[str, Any]],
|
||
probabilities: dict[str, np.ndarray],
|
||
thresholds: dict[str, dict[str, Any]],
|
||
) -> dict[str, Any]:
|
||
predictions = runtime_predictions(records, probabilities, thresholds)
|
||
per_intent = {}
|
||
for intent in INTENTS:
|
||
known_mask = np.array(
|
||
[is_known(record, intent) for record in records],
|
||
dtype=bool,
|
||
)
|
||
expected = np.array(
|
||
[record_label(record, intent) for record in records], dtype=bool
|
||
)
|
||
counts = BinaryCounts()
|
||
counts.update(expected[known_mask], predictions[intent][known_mask])
|
||
per_intent[intent] = counts.metrics()
|
||
return aggregate_metrics(per_intent)
|
||
|
||
|
||
def aggregate_metrics(per_intent: dict[str, dict[str, Any]]) -> dict[str, Any]:
|
||
return {
|
||
"recordCount": next(iter(per_intent.values()))["total"]
|
||
if per_intent
|
||
else 0,
|
||
"macroPrecision": rounded(
|
||
np.mean([value["precision"] for value in per_intent.values()])
|
||
),
|
||
"macroRecall": rounded(
|
||
np.mean([value["recall"] for value in per_intent.values()])
|
||
),
|
||
"macroF1": rounded(
|
||
np.mean([value["f1"] for value in per_intent.values()])
|
||
),
|
||
"minimumPrecision": min(
|
||
(value["precision"] for value in per_intent.values()), default=0
|
||
),
|
||
"intentsMeeting95Precision": sum(
|
||
value["precision"] >= 0.95
|
||
and value["predictedPositive"] >= 10
|
||
for value in per_intent.values()
|
||
),
|
||
"perIntent": per_intent,
|
||
}
|
||
|
||
|
||
def prediction_probabilities(
|
||
matrix: sparse.spmatrix,
|
||
models: dict[str, SGDClassifier],
|
||
) -> dict[str, np.ndarray]:
|
||
return {
|
||
intent: model.predict_proba(matrix)[:, 1]
|
||
for intent, model in models.items()
|
||
}
|
||
|
||
|
||
def pseudo_labels_from_committee(
|
||
unknown_records: list[dict[str, Any]],
|
||
history: dict[str, deque[tuple[np.ndarray, np.ndarray]]],
|
||
round_number: int,
|
||
) -> dict[str, list[tuple[int, bool]]]:
|
||
result: dict[str, list[tuple[int, bool]]] = defaultdict(list)
|
||
for intent in INTENTS:
|
||
if len(history[intent]) < 3:
|
||
continue
|
||
decisions = np.stack(
|
||
[
|
||
probabilities >= thresholds
|
||
for probabilities, thresholds in history[intent]
|
||
]
|
||
)
|
||
probabilities = np.stack(
|
||
[probabilities for probabilities, _ in history[intent]]
|
||
)
|
||
positive = np.all(decisions, axis=0) & (
|
||
np.min(probabilities, axis=0) >= 0.985
|
||
)
|
||
negative = np.all(~decisions, axis=0) & (
|
||
np.max(probabilities, axis=0) <= 0.015
|
||
)
|
||
if intent == "blessing":
|
||
positive &= np.array(
|
||
[has_explicit_blessing(record["text"]) for record in unknown_records]
|
||
)
|
||
candidates = [
|
||
(index, True)
|
||
for index in np.flatnonzero(positive)
|
||
if not is_known(unknown_records[index], intent)
|
||
] + [
|
||
(index, False)
|
||
for index in np.flatnonzero(negative)
|
||
if not is_known(unknown_records[index], intent)
|
||
]
|
||
candidates.sort(
|
||
key=lambda item: stable_value(
|
||
f"{round_number}|pseudo|{intent}|{unknown_records[item[0]]['id']}"
|
||
)
|
||
)
|
||
positive_count = 0
|
||
negative_count = 0
|
||
for item in candidates:
|
||
if item[1] and positive_count >= 250:
|
||
continue
|
||
if not item[1] and negative_count >= 250:
|
||
continue
|
||
result[intent].append(item)
|
||
if item[1]:
|
||
positive_count += 1
|
||
else:
|
||
negative_count += 1
|
||
return result
|
||
|
||
|
||
def train_round(
|
||
configuration: RoundConfiguration,
|
||
base_training: list[dict[str, Any]],
|
||
open_records: list[dict[str, Any]],
|
||
validation: list[dict[str, Any]],
|
||
test: list[dict[str, Any]],
|
||
pseudo: dict[str, list[tuple[int, bool]]],
|
||
) -> tuple[
|
||
FeatureUnion,
|
||
dict[str, SGDClassifier],
|
||
dict[str, dict[str, Any]],
|
||
dict[str, Any],
|
||
dict[str, np.ndarray],
|
||
]:
|
||
augmentations = augmented_records(base_training, configuration)
|
||
feature_records = base_training + open_records + augmentations
|
||
texts = [record["text"] for record in feature_records]
|
||
fitted_vectorizer = vectorizer(configuration)
|
||
training_matrix = fitted_vectorizer.fit_transform(texts)
|
||
validation_matrix = fitted_vectorizer.transform(
|
||
[record["text"] for record in validation]
|
||
)
|
||
test_matrix = fitted_vectorizer.transform(
|
||
[record["text"] for record in test]
|
||
)
|
||
|
||
models: dict[str, SGDClassifier] = {}
|
||
for intent_index, intent in enumerate(INTENTS):
|
||
indices = [
|
||
index
|
||
for index, record in enumerate(feature_records)
|
||
if is_known(record, intent)
|
||
]
|
||
labels = [
|
||
record_label(feature_records[index], intent) for index in indices
|
||
]
|
||
weights = [
|
||
sample_weight(feature_records[index], configuration)
|
||
for index in indices
|
||
]
|
||
pseudo_rows = pseudo.get(intent, [])
|
||
if pseudo_rows:
|
||
open_offset = len(base_training)
|
||
for open_index, label in pseudo_rows:
|
||
indices.append(open_offset + open_index)
|
||
labels.append(label)
|
||
weights.append(configuration.pseudo_weight)
|
||
model = SGDClassifier(
|
||
loss="log_loss",
|
||
penalty="elasticnet",
|
||
alpha=configuration.alpha,
|
||
l1_ratio=configuration.l1_ratio,
|
||
class_weight="balanced",
|
||
max_iter=1_500,
|
||
tol=1e-5,
|
||
random_state=SEED + configuration.round * 31 + intent_index,
|
||
average=True,
|
||
)
|
||
model.fit(
|
||
training_matrix[indices],
|
||
np.array(labels, dtype=np.int8),
|
||
sample_weight=np.array(weights, dtype=np.float32),
|
||
)
|
||
models[intent] = model
|
||
|
||
validation_probabilities = prediction_probabilities(
|
||
validation_matrix, models
|
||
)
|
||
thresholds = calibrated_thresholds(
|
||
validation, validation_probabilities
|
||
)
|
||
validation_metrics = metrics_for_records(
|
||
validation, validation_probabilities, thresholds
|
||
)
|
||
test_metrics = metrics_for_records(
|
||
test,
|
||
prediction_probabilities(test_matrix, models),
|
||
thresholds,
|
||
)
|
||
objective = rounded(
|
||
validation_metrics["macroF1"] * 0.55
|
||
+ validation_metrics["macroPrecision"] * 0.35
|
||
+ validation_metrics["macroRecall"] * 0.10
|
||
)
|
||
report = {
|
||
"round": configuration.round,
|
||
"configuration": asdict(configuration),
|
||
"trainingRecords": len(feature_records),
|
||
"featureCount": int(training_matrix.shape[1]),
|
||
"pseudoLabelCounts": {
|
||
intent: {
|
||
"positive": sum(label for _, label in values),
|
||
"negative": sum(not label for _, label in values),
|
||
}
|
||
for intent, values in pseudo.items()
|
||
},
|
||
"thresholds": thresholds,
|
||
"validation": validation_metrics,
|
||
"syntheticRegression": test_metrics,
|
||
"selectionObjective": objective,
|
||
}
|
||
return (
|
||
fitted_vectorizer,
|
||
models,
|
||
thresholds,
|
||
report,
|
||
validation_probabilities,
|
||
)
|
||
|
||
|
||
def batched_evaluation(
|
||
records: list[dict[str, Any]],
|
||
fitted_vectorizer: FeatureUnion | None,
|
||
models: dict[str, SGDClassifier] | None,
|
||
thresholds: dict[str, dict[str, Any]],
|
||
intent_bundles: dict[
|
||
str, tuple[FeatureUnion, SGDClassifier]
|
||
] | None = None,
|
||
batch_size: int = 2_000,
|
||
) -> dict[str, Any]:
|
||
counts = {intent: BinaryCounts() for intent in INTENTS}
|
||
by_language: dict[str, dict[str, BinaryCounts]] = defaultdict(
|
||
lambda: {intent: BinaryCounts() for intent in INTENTS}
|
||
)
|
||
by_source: dict[str, dict[str, BinaryCounts]] = defaultdict(
|
||
lambda: {intent: BinaryCounts() for intent in INTENTS}
|
||
)
|
||
for offset in range(0, len(records), batch_size):
|
||
batch = records[offset : offset + batch_size]
|
||
texts = [record["text"] for record in batch]
|
||
if intent_bundles is not None:
|
||
probabilities = {
|
||
intent: model.predict_proba(
|
||
intent_vectorizer.transform(texts)
|
||
)[:, 1]
|
||
for intent, (intent_vectorizer, model) in intent_bundles.items()
|
||
}
|
||
else:
|
||
assert fitted_vectorizer is not None and models is not None
|
||
matrix = fitted_vectorizer.transform(texts)
|
||
probabilities = prediction_probabilities(matrix, models)
|
||
predictions = runtime_predictions(batch, probabilities, thresholds)
|
||
for intent in INTENTS:
|
||
known_mask = np.array(
|
||
[is_known(record, intent) for record in batch],
|
||
dtype=bool,
|
||
)
|
||
expected = np.array(
|
||
[record_label(record, intent) for record in batch], dtype=bool
|
||
)
|
||
counts[intent].update(
|
||
expected[known_mask],
|
||
predictions[intent][known_mask],
|
||
)
|
||
for language in {record["language"] for record in batch}:
|
||
mask = np.array(
|
||
[record["language"] == language for record in batch]
|
||
) & known_mask
|
||
by_language[language][intent].update(
|
||
expected[mask], predictions[intent][mask]
|
||
)
|
||
for source in {
|
||
record.get("sourceDataset") or record.get("family", "unknown")
|
||
for record in batch
|
||
}:
|
||
mask = np.array(
|
||
[
|
||
(
|
||
record.get("sourceDataset")
|
||
or record.get("family", "unknown")
|
||
)
|
||
== source
|
||
for record in batch
|
||
]
|
||
) & known_mask
|
||
by_source[source][intent].update(
|
||
expected[mask], predictions[intent][mask]
|
||
)
|
||
per_intent = {
|
||
intent: value.metrics() for intent, value in counts.items()
|
||
}
|
||
return {
|
||
**aggregate_metrics(per_intent),
|
||
"byLanguage": {
|
||
language: aggregate_metrics(
|
||
{
|
||
intent: value.metrics()
|
||
for intent, value in intent_counts.items()
|
||
}
|
||
)
|
||
for language, intent_counts in sorted(by_language.items())
|
||
},
|
||
"bySource": {
|
||
source: aggregate_metrics(
|
||
{
|
||
intent: value.metrics()
|
||
for intent, value in intent_counts.items()
|
||
}
|
||
)
|
||
for source, intent_counts in sorted(by_source.items())
|
||
},
|
||
}
|
||
|
||
|
||
def parse_arguments() -> argparse.Namespace:
|
||
parser = argparse.ArgumentParser()
|
||
parser.add_argument("--rounds", type=int, default=20)
|
||
parser.add_argument("--base-corpus", type=Path, default=BASE_CORPUS)
|
||
parser.add_argument("--open-corpus", type=Path, default=OPEN_CORPUS)
|
||
parser.add_argument("--output-directory", type=Path, default=OUTPUT_DIRECTORY)
|
||
parser.add_argument("--skip-comprehensive", action="store_true")
|
||
return parser.parse_args()
|
||
|
||
|
||
def write_json(path: Path, value: Any) -> None:
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
path.write_text(
|
||
json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||
encoding="utf-8",
|
||
)
|
||
|
||
|
||
def main() -> None:
|
||
arguments = parse_arguments()
|
||
all_configurations = configurations()
|
||
if arguments.rounds != 20:
|
||
all_configurations = all_configurations[: arguments.rounds]
|
||
if len(all_configurations) != arguments.rounds:
|
||
raise ValueError("The research matrix defines at most 20 rounds")
|
||
|
||
base_records = read_json_lines(arguments.base_corpus)
|
||
open_records = read_json_lines(arguments.open_corpus)
|
||
if not base_records:
|
||
raise ValueError("Base corpus is empty")
|
||
splits = {
|
||
split: [
|
||
record for record in base_records if record.get("split") == split
|
||
]
|
||
for split in ("train", "validation", "test", "golden")
|
||
}
|
||
if any(not values for values in splits.values()):
|
||
raise ValueError("Base train/validation/test/golden splits are required")
|
||
|
||
training_fingerprints = {
|
||
" ".join(record["text"].casefold().split())
|
||
for record in splits["train"] + open_records
|
||
}
|
||
frozen_paths = [RANDOM_HOLDOUT, TARGETED_HOLDOUT, COMPREHENSIVE_HOLDOUT]
|
||
overlap_counts = {}
|
||
for path in frozen_paths:
|
||
records = read_json_lines(path)
|
||
overlap_counts[str(path)] = sum(
|
||
" ".join(record["text"].casefold().split())
|
||
in training_fingerprints
|
||
for record in records
|
||
)
|
||
if any(overlap_counts.values()):
|
||
raise ValueError(f"Training/holdout overlap: {overlap_counts}")
|
||
|
||
unknown_matrix_records = open_records
|
||
committee_history: dict[
|
||
str, deque[tuple[np.ndarray, np.ndarray]]
|
||
] = {
|
||
intent: deque(maxlen=3) for intent in INTENTS
|
||
}
|
||
pseudo: dict[str, list[tuple[int, bool]]] = {}
|
||
round_reports = []
|
||
best: tuple[
|
||
float,
|
||
FeatureUnion,
|
||
dict[str, SGDClassifier],
|
||
dict[str, dict[str, Any]],
|
||
dict[str, Any],
|
||
] | None = None
|
||
best_per_intent: dict[
|
||
str,
|
||
tuple[
|
||
tuple[bool, float, float],
|
||
FeatureUnion,
|
||
SGDClassifier,
|
||
dict[str, Any],
|
||
RoundConfiguration,
|
||
],
|
||
] = {}
|
||
|
||
started_at = time.monotonic()
|
||
for configuration in all_configurations:
|
||
(
|
||
fitted_vectorizer,
|
||
models,
|
||
thresholds,
|
||
report,
|
||
_,
|
||
) = train_round(
|
||
configuration,
|
||
splits["train"],
|
||
open_records,
|
||
splits["validation"],
|
||
splits["test"],
|
||
pseudo,
|
||
)
|
||
round_reports.append(report)
|
||
objective = report["selectionObjective"]
|
||
if best is None or objective > best[0]:
|
||
best = (
|
||
objective,
|
||
fitted_vectorizer,
|
||
models,
|
||
thresholds,
|
||
report,
|
||
)
|
||
for intent in INTENTS:
|
||
metrics = report["validation"]["perIntent"][intent]
|
||
intent_score = (
|
||
metrics["precision"] >= 0.95
|
||
and metrics["predictedPositive"] >= 10,
|
||
metrics["f1"],
|
||
metrics["recall"],
|
||
)
|
||
current = best_per_intent.get(intent)
|
||
if current is None or intent_score > current[0]:
|
||
best_per_intent[intent] = (
|
||
intent_score,
|
||
fitted_vectorizer,
|
||
models[intent],
|
||
thresholds[intent],
|
||
configuration,
|
||
)
|
||
|
||
if unknown_matrix_records:
|
||
unknown_matrix = fitted_vectorizer.transform(
|
||
[record["text"] for record in unknown_matrix_records]
|
||
)
|
||
unknown_probabilities = prediction_probabilities(
|
||
unknown_matrix, models
|
||
)
|
||
for intent in INTENTS:
|
||
per_record_thresholds = thresholds_for_records(
|
||
unknown_matrix_records, intent, thresholds
|
||
)
|
||
committee_history[intent].append(
|
||
(unknown_probabilities[intent], per_record_thresholds)
|
||
)
|
||
pseudo = pseudo_labels_from_committee(
|
||
unknown_matrix_records,
|
||
committee_history,
|
||
configuration.round + 1,
|
||
)
|
||
print(
|
||
"ITERATIVE_ROUND "
|
||
f"round={configuration.round} "
|
||
f"objective={objective:.4f} "
|
||
f"validationPrecision={report['validation']['macroPrecision']:.4f} "
|
||
f"validationF1={report['validation']['macroF1']:.4f} "
|
||
f"testF1={report['syntheticRegression']['macroF1']:.4f}"
|
||
)
|
||
|
||
assert best is not None
|
||
_, fitted_vectorizer, models, thresholds, selected_report = best
|
||
composite_bundles = {
|
||
intent: (values[1], values[2])
|
||
for intent, values in best_per_intent.items()
|
||
}
|
||
composite_thresholds = {
|
||
intent: values[3] for intent, values in best_per_intent.items()
|
||
}
|
||
selected_rounds_by_intent = {
|
||
intent: values[4].round for intent, values in best_per_intent.items()
|
||
}
|
||
final_sets = {
|
||
"golden": splits["golden"],
|
||
"randomHoldout": read_json_lines(RANDOM_HOLDOUT),
|
||
"targetedRelease": read_json_lines(TARGETED_HOLDOUT),
|
||
}
|
||
if not arguments.skip_comprehensive:
|
||
final_sets["researchOnlyComprehensive"] = read_json_lines(
|
||
COMPREHENSIVE_HOLDOUT
|
||
)
|
||
final_evaluations = {
|
||
name: batched_evaluation(
|
||
records,
|
||
None,
|
||
None,
|
||
composite_thresholds,
|
||
intent_bundles=composite_bundles,
|
||
)
|
||
for name, records in final_sets.items()
|
||
if records
|
||
}
|
||
|
||
release_sets = [
|
||
final_evaluations.get("golden"),
|
||
final_evaluations.get("randomHoldout"),
|
||
final_evaluations.get("targetedRelease"),
|
||
]
|
||
release_gate = bool(release_sets) and all(
|
||
evaluation
|
||
and evaluation["macroPrecision"] >= 0.95
|
||
and evaluation["intentsMeeting95Precision"] == len(INTENTS)
|
||
for evaluation in release_sets
|
||
)
|
||
report = {
|
||
"schemaVersion": 1,
|
||
"generatedAtUnix": int(time.time()),
|
||
"seed": SEED,
|
||
"roundCount": len(round_reports),
|
||
"trainingPolicy": (
|
||
"No user clipboard data. Official partial labels plus deterministic "
|
||
"label-preserving augmentation and three-round high-confidence "
|
||
"self-training consensus. Frozen holdouts never enter training or "
|
||
"threshold calibration."
|
||
),
|
||
"platformLimitation": (
|
||
"Linux surrogate research only. Apple Create ML/Core ML artifacts "
|
||
"must be retrained and tested on macOS before deployment."
|
||
),
|
||
"baseTrainingRecords": len(splits["train"]),
|
||
"openTrainingRecords": len(open_records),
|
||
"overlapChecks": overlap_counts,
|
||
"elapsedSeconds": rounded(time.monotonic() - started_at),
|
||
"selectedRound": selected_report["round"],
|
||
"selectedRoundsByIntent": selected_rounds_by_intent,
|
||
"selectedConfiguration": selected_report["configuration"],
|
||
"selectedThresholds": composite_thresholds,
|
||
"selectionObjective": selected_report["selectionObjective"],
|
||
"finalEvaluations": final_evaluations,
|
||
"releaseGatePassed": release_gate,
|
||
"deploymentDecision": (
|
||
"eligible-for-macos-replay"
|
||
if release_gate
|
||
else "retain-current-deployed-models"
|
||
),
|
||
}
|
||
replay = {
|
||
"schemaVersion": 1,
|
||
"selectedRound": selected_report["round"],
|
||
"selectedRoundsByIntent": selected_rounds_by_intent,
|
||
"seed": SEED,
|
||
"baseCorpus": str(arguments.base_corpus),
|
||
"openCorpus": str(arguments.open_corpus),
|
||
"configuration": selected_report["configuration"],
|
||
"configurationsByIntent": {
|
||
intent: asdict(values[4])
|
||
for intent, values in best_per_intent.items()
|
||
},
|
||
"thresholds": composite_thresholds,
|
||
"requiredCommands": [
|
||
"python3 Scripts/clipboard_semantics/generate_open_training_corpus.py",
|
||
(
|
||
"xcrun swift Scripts/clipboard_semantics/train_models.swift "
|
||
"--algorithms maxEnt "
|
||
"--corpus ModelTraining/ClipboardSemantics/combined-training-corpus.jsonl"
|
||
),
|
||
(
|
||
"xcrun swift Scripts/clipboard_semantics/evaluate_random_holdout.swift "
|
||
"--corpus ModelTraining/ClipboardSemantics/random-holdout-corpus.jsonl"
|
||
),
|
||
],
|
||
"automaticPromotionAllowed": False,
|
||
"reason": (
|
||
"Surrogate feature weights are not deployable NLModel assets and "
|
||
"cannot bypass the existing macOS acceptance policy."
|
||
),
|
||
}
|
||
write_json(arguments.output_directory / "rounds.json", round_reports)
|
||
write_json(arguments.output_directory / "final-report.json", report)
|
||
write_json(arguments.output_directory / "macos-replay.json", replay)
|
||
print(
|
||
"ITERATIVE_RETRAINING_DONE "
|
||
f"rounds={len(round_reports)} selected={selected_report['round']} "
|
||
f"releaseGate={str(release_gate).lower()} "
|
||
f"elapsed={report['elapsedSeconds']}"
|
||
)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|