chore(semantics): add v6 release gate pipeline
- Add reproducible v6 boundary, blessing, and consensus-adjudication corpora, plus the tiny-transformer trainer and v6 release-gate evaluator that gate every candidate on the deployed baselines. - Wire consensus-label merging, product-policy anchor evaluation, and sealed blessing benchmark review with their pytest coverage. - Refresh open-training corpus generation, iterative retraining runner, and random-holdout evaluation so v6 candidates can be benchmarked end-to-end.
This commit is contained in:
Rocky
@@ -0,0 +1,563 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Prepare and merge evidence-backed AI adjudication of consensus conflicts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import unicodedata
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
from merge_consensus_labels_v2 import DOMAINS, INTENT_LABELS, stable_split
|
||||
|
||||
FLAG_FIELDS = {"ambiguous", "quotedOrMeta"}
|
||||
LABEL_STATES = {"true", "false", "unknown"}
|
||||
SENTIMENT_STATES = {"positive", "neutral", "negative", "unknown"}
|
||||
RECORD_DISPOSITIONS = {"keep", "exclude-device-command"}
|
||||
PRODUCT_POLICY_FIELDS = (
|
||||
"task",
|
||||
"question",
|
||||
"invitation",
|
||||
"complaint",
|
||||
"followUpReminder",
|
||||
"blessing",
|
||||
"replyableMessage",
|
||||
"assistantCommand",
|
||||
"informationQuery",
|
||||
"systemNotification",
|
||||
"domain",
|
||||
)
|
||||
PROMPT_VERSION = "clipboard-adjudication-v5"
|
||||
|
||||
|
||||
def normalize(value: str) -> str:
|
||||
return " ".join(unicodedata.normalize("NFKC", value).casefold().split())
|
||||
|
||||
|
||||
def read_json_lines(path: Path) -> list[dict]:
|
||||
return [
|
||||
json.loads(line)
|
||||
for line in path.read_text(encoding="utf-8").splitlines()
|
||||
if line.strip()
|
||||
]
|
||||
|
||||
|
||||
def write_json_lines(path: Path, records: list[dict]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", encoding="utf-8") as handle:
|
||||
for record in records:
|
||||
handle.write(
|
||||
json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n"
|
||||
)
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def prepare(arguments: argparse.Namespace) -> dict:
|
||||
conflicts = read_json_lines(arguments.conflicts)
|
||||
include_policy_fields = getattr(
|
||||
arguments,
|
||||
"include_product_policy_fields",
|
||||
False,
|
||||
)
|
||||
queue = [
|
||||
{
|
||||
"id": record["id"],
|
||||
"text": record["text"],
|
||||
"language": record["language"],
|
||||
"unresolvedFields": list(
|
||||
dict.fromkeys(
|
||||
(
|
||||
*PRODUCT_POLICY_FIELDS,
|
||||
*record["unresolvedFields"],
|
||||
)
|
||||
if include_policy_fields
|
||||
else record["unresolvedFields"]
|
||||
)
|
||||
),
|
||||
}
|
||||
for record in conflicts
|
||||
]
|
||||
if len({record["id"] for record in queue}) != len(queue):
|
||||
raise ValueError("Conflict queue contains duplicate ids")
|
||||
write_json_lines(arguments.queue, queue)
|
||||
arguments.chunk_directory.mkdir(parents=True, exist_ok=True)
|
||||
chunks = []
|
||||
for start in range(0, len(queue), arguments.chunk_size):
|
||||
index = len(chunks) + 1
|
||||
path = arguments.chunk_directory / f"chunk-{index:03d}.jsonl"
|
||||
values = queue[start : start + arguments.chunk_size]
|
||||
write_json_lines(path, values)
|
||||
chunks.append(
|
||||
{
|
||||
"path": str(path),
|
||||
"records": len(values),
|
||||
"sha256": sha256_file(path),
|
||||
}
|
||||
)
|
||||
report = {
|
||||
"schemaVersion": 1,
|
||||
"promptVersion": PROMPT_VERSION,
|
||||
"queueCount": len(queue),
|
||||
"queueSHA256": sha256_file(arguments.queue),
|
||||
"chunkSize": arguments.chunk_size,
|
||||
"chunkCount": len(chunks),
|
||||
"includesProductPolicyFields": include_policy_fields,
|
||||
"productPolicyFields": (
|
||||
list(PRODUCT_POLICY_FIELDS) if include_policy_fields else []
|
||||
),
|
||||
"chunks": chunks,
|
||||
}
|
||||
arguments.report.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return report
|
||||
|
||||
|
||||
def valid_state(field: str, value: object) -> bool:
|
||||
if field == "sentiment":
|
||||
return value in SENTIMENT_STATES
|
||||
if field == "domain":
|
||||
return value in {*DOMAINS, "unknown"}
|
||||
return value in LABEL_STATES
|
||||
|
||||
|
||||
def validate_adjudication(record: dict, queue_record: dict) -> dict:
|
||||
identifier = queue_record["id"]
|
||||
if record.get("id") != identifier:
|
||||
raise ValueError(f"Unexpected adjudication id: {record.get('id')}")
|
||||
disposition = record.get("recordDisposition")
|
||||
if disposition not in RECORD_DISPOSITIONS:
|
||||
raise ValueError(f"Invalid record disposition: {identifier}")
|
||||
disposition_confidence = record.get("dispositionConfidence")
|
||||
if (
|
||||
not isinstance(disposition_confidence, (int, float))
|
||||
or not 0 <= disposition_confidence <= 1
|
||||
):
|
||||
raise ValueError(f"Invalid disposition confidence: {identifier}")
|
||||
disposition_evidence = record.get("dispositionEvidence")
|
||||
text = normalize(queue_record["text"])
|
||||
if (
|
||||
not isinstance(disposition_evidence, str)
|
||||
or not normalize(disposition_evidence)
|
||||
or normalize(disposition_evidence) not in text
|
||||
):
|
||||
raise ValueError(f"Invalid disposition evidence: {identifier}")
|
||||
expected = set(queue_record["unresolvedFields"])
|
||||
for key in ("resolutions", "confidence", "evidence"):
|
||||
if not isinstance(record.get(key), dict) or set(record[key]) != expected:
|
||||
raise ValueError(f"{key} fields do not match unresolved fields: {identifier}")
|
||||
for field in expected:
|
||||
if not valid_state(field, record["resolutions"][field]):
|
||||
raise ValueError(f"Invalid resolution for {identifier}/{field}")
|
||||
confidence = record["confidence"][field]
|
||||
if not isinstance(confidence, (int, float)) or not 0 <= confidence <= 1:
|
||||
raise ValueError(f"Invalid confidence for {identifier}/{field}")
|
||||
evidence = record["evidence"][field]
|
||||
if not isinstance(evidence, str) or not normalize(evidence):
|
||||
raise ValueError(f"Missing evidence for {identifier}/{field}")
|
||||
if normalize(evidence) not in text:
|
||||
raise ValueError(f"Evidence is not an exact text quote: {identifier}/{field}")
|
||||
return {
|
||||
"id": identifier,
|
||||
"recordDisposition": disposition,
|
||||
"dispositionConfidence": round(float(disposition_confidence), 4),
|
||||
"dispositionEvidence": disposition_evidence,
|
||||
"resolutions": {
|
||||
field: record["resolutions"][field] for field in sorted(expected)
|
||||
},
|
||||
"confidence": {
|
||||
field: round(float(record["confidence"][field]), 4)
|
||||
for field in sorted(expected)
|
||||
},
|
||||
"evidence": {
|
||||
field: record["evidence"][field] for field in sorted(expected)
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def load_adjudicator(
|
||||
paths: list[Path],
|
||||
queue_by_id: dict[str, dict],
|
||||
) -> dict[str, dict]:
|
||||
values = []
|
||||
for path in paths:
|
||||
values.extend(read_json_lines(path))
|
||||
by_id = {}
|
||||
for value in values:
|
||||
identifier = value.get("id")
|
||||
if identifier not in queue_by_id:
|
||||
raise ValueError(f"Unexpected adjudication id: {identifier}")
|
||||
if identifier in by_id:
|
||||
raise ValueError(f"Duplicate adjudication id: {identifier}")
|
||||
by_id[identifier] = validate_adjudication(
|
||||
value,
|
||||
queue_by_id[identifier],
|
||||
)
|
||||
if set(by_id) != set(queue_by_id):
|
||||
raise ValueError("Adjudicator outputs do not cover the complete queue")
|
||||
return by_id
|
||||
|
||||
|
||||
def resolved_base_field(conflict: dict, field: str) -> str:
|
||||
votes = conflict.get("modelVotes", {}).get(field, {})
|
||||
if not votes:
|
||||
return "unknown"
|
||||
value, count = max(votes.items(), key=lambda item: item[1])
|
||||
return value if count >= 4 else "unknown"
|
||||
|
||||
|
||||
def tier_c_record(conflict: dict, resolutions: dict, evidence: dict) -> dict:
|
||||
states = {}
|
||||
for field in (*INTENT_LABELS, "sentiment", "domain"):
|
||||
states[field] = resolutions.get(
|
||||
field,
|
||||
resolved_base_field(conflict, field),
|
||||
)
|
||||
known_labels = [
|
||||
field
|
||||
for field in (*INTENT_LABELS, "sentiment", "domain")
|
||||
if states[field] != "unknown"
|
||||
]
|
||||
return {
|
||||
"id": f"adjudicated-v5-{conflict['id']}",
|
||||
"sourceRecordID": conflict["id"],
|
||||
"text": conflict["text"],
|
||||
"language": conflict["language"],
|
||||
"family": "ai_adjudicated_v5",
|
||||
"split": stable_split(conflict["id"]),
|
||||
**{
|
||||
("replyable" if label == "replyableMessage" else label): (
|
||||
states[label] == "true"
|
||||
)
|
||||
for label in INTENT_LABELS
|
||||
},
|
||||
"sentiment": (
|
||||
states["sentiment"]
|
||||
if states["sentiment"] != "unknown"
|
||||
else "neutral"
|
||||
),
|
||||
"domain": (
|
||||
states["domain"] if states["domain"] != "unknown" else None
|
||||
),
|
||||
"knownLabels": known_labels,
|
||||
"labelQualityTier": "C",
|
||||
"sampleWeight": 0.35,
|
||||
"promptVersion": PROMPT_VERSION,
|
||||
"adjudicationEvidence": evidence,
|
||||
}
|
||||
|
||||
|
||||
def merge(arguments: argparse.Namespace) -> dict:
|
||||
conflicts = read_json_lines(arguments.conflicts)
|
||||
conflicts_by_id = {record["id"]: record for record in conflicts}
|
||||
queue = read_json_lines(arguments.queue)
|
||||
queue_by_id = {record["id"]: record for record in queue}
|
||||
if set(conflicts_by_id) != set(queue_by_id):
|
||||
raise ValueError("Conflict and adjudication queue ids differ")
|
||||
adjudicator_a = load_adjudicator(arguments.adjudicator_a, queue_by_id)
|
||||
adjudicator_b = load_adjudicator(arguments.adjudicator_b, queue_by_id)
|
||||
accepted = []
|
||||
excluded = []
|
||||
remaining = []
|
||||
rejection_reasons = Counter()
|
||||
for identifier in sorted(queue_by_id):
|
||||
conflict = conflicts_by_id[identifier]
|
||||
first = adjudicator_a[identifier]
|
||||
second = adjudicator_b[identifier]
|
||||
resolutions = {}
|
||||
evidence = {}
|
||||
rejected_fields = {}
|
||||
first_disposition = first["recordDisposition"]
|
||||
second_disposition = second["recordDisposition"]
|
||||
disposition_reasons = []
|
||||
if first_disposition != second_disposition:
|
||||
disposition_reasons.append("adjudicator-disagreement")
|
||||
if min(
|
||||
first["dispositionConfidence"],
|
||||
second["dispositionConfidence"],
|
||||
) < arguments.minimum_confidence:
|
||||
disposition_reasons.append("low-confidence")
|
||||
if disposition_reasons:
|
||||
rejected_fields["recordDisposition"] = sorted(
|
||||
set(disposition_reasons)
|
||||
)
|
||||
rejection_reasons.update(set(disposition_reasons))
|
||||
elif first_disposition == "exclude-device-command":
|
||||
excluded.append(
|
||||
{
|
||||
"id": identifier,
|
||||
"text": conflict["text"],
|
||||
"language": conflict["language"],
|
||||
"disposition": first_disposition,
|
||||
"promptVersion": PROMPT_VERSION,
|
||||
"evidence": {
|
||||
arguments.adjudicator_a_name: first[
|
||||
"dispositionEvidence"
|
||||
],
|
||||
arguments.adjudicator_b_name: second[
|
||||
"dispositionEvidence"
|
||||
],
|
||||
},
|
||||
}
|
||||
)
|
||||
continue
|
||||
for field in queue_by_id[identifier]["unresolvedFields"]:
|
||||
first_value = first["resolutions"][field]
|
||||
second_value = second["resolutions"][field]
|
||||
reasons = []
|
||||
if first_value != second_value:
|
||||
reasons.append("adjudicator-disagreement")
|
||||
if "unknown" in {first_value, second_value}:
|
||||
reasons.append("unknown")
|
||||
if min(
|
||||
first["confidence"][field],
|
||||
second["confidence"][field],
|
||||
) < arguments.minimum_confidence:
|
||||
reasons.append("low-confidence")
|
||||
if field == "ambiguous" and first_value == "true":
|
||||
reasons.append("materially-ambiguous")
|
||||
if reasons:
|
||||
rejected_fields[field] = sorted(set(reasons))
|
||||
rejection_reasons.update(set(reasons))
|
||||
continue
|
||||
resolutions[field] = first_value
|
||||
evidence[field] = {
|
||||
arguments.adjudicator_a_name: first["evidence"][field],
|
||||
arguments.adjudicator_b_name: second["evidence"][field],
|
||||
}
|
||||
if rejected_fields:
|
||||
remaining.append(
|
||||
{
|
||||
**conflict,
|
||||
"aiAdjudication": {
|
||||
"rejectedFields": rejected_fields,
|
||||
"adjudicatorA": first,
|
||||
"adjudicatorB": second,
|
||||
},
|
||||
}
|
||||
)
|
||||
continue
|
||||
accepted.append(
|
||||
tier_c_record(conflict, resolutions, evidence)
|
||||
)
|
||||
write_json_lines(arguments.accepted, accepted)
|
||||
write_json_lines(arguments.excluded, excluded)
|
||||
write_json_lines(arguments.remaining, remaining)
|
||||
report = {
|
||||
"schemaVersion": 1,
|
||||
"promptVersion": PROMPT_VERSION,
|
||||
"queueCount": len(queue),
|
||||
"queueSHA256": sha256_file(arguments.queue),
|
||||
"minimumConfidence": arguments.minimum_confidence,
|
||||
"acceptedTierCCount": len(accepted),
|
||||
"excludedDeviceCommandCount": len(excluded),
|
||||
"remainingHumanReviewCount": len(remaining),
|
||||
"resolvedCount": len(accepted) + len(excluded),
|
||||
"resolutionRate": round(
|
||||
(len(accepted) + len(excluded)) / max(len(queue), 1),
|
||||
4,
|
||||
),
|
||||
"rejectionReasonCounts": dict(sorted(rejection_reasons.items())),
|
||||
"adjudicators": [
|
||||
arguments.adjudicator_a_name,
|
||||
arguments.adjudicator_b_name,
|
||||
],
|
||||
"acceptedLanguageCounts": dict(
|
||||
sorted(Counter(record["language"] for record in accepted).items())
|
||||
),
|
||||
"remainingLanguageCounts": dict(
|
||||
sorted(Counter(record["language"] for record in remaining).items())
|
||||
),
|
||||
}
|
||||
arguments.report.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return report
|
||||
|
||||
|
||||
def review_priority(record: dict) -> tuple:
|
||||
reasons = {
|
||||
reason
|
||||
for field_reasons in record["aiAdjudication"]["rejectedFields"].values()
|
||||
for reason in field_reasons
|
||||
}
|
||||
severity = (
|
||||
0 if "adjudicator-disagreement" in reasons else 1,
|
||||
0 if "unknown" in reasons else 1,
|
||||
0 if "materially-ambiguous" in reasons else 1,
|
||||
)
|
||||
return (*severity, record["id"])
|
||||
|
||||
|
||||
def adjudicator_field_review(adjudication: dict, field: str) -> dict:
|
||||
if field == "recordDisposition":
|
||||
return {
|
||||
"value": adjudication["recordDisposition"],
|
||||
"confidence": adjudication["dispositionConfidence"],
|
||||
"evidence": adjudication["dispositionEvidence"],
|
||||
}
|
||||
return {
|
||||
"value": adjudication["resolutions"][field],
|
||||
"confidence": adjudication["confidence"][field],
|
||||
"evidence": adjudication["evidence"][field],
|
||||
}
|
||||
|
||||
|
||||
def review_sample(arguments: argparse.Namespace) -> dict:
|
||||
records = read_json_lines(arguments.remaining)
|
||||
grouped: dict[tuple[str, str], list[dict]] = {}
|
||||
for record in records:
|
||||
for field in record["aiAdjudication"]["rejectedFields"]:
|
||||
grouped.setdefault((record["language"], field), []).append(record)
|
||||
for values in grouped.values():
|
||||
values.sort(key=review_priority)
|
||||
|
||||
selected = []
|
||||
selected_ids = set()
|
||||
offsets = {key: 0 for key in grouped}
|
||||
keys = sorted(grouped)
|
||||
while len(selected) < min(arguments.sample_size, len(records)):
|
||||
added = False
|
||||
for key in keys:
|
||||
values = grouped[key]
|
||||
while (
|
||||
offsets[key] < len(values)
|
||||
and values[offsets[key]]["id"] in selected_ids
|
||||
):
|
||||
offsets[key] += 1
|
||||
if offsets[key] >= len(values):
|
||||
continue
|
||||
record = values[offsets[key]]
|
||||
offsets[key] += 1
|
||||
selected.append(record)
|
||||
selected_ids.add(record["id"])
|
||||
added = True
|
||||
if len(selected) >= arguments.sample_size:
|
||||
break
|
||||
if not added:
|
||||
break
|
||||
|
||||
output = []
|
||||
for record in selected:
|
||||
first = record["aiAdjudication"]["adjudicatorA"]
|
||||
second = record["aiAdjudication"]["adjudicatorB"]
|
||||
fields = record["aiAdjudication"]["rejectedFields"]
|
||||
output.append(
|
||||
{
|
||||
"id": record["id"],
|
||||
"text": record["text"],
|
||||
"language": record["language"],
|
||||
"fieldReviews": {
|
||||
field: {
|
||||
"rejectionReasons": fields[field],
|
||||
"adjudicatorA": adjudicator_field_review(first, field),
|
||||
"adjudicatorB": adjudicator_field_review(second, field),
|
||||
}
|
||||
for field in sorted(fields)
|
||||
},
|
||||
"humanDecision": {field: None for field in sorted(fields)},
|
||||
"notes": "",
|
||||
}
|
||||
)
|
||||
write_json_lines(arguments.sample, output)
|
||||
report = {
|
||||
"schemaVersion": 1,
|
||||
"promptVersion": PROMPT_VERSION,
|
||||
"remainingCount": len(records),
|
||||
"sampleCount": len(output),
|
||||
"sampleSHA256": sha256_file(arguments.sample),
|
||||
"languageCounts": dict(
|
||||
sorted(Counter(record["language"] for record in output).items())
|
||||
),
|
||||
"fieldCounts": dict(
|
||||
sorted(
|
||||
Counter(
|
||||
field
|
||||
for record in output
|
||||
for field in record["fieldReviews"]
|
||||
).items()
|
||||
)
|
||||
),
|
||||
}
|
||||
arguments.report.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return report
|
||||
|
||||
|
||||
def parser() -> argparse.ArgumentParser:
|
||||
root = argparse.ArgumentParser()
|
||||
commands = root.add_subparsers(dest="command", required=True)
|
||||
|
||||
prepare_parser = commands.add_parser("prepare")
|
||||
prepare_parser.add_argument("--conflicts", type=Path, required=True)
|
||||
prepare_parser.add_argument("--queue", type=Path, required=True)
|
||||
prepare_parser.add_argument("--chunk-directory", type=Path, required=True)
|
||||
prepare_parser.add_argument("--chunk-size", type=int, default=80)
|
||||
prepare_parser.add_argument(
|
||||
"--include-product-policy-fields",
|
||||
action="store_true",
|
||||
)
|
||||
prepare_parser.add_argument("--report", type=Path, required=True)
|
||||
prepare_parser.set_defaults(handler=prepare)
|
||||
|
||||
merge_parser = commands.add_parser("merge")
|
||||
merge_parser.add_argument("--conflicts", type=Path, required=True)
|
||||
merge_parser.add_argument("--queue", type=Path, required=True)
|
||||
merge_parser.add_argument(
|
||||
"--adjudicator-a",
|
||||
action="append",
|
||||
type=Path,
|
||||
required=True,
|
||||
)
|
||||
merge_parser.add_argument(
|
||||
"--adjudicator-b",
|
||||
action="append",
|
||||
type=Path,
|
||||
required=True,
|
||||
)
|
||||
merge_parser.add_argument("--adjudicator-a-name", required=True)
|
||||
merge_parser.add_argument("--adjudicator-b-name", required=True)
|
||||
merge_parser.add_argument("--minimum-confidence", type=float, default=0.9)
|
||||
merge_parser.add_argument("--accepted", type=Path, required=True)
|
||||
merge_parser.add_argument("--excluded", type=Path, required=True)
|
||||
merge_parser.add_argument("--remaining", type=Path, required=True)
|
||||
merge_parser.add_argument("--report", type=Path, required=True)
|
||||
merge_parser.set_defaults(handler=merge)
|
||||
|
||||
sample_parser = commands.add_parser("sample-review")
|
||||
sample_parser.add_argument("--remaining", type=Path, required=True)
|
||||
sample_parser.add_argument("--sample", type=Path, required=True)
|
||||
sample_parser.add_argument("--sample-size", type=int, default=60)
|
||||
sample_parser.add_argument("--report", type=Path, required=True)
|
||||
sample_parser.set_defaults(handler=review_sample)
|
||||
return root
|
||||
|
||||
|
||||
def main() -> None:
|
||||
arguments = parser().parse_args()
|
||||
report = arguments.handler(arguments)
|
||||
print(
|
||||
f"AI_ADJUDICATION_{arguments.command.upper()} "
|
||||
+ " ".join(
|
||||
f"{key}={value}"
|
||||
for key, value in report.items()
|
||||
if key.endswith("Count")
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,130 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Assemble registry-approved training data with a frozen v6 evaluation set."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import unicodedata
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
DEFAULT_TRAIN = Path(
|
||||
"ModelTraining/ClipboardSemantics/CorpusRegistry/train-candidates.jsonl"
|
||||
)
|
||||
DEFAULT_EVALUATION = Path(
|
||||
"ModelTraining/ClipboardSemantics/v6-blind-evaluation-corpus.jsonl"
|
||||
)
|
||||
DEFAULT_OUTPUT = Path(
|
||||
"ModelTraining/ClipboardSemantics/Generated/v6-model-corpus.jsonl"
|
||||
)
|
||||
DEFAULT_REPORT = Path(
|
||||
"ModelTraining/ClipboardSemantics/Generated/v6-model-corpus-report.json"
|
||||
)
|
||||
EVALUATION_SPLITS = {"validation", "test", "golden"}
|
||||
|
||||
|
||||
def read_json_lines(path: Path) -> list[dict]:
|
||||
return [
|
||||
json.loads(line)
|
||||
for line in path.read_text(encoding="utf-8").splitlines()
|
||||
if line.strip()
|
||||
]
|
||||
|
||||
|
||||
def fingerprint(text: str) -> str:
|
||||
normalized = unicodedata.normalize("NFKC", text)
|
||||
return " ".join(normalized.casefold().split())
|
||||
|
||||
|
||||
def assemble(train_path: Path, evaluation_path: Path) -> tuple[list[dict], dict]:
|
||||
training = read_json_lines(train_path)
|
||||
evaluation = read_json_lines(evaluation_path)
|
||||
if any(record.get("split") != "train" for record in training):
|
||||
raise ValueError("Registry train candidates must all use split=train")
|
||||
if any(record.get("split") not in EVALUATION_SPLITS for record in evaluation):
|
||||
raise ValueError("Evaluation records must use validation, test, or golden")
|
||||
|
||||
training_fingerprints = {fingerprint(record["text"]) for record in training}
|
||||
overlap = [
|
||||
record["id"]
|
||||
for record in evaluation
|
||||
if fingerprint(record["text"]) in training_fingerprints
|
||||
]
|
||||
if overlap:
|
||||
raise ValueError(f"Frozen evaluation overlap detected: {overlap[:5]}")
|
||||
|
||||
identifiers = [record["id"] for record in (*training, *evaluation)]
|
||||
if len(identifiers) != len(set(identifiers)):
|
||||
raise ValueError("Duplicate record ids in assembled corpus")
|
||||
|
||||
records = training + evaluation
|
||||
report = {
|
||||
"schemaVersion": 1,
|
||||
"trainSource": str(train_path),
|
||||
"evaluationSource": str(evaluation_path),
|
||||
"recordCount": len(records),
|
||||
"splitCounts": dict(
|
||||
sorted(Counter(record["split"] for record in records).items())
|
||||
),
|
||||
"languageCounts": dict(
|
||||
sorted(Counter(record["language"] for record in records).items())
|
||||
),
|
||||
"evaluationOverlapCount": 0,
|
||||
}
|
||||
return records, report
|
||||
|
||||
|
||||
def write_outputs(
|
||||
records: list[dict],
|
||||
report: dict,
|
||||
output_path: Path,
|
||||
report_path: Path,
|
||||
) -> dict:
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
payload = "".join(
|
||||
json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n"
|
||||
for record in records
|
||||
)
|
||||
output_path.write_text(payload, encoding="utf-8")
|
||||
final_report = {
|
||||
**report,
|
||||
"corpusSHA256": hashlib.sha256(payload.encode()).hexdigest(),
|
||||
}
|
||||
report_path.write_text(
|
||||
json.dumps(final_report, ensure_ascii=False, indent=2, sort_keys=True)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return final_report
|
||||
|
||||
|
||||
def parser() -> argparse.ArgumentParser:
|
||||
root = argparse.ArgumentParser()
|
||||
root.add_argument("--train", type=Path, default=DEFAULT_TRAIN)
|
||||
root.add_argument("--evaluation", type=Path, default=DEFAULT_EVALUATION)
|
||||
root.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
|
||||
root.add_argument("--report", type=Path, default=DEFAULT_REPORT)
|
||||
return root
|
||||
|
||||
|
||||
def main() -> None:
|
||||
arguments = parser().parse_args()
|
||||
records, report = assemble(arguments.train, arguments.evaluation)
|
||||
final_report = write_outputs(
|
||||
records,
|
||||
report,
|
||||
arguments.output,
|
||||
arguments.report,
|
||||
)
|
||||
print(
|
||||
"V6_MODEL_CORPUS "
|
||||
f"records={final_report['recordCount']} "
|
||||
f"sha256={final_report['corpusSHA256']}"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,569 @@
|
||||
#!/usr/bin/env xcrun swift
|
||||
|
||||
import CoreML
|
||||
import Darwin
|
||||
import Foundation
|
||||
import NaturalLanguage
|
||||
|
||||
private enum BenchmarkError: LocalizedError {
|
||||
case invalidArguments(String)
|
||||
case invalidManifest(String)
|
||||
case invalidCorpus(String)
|
||||
case missingFile(String)
|
||||
|
||||
var errorDescription: String? {
|
||||
switch self {
|
||||
case .invalidArguments(let message),
|
||||
.invalidManifest(let message),
|
||||
.invalidCorpus(let message),
|
||||
.missingFile(let message):
|
||||
return message
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private struct Arguments {
|
||||
let modelDirectory: URL
|
||||
let corpus: URL
|
||||
let report: URL
|
||||
let maximumSamples: Int
|
||||
let warmRounds: Int
|
||||
|
||||
static let usage = """
|
||||
Usage: benchmark_v6_models.swift \
|
||||
--model-directory <directory> \
|
||||
--corpus <corpus.jsonl> \
|
||||
--report <report.json> \
|
||||
[--max-samples <positive integer>] \
|
||||
[--warm-rounds <positive integer>]
|
||||
"""
|
||||
|
||||
static func parse(_ rawArguments: [String]) throws -> Arguments {
|
||||
var values: [String: String] = [:]
|
||||
var index = 0
|
||||
let supportedFlags = Set([
|
||||
"--model-directory",
|
||||
"--corpus",
|
||||
"--report",
|
||||
"--max-samples",
|
||||
"--warm-rounds"
|
||||
])
|
||||
|
||||
while index < rawArguments.count {
|
||||
let flag = rawArguments[index]
|
||||
guard supportedFlags.contains(flag) else {
|
||||
throw BenchmarkError.invalidArguments("Unknown argument: \(flag)\n\(usage)")
|
||||
}
|
||||
guard index + 1 < rawArguments.count,
|
||||
!rawArguments[index + 1].hasPrefix("--") else {
|
||||
throw BenchmarkError.invalidArguments("Missing value for \(flag)\n\(usage)")
|
||||
}
|
||||
guard values[flag] == nil else {
|
||||
throw BenchmarkError.invalidArguments("Duplicate argument: \(flag)\n\(usage)")
|
||||
}
|
||||
values[flag] = rawArguments[index + 1]
|
||||
index += 2
|
||||
}
|
||||
|
||||
let requiredFlags = ["--model-directory", "--corpus", "--report"]
|
||||
for flag in requiredFlags where values[flag] == nil {
|
||||
throw BenchmarkError.invalidArguments("Missing required argument: \(flag)\n\(usage)")
|
||||
}
|
||||
|
||||
let maximumSamples = try positiveInteger(
|
||||
values["--max-samples"] ?? "120",
|
||||
flag: "--max-samples"
|
||||
)
|
||||
let warmRounds = try positiveInteger(
|
||||
values["--warm-rounds"] ?? "5",
|
||||
flag: "--warm-rounds"
|
||||
)
|
||||
let currentDirectory = URL(
|
||||
fileURLWithPath: FileManager.default.currentDirectoryPath,
|
||||
isDirectory: true
|
||||
)
|
||||
|
||||
return Arguments(
|
||||
modelDirectory: resolvedURL(values["--model-directory"]!, relativeTo: currentDirectory),
|
||||
corpus: resolvedURL(values["--corpus"]!, relativeTo: currentDirectory),
|
||||
report: resolvedURL(values["--report"]!, relativeTo: currentDirectory),
|
||||
maximumSamples: maximumSamples,
|
||||
warmRounds: warmRounds
|
||||
)
|
||||
}
|
||||
|
||||
private static func positiveInteger(_ value: String, flag: String) throws -> Int {
|
||||
guard let result = Int(value), result > 0 else {
|
||||
throw BenchmarkError.invalidArguments(
|
||||
"\(flag) must be a positive integer, received: \(value)"
|
||||
)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
private static func resolvedURL(_ path: String, relativeTo baseURL: URL) -> URL {
|
||||
URL(fileURLWithPath: path, relativeTo: baseURL).standardizedFileURL
|
||||
}
|
||||
}
|
||||
|
||||
private struct Manifest: Decodable {
|
||||
let schemaVersion: Int
|
||||
let classifiers: [ManifestClassifier]
|
||||
}
|
||||
|
||||
private struct ManifestClassifier: Decodable {
|
||||
let id: String
|
||||
let modelFile: String
|
||||
let algorithm: String
|
||||
let labels: [String]
|
||||
let positiveLabel: String?
|
||||
}
|
||||
|
||||
private struct CorpusRecord: Decodable {
|
||||
let text: String
|
||||
let split: String
|
||||
}
|
||||
|
||||
private struct MemorySnapshot: Encodable {
|
||||
let currentRSSBytes: UInt64?
|
||||
let peakRSSBytes: UInt64?
|
||||
}
|
||||
|
||||
private struct TimingDistribution: Encodable {
|
||||
let rounds: Int
|
||||
let samplesPerRound: Int
|
||||
let measurementCount: Int
|
||||
let averageMilliseconds: Double
|
||||
let p50Milliseconds: Double
|
||||
let p95Milliseconds: Double
|
||||
let minimumMilliseconds: Double
|
||||
let maximumMilliseconds: Double
|
||||
}
|
||||
|
||||
private struct ModelBenchmark: Encodable {
|
||||
let id: String
|
||||
let modelFile: String
|
||||
let algorithm: String
|
||||
let labels: [String]
|
||||
let positiveLabel: String?
|
||||
let modelBytes: UInt64
|
||||
let compiledModelBytes: UInt64
|
||||
let compileMilliseconds: Double
|
||||
let coldLoadMilliseconds: Double
|
||||
let firstPredictionMilliseconds: Double
|
||||
let warmPrediction: TimingDistribution
|
||||
let predictedLabelCounts: [String: Int]
|
||||
let memoryAfterCompile: MemorySnapshot
|
||||
let memoryAfterLoad: MemorySnapshot
|
||||
let memoryAfterPredictions: MemorySnapshot
|
||||
}
|
||||
|
||||
private struct CorpusSummary: Encodable {
|
||||
let path: String
|
||||
let eligibleSplits: [String]
|
||||
let maximumSamples: Int
|
||||
let selectedSamples: Int
|
||||
let selectedSamplesBySplit: [String: Int]
|
||||
let selectionPolicy: String
|
||||
}
|
||||
|
||||
private struct BenchmarkReport: Encodable {
|
||||
let schemaVersion: Int
|
||||
let generatedAt: String
|
||||
let manifestSchemaVersion: Int
|
||||
let modelDirectory: String
|
||||
let corpus: CorpusSummary
|
||||
let warmRounds: Int
|
||||
let clock: String
|
||||
let percentileMethod: String
|
||||
let memoryAtStart: MemorySnapshot
|
||||
let memoryAtEnd: MemorySnapshot
|
||||
let models: [ModelBenchmark]
|
||||
}
|
||||
|
||||
private let fileManager = FileManager.default
|
||||
private let eligibleSplits = Set(["validation", "test", "golden"])
|
||||
|
||||
private func milliseconds(_ duration: Duration) -> Double {
|
||||
Double(duration.components.seconds) * 1_000
|
||||
+ Double(duration.components.attoseconds) / 1_000_000_000_000_000
|
||||
}
|
||||
|
||||
private func rounded(_ value: Double, places: Int = 6) -> Double {
|
||||
guard value.isFinite else { return 0 }
|
||||
let scale = pow(10, Double(places))
|
||||
return (value * scale).rounded() / scale
|
||||
}
|
||||
|
||||
private func currentRSSBytes() -> UInt64? {
|
||||
var info = mach_task_basic_info()
|
||||
var count = mach_msg_type_number_t(
|
||||
MemoryLayout<mach_task_basic_info>.size / MemoryLayout<natural_t>.size
|
||||
)
|
||||
let result = withUnsafeMutablePointer(to: &info) { pointer in
|
||||
pointer.withMemoryRebound(to: integer_t.self, capacity: Int(count)) { rebound in
|
||||
task_info(
|
||||
mach_task_self_,
|
||||
task_flavor_t(MACH_TASK_BASIC_INFO),
|
||||
rebound,
|
||||
&count
|
||||
)
|
||||
}
|
||||
}
|
||||
guard result == KERN_SUCCESS else { return nil }
|
||||
return UInt64(info.resident_size)
|
||||
}
|
||||
|
||||
private func peakRSSBytes() -> UInt64? {
|
||||
var usage = rusage()
|
||||
guard getrusage(RUSAGE_SELF, &usage) == 0, usage.ru_maxrss >= 0 else {
|
||||
return nil
|
||||
}
|
||||
// Darwin reports ru_maxrss in bytes; Linux reports KiB.
|
||||
#if os(macOS)
|
||||
return UInt64(usage.ru_maxrss)
|
||||
#else
|
||||
return UInt64(usage.ru_maxrss) * 1_024
|
||||
#endif
|
||||
}
|
||||
|
||||
private func memorySnapshot() -> MemorySnapshot {
|
||||
MemorySnapshot(
|
||||
currentRSSBytes: currentRSSBytes(),
|
||||
peakRSSBytes: peakRSSBytes()
|
||||
)
|
||||
}
|
||||
|
||||
private func validateReadableFile(_ url: URL, description: String) throws {
|
||||
var isDirectory: ObjCBool = false
|
||||
guard fileManager.fileExists(atPath: url.path, isDirectory: &isDirectory),
|
||||
!isDirectory.boolValue,
|
||||
fileManager.isReadableFile(atPath: url.path) else {
|
||||
throw BenchmarkError.missingFile("\(description) is not a readable file: \(url.path)")
|
||||
}
|
||||
}
|
||||
|
||||
private func validateModelDirectory(_ url: URL) throws {
|
||||
var isDirectory: ObjCBool = false
|
||||
guard fileManager.fileExists(atPath: url.path, isDirectory: &isDirectory),
|
||||
isDirectory.boolValue else {
|
||||
throw BenchmarkError.missingFile("Model directory does not exist: \(url.path)")
|
||||
}
|
||||
}
|
||||
|
||||
private func modelURL(for modelFile: String, in directory: URL) throws -> URL {
|
||||
guard !modelFile.isEmpty else {
|
||||
throw BenchmarkError.invalidManifest("Manifest contains an empty modelFile")
|
||||
}
|
||||
let baseURL = directory.resolvingSymlinksInPath().standardizedFileURL
|
||||
let candidateURL = directory
|
||||
.appendingPathComponent(modelFile)
|
||||
.resolvingSymlinksInPath()
|
||||
.standardizedFileURL
|
||||
let basePrefix = baseURL.path.hasSuffix("/") ? baseURL.path : baseURL.path + "/"
|
||||
guard candidateURL.path.hasPrefix(basePrefix) else {
|
||||
throw BenchmarkError.invalidManifest(
|
||||
"Model file resolves outside --model-directory: \(modelFile)"
|
||||
)
|
||||
}
|
||||
try validateReadableFile(candidateURL, description: "Model")
|
||||
return candidateURL
|
||||
}
|
||||
|
||||
private func loadManifest(from modelDirectory: URL) throws -> Manifest {
|
||||
let manifestURL = modelDirectory.appendingPathComponent(
|
||||
"clipboard-semantic-models.json"
|
||||
)
|
||||
try validateReadableFile(manifestURL, description: "Manifest")
|
||||
let manifest = try JSONDecoder().decode(
|
||||
Manifest.self,
|
||||
from: Data(contentsOf: manifestURL)
|
||||
)
|
||||
guard (1...4).contains(manifest.schemaVersion) else {
|
||||
throw BenchmarkError.invalidManifest(
|
||||
"Unsupported manifest schema \(manifest.schemaVersion); expected 1...4"
|
||||
)
|
||||
}
|
||||
guard !manifest.classifiers.isEmpty else {
|
||||
throw BenchmarkError.invalidManifest("Manifest has no classifiers")
|
||||
}
|
||||
let classifierIDs = manifest.classifiers.map(\.id)
|
||||
guard Set(classifierIDs).count == classifierIDs.count else {
|
||||
throw BenchmarkError.invalidManifest("Manifest contains duplicate classifier IDs")
|
||||
}
|
||||
for classifier in manifest.classifiers {
|
||||
guard !classifier.id.isEmpty, !classifier.labels.isEmpty else {
|
||||
throw BenchmarkError.invalidManifest(
|
||||
"Manifest classifier IDs and labels must not be empty"
|
||||
)
|
||||
}
|
||||
if classifier.id == "domain", classifier.positiveLabel != nil {
|
||||
throw BenchmarkError.invalidManifest(
|
||||
"The multiclass domain classifier must not define positiveLabel"
|
||||
)
|
||||
}
|
||||
}
|
||||
return manifest
|
||||
}
|
||||
|
||||
private func loadCorpus(from url: URL, maximumSamples: Int) throws -> [CorpusRecord] {
|
||||
try validateReadableFile(url, description: "Corpus")
|
||||
let content = try String(contentsOf: url, encoding: .utf8)
|
||||
let decoder = JSONDecoder()
|
||||
var records: [CorpusRecord] = []
|
||||
|
||||
for (offset, line) in content.split(separator: "\n").enumerated() {
|
||||
let record: CorpusRecord
|
||||
do {
|
||||
record = try decoder.decode(CorpusRecord.self, from: Data(line.utf8))
|
||||
} catch {
|
||||
throw BenchmarkError.invalidCorpus(
|
||||
"Invalid JSONL record at line \(offset + 1): \(error.localizedDescription)"
|
||||
)
|
||||
}
|
||||
guard eligibleSplits.contains(record.split) else { continue }
|
||||
records.append(record)
|
||||
if records.count == maximumSamples {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
guard !records.isEmpty else {
|
||||
throw BenchmarkError.invalidCorpus(
|
||||
"Corpus has no records in validation, test, or golden splits"
|
||||
)
|
||||
}
|
||||
return records
|
||||
}
|
||||
|
||||
private func recursiveSize(of url: URL) throws -> UInt64 {
|
||||
let resourceValues = try url.resourceValues(
|
||||
forKeys: [.isDirectoryKey, .isRegularFileKey, .fileSizeKey]
|
||||
)
|
||||
if resourceValues.isRegularFile == true {
|
||||
return UInt64(resourceValues.fileSize ?? 0)
|
||||
}
|
||||
guard resourceValues.isDirectory == true else { return 0 }
|
||||
guard let enumerator = fileManager.enumerator(
|
||||
at: url,
|
||||
includingPropertiesForKeys: [.isRegularFileKey, .fileSizeKey],
|
||||
options: [.skipsHiddenFiles]
|
||||
) else {
|
||||
return 0
|
||||
}
|
||||
|
||||
var total: UInt64 = 0
|
||||
for case let childURL as URL in enumerator {
|
||||
let childValues = try childURL.resourceValues(
|
||||
forKeys: [.isRegularFileKey, .fileSizeKey]
|
||||
)
|
||||
if childValues.isRegularFile == true {
|
||||
total += UInt64(childValues.fileSize ?? 0)
|
||||
}
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
private func compileModel(sourceURL: URL, outputDirectory: URL) throws -> URL {
|
||||
let generatedURL = try MLModel.compileModel(at: sourceURL)
|
||||
let destinationURL = outputDirectory
|
||||
.appendingPathComponent(sourceURL.deletingPathExtension().lastPathComponent)
|
||||
.appendingPathExtension("mlmodelc")
|
||||
if fileManager.fileExists(atPath: destinationURL.path) {
|
||||
try fileManager.removeItem(at: destinationURL)
|
||||
}
|
||||
try fileManager.moveItem(at: generatedURL, to: destinationURL)
|
||||
return destinationURL
|
||||
}
|
||||
|
||||
private func percentile(_ sortedValues: [Double], fraction: Double) -> Double {
|
||||
guard !sortedValues.isEmpty else { return 0 }
|
||||
let rank = max(1, Int(ceil(fraction * Double(sortedValues.count))))
|
||||
return sortedValues[min(rank - 1, sortedValues.count - 1)]
|
||||
}
|
||||
|
||||
private func timingDistribution(
|
||||
values: [Double],
|
||||
rounds: Int,
|
||||
samplesPerRound: Int
|
||||
) -> TimingDistribution {
|
||||
let sortedValues = values.sorted()
|
||||
let average = values.reduce(0, +) / Double(values.count)
|
||||
return TimingDistribution(
|
||||
rounds: rounds,
|
||||
samplesPerRound: samplesPerRound,
|
||||
measurementCount: values.count,
|
||||
averageMilliseconds: rounded(average),
|
||||
p50Milliseconds: rounded(percentile(sortedValues, fraction: 0.50)),
|
||||
p95Milliseconds: rounded(percentile(sortedValues, fraction: 0.95)),
|
||||
minimumMilliseconds: rounded(sortedValues.first ?? 0),
|
||||
maximumMilliseconds: rounded(sortedValues.last ?? 0)
|
||||
)
|
||||
}
|
||||
|
||||
private func benchmark(
|
||||
classifier: ManifestClassifier,
|
||||
modelDirectory: URL,
|
||||
temporaryDirectory: URL,
|
||||
records: [CorpusRecord],
|
||||
warmRounds: Int
|
||||
) throws -> ModelBenchmark {
|
||||
let sourceURL = try modelURL(for: classifier.modelFile, in: modelDirectory)
|
||||
let sourceBytes = try recursiveSize(of: sourceURL)
|
||||
|
||||
let compileStartedAt = ContinuousClock.now
|
||||
let compiledURL = try compileModel(
|
||||
sourceURL: sourceURL,
|
||||
outputDirectory: temporaryDirectory
|
||||
)
|
||||
let compileMilliseconds = milliseconds(compileStartedAt.duration(to: .now))
|
||||
let memoryAfterCompile = memorySnapshot()
|
||||
let compiledBytes = try recursiveSize(of: compiledURL)
|
||||
|
||||
let loadStartedAt = ContinuousClock.now
|
||||
let model = try NLModel(contentsOf: compiledURL)
|
||||
let loadMilliseconds = milliseconds(loadStartedAt.duration(to: .now))
|
||||
let memoryAfterLoad = memorySnapshot()
|
||||
|
||||
let firstPredictionStartedAt = ContinuousClock.now
|
||||
_ = model.predictedLabel(for: records[0].text)
|
||||
let firstPredictionMilliseconds = milliseconds(
|
||||
firstPredictionStartedAt.duration(to: .now)
|
||||
)
|
||||
|
||||
var warmMeasurements: [Double] = []
|
||||
warmMeasurements.reserveCapacity(records.count * warmRounds)
|
||||
var predictedLabelCounts: [String: Int] = [:]
|
||||
for _ in 0..<warmRounds {
|
||||
for record in records {
|
||||
let predictionStartedAt = ContinuousClock.now
|
||||
let predictedLabel = model.predictedLabel(for: record.text) ?? "__noPrediction__"
|
||||
warmMeasurements.append(
|
||||
milliseconds(predictionStartedAt.duration(to: .now))
|
||||
)
|
||||
predictedLabelCounts[predictedLabel, default: 0] += 1
|
||||
}
|
||||
}
|
||||
|
||||
return ModelBenchmark(
|
||||
id: classifier.id,
|
||||
modelFile: classifier.modelFile,
|
||||
algorithm: classifier.algorithm,
|
||||
labels: classifier.labels,
|
||||
positiveLabel: classifier.positiveLabel,
|
||||
modelBytes: sourceBytes,
|
||||
compiledModelBytes: compiledBytes,
|
||||
compileMilliseconds: rounded(compileMilliseconds),
|
||||
coldLoadMilliseconds: rounded(loadMilliseconds),
|
||||
firstPredictionMilliseconds: rounded(firstPredictionMilliseconds),
|
||||
warmPrediction: timingDistribution(
|
||||
values: warmMeasurements,
|
||||
rounds: warmRounds,
|
||||
samplesPerRound: records.count
|
||||
),
|
||||
predictedLabelCounts: predictedLabelCounts,
|
||||
memoryAfterCompile: memoryAfterCompile,
|
||||
memoryAfterLoad: memoryAfterLoad,
|
||||
memoryAfterPredictions: memorySnapshot()
|
||||
)
|
||||
}
|
||||
|
||||
private func writeReport(_ report: BenchmarkReport, to url: URL) throws {
|
||||
try fileManager.createDirectory(
|
||||
at: url.deletingLastPathComponent(),
|
||||
withIntermediateDirectories: true
|
||||
)
|
||||
let encoder = JSONEncoder()
|
||||
encoder.outputFormatting = [.prettyPrinted, .sortedKeys, .withoutEscapingSlashes]
|
||||
try encoder.encode(report).write(to: url, options: .atomic)
|
||||
}
|
||||
|
||||
private func formattedMiB(_ bytes: UInt64?) -> String {
|
||||
guard let bytes else { return "unavailable" }
|
||||
return String(format: "%.1f MiB", Double(bytes) / 1_048_576)
|
||||
}
|
||||
|
||||
private func run() throws {
|
||||
let arguments = try Arguments.parse(Array(CommandLine.arguments.dropFirst()))
|
||||
try validateModelDirectory(arguments.modelDirectory)
|
||||
let manifest = try loadManifest(from: arguments.modelDirectory)
|
||||
let records = try loadCorpus(
|
||||
from: arguments.corpus,
|
||||
maximumSamples: arguments.maximumSamples
|
||||
)
|
||||
let splitCounts = Dictionary(grouping: records, by: \.split).mapValues(\.count)
|
||||
let memoryAtStart = memorySnapshot()
|
||||
|
||||
let temporaryDirectory = fileManager.temporaryDirectory.appendingPathComponent(
|
||||
"osg-v6-model-benchmark-\(UUID().uuidString)",
|
||||
isDirectory: true
|
||||
)
|
||||
try fileManager.createDirectory(
|
||||
at: temporaryDirectory,
|
||||
withIntermediateDirectories: true
|
||||
)
|
||||
defer { try? fileManager.removeItem(at: temporaryDirectory) }
|
||||
|
||||
var modelBenchmarks: [ModelBenchmark] = []
|
||||
modelBenchmarks.reserveCapacity(manifest.classifiers.count)
|
||||
for classifier in manifest.classifiers {
|
||||
modelBenchmarks.append(
|
||||
try benchmark(
|
||||
classifier: classifier,
|
||||
modelDirectory: arguments.modelDirectory,
|
||||
temporaryDirectory: temporaryDirectory,
|
||||
records: records,
|
||||
warmRounds: arguments.warmRounds
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
let report = BenchmarkReport(
|
||||
schemaVersion: 1,
|
||||
generatedAt: ISO8601DateFormatter().string(from: Date()),
|
||||
manifestSchemaVersion: manifest.schemaVersion,
|
||||
modelDirectory: arguments.modelDirectory.path,
|
||||
corpus: CorpusSummary(
|
||||
path: arguments.corpus.path,
|
||||
eligibleSplits: eligibleSplits.sorted(),
|
||||
maximumSamples: arguments.maximumSamples,
|
||||
selectedSamples: records.count,
|
||||
selectedSamplesBySplit: splitCounts,
|
||||
selectionPolicy: "first eligible records in corpus order"
|
||||
),
|
||||
warmRounds: arguments.warmRounds,
|
||||
clock: "ContinuousClock",
|
||||
percentileMethod: "nearest-rank",
|
||||
memoryAtStart: memoryAtStart,
|
||||
memoryAtEnd: memorySnapshot(),
|
||||
models: modelBenchmarks
|
||||
)
|
||||
try writeReport(report, to: arguments.report)
|
||||
|
||||
let totalCompile = modelBenchmarks.reduce(0) { $0 + $1.compileMilliseconds }
|
||||
let totalLoad = modelBenchmarks.reduce(0) { $0 + $1.coldLoadMilliseconds }
|
||||
let warmAverage = modelBenchmarks.reduce(0) {
|
||||
$0 + $1.warmPrediction.averageMilliseconds
|
||||
} / Double(modelBenchmarks.count)
|
||||
print(
|
||||
String(
|
||||
format: "V6 benchmark: %d models, %d samples × %d rounds; compile %.3f ms, cold load %.3f ms, warm avg %.3f ms, peak RSS %@; report %@",
|
||||
modelBenchmarks.count,
|
||||
records.count,
|
||||
arguments.warmRounds,
|
||||
totalCompile,
|
||||
totalLoad,
|
||||
warmAverage,
|
||||
formattedMiB(report.memoryAtEnd.peakRSSBytes),
|
||||
arguments.report.path
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
do {
|
||||
try run()
|
||||
} catch {
|
||||
let message = "benchmark_v6_models: \(error.localizedDescription)\n"
|
||||
FileHandle.standardError.write(Data(message.utf8))
|
||||
exit(EXIT_FAILURE)
|
||||
}
|
||||
@@ -0,0 +1,653 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build a provenance-preserving clipboard-semantics corpus registry."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import unicodedata
|
||||
from collections import Counter, defaultdict
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Iterator
|
||||
|
||||
INTENT_FIELDS = (
|
||||
"task",
|
||||
"question",
|
||||
"invitation",
|
||||
"complaint",
|
||||
"scheduleNegotiation",
|
||||
"confirmationDecision",
|
||||
"followUpReminder",
|
||||
"blessing",
|
||||
"replyableMessage",
|
||||
"assistantCommand",
|
||||
"informationQuery",
|
||||
"systemNotification",
|
||||
)
|
||||
DOMAINS = {
|
||||
"finance",
|
||||
"travel",
|
||||
"calendar",
|
||||
"communication",
|
||||
"media",
|
||||
"smartHome",
|
||||
"shopping",
|
||||
"dining",
|
||||
"health",
|
||||
"weather",
|
||||
"accountService",
|
||||
"generalKnowledge",
|
||||
}
|
||||
SPLIT_USE = {
|
||||
"train": "train",
|
||||
"silverTrain": "train",
|
||||
"validation": "calibration-only",
|
||||
"silverCalibration": "calibration-only",
|
||||
"test": "evaluation-only",
|
||||
"golden": "evaluation-only",
|
||||
"silverAcceptance": "evaluation-only",
|
||||
"calibration": "calibration-only",
|
||||
}
|
||||
USE_PRIORITY = {
|
||||
"train": 0,
|
||||
"calibration-only": 1,
|
||||
"evaluation-only": 2,
|
||||
"research-only": 3,
|
||||
}
|
||||
TRAIN_LICENSES = {
|
||||
"Apache-2.0",
|
||||
"CC0-1.0 source / Apache-2.0 mirror",
|
||||
"CC-BY-3.0",
|
||||
"CC-BY-4.0",
|
||||
"CDLA-Permissive-1.0",
|
||||
"MIT",
|
||||
"OSGKeyboard project license",
|
||||
"CC0-1.0",
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Source:
|
||||
identifier: str
|
||||
path: Path
|
||||
license: str
|
||||
default_use: str
|
||||
source_type: str
|
||||
all_intents_known: bool
|
||||
known_intent_labels: frozenset[str] | None
|
||||
sentiment_known: bool
|
||||
required: bool
|
||||
sensitive: bool
|
||||
weight: float
|
||||
|
||||
|
||||
def normalize_text(value: str) -> str:
|
||||
return " ".join(
|
||||
unicodedata.normalize("NFKC", value)
|
||||
.replace("\u0000", " ")
|
||||
.casefold()
|
||||
.split()
|
||||
).strip()
|
||||
|
||||
|
||||
def text_hash(value: str) -> str:
|
||||
return hashlib.sha256(normalize_text(value).encode()).hexdigest()
|
||||
|
||||
|
||||
def cluster_signature(text: str, language: str, family: str) -> str:
|
||||
normalized = normalize_text(text)
|
||||
normalized = re.sub(r"https?://\S+|www\.\S+", "<url>", normalized)
|
||||
normalized = re.sub(r"[\w.+-]+@[\w.-]+\.[a-z]{2,}", "<email>", normalized)
|
||||
normalized = re.sub(r"\d+(?:[./:-]\d+)*", "<n>", normalized)
|
||||
normalized = re.sub(r"[^\w\u3400-\u9fff<>]+", " ", normalized)
|
||||
tokens = normalized.split()
|
||||
skeleton = tokens[:12] + (["|"] + tokens[-8:] if len(tokens) > 20 else [])
|
||||
value = f"{language}|{family}|{' '.join(skeleton)}"
|
||||
return hashlib.sha256(value.encode()).hexdigest()
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def iter_records(path: Path) -> Iterator[dict]:
|
||||
with path.open(encoding="utf-8") as handle:
|
||||
first = ""
|
||||
while not first:
|
||||
first = handle.readline()
|
||||
if not first:
|
||||
return
|
||||
first = first.strip()
|
||||
if first.startswith("["):
|
||||
payload = json.loads(first + handle.read())
|
||||
if not isinstance(payload, list):
|
||||
raise TypeError(f"Expected JSON array in {path}")
|
||||
yield from payload
|
||||
return
|
||||
yield json.loads(first)
|
||||
for line in handle:
|
||||
if line.strip():
|
||||
yield json.loads(line)
|
||||
|
||||
|
||||
def resolve_path(raw_path: str, repository_root: Path) -> Path:
|
||||
expanded = raw_path.replace("${REPO_ROOT}", str(repository_root))
|
||||
path = Path(expanded).expanduser()
|
||||
return path if path.is_absolute() else repository_root / path
|
||||
|
||||
|
||||
def load_sources(manifest_path: Path, repository_root: Path) -> tuple[list[Source], dict]:
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
sources = []
|
||||
for raw in manifest["sources"]:
|
||||
if not raw.get("enabled", True):
|
||||
continue
|
||||
weight = float(raw.get("weight", 1.0))
|
||||
if not 0 < weight <= 1:
|
||||
raise ValueError(f"Invalid source weight for {raw['id']}: {weight}")
|
||||
known_labels = raw.get("knownIntentLabels")
|
||||
if known_labels is not None:
|
||||
unknown = set(known_labels) - set(INTENT_FIELDS)
|
||||
if unknown:
|
||||
raise ValueError(
|
||||
f"Unknown intent labels for {raw['id']}: {sorted(unknown)}"
|
||||
)
|
||||
sources.append(
|
||||
Source(
|
||||
identifier=raw["id"],
|
||||
path=resolve_path(raw["path"], repository_root),
|
||||
license=raw["license"],
|
||||
default_use=raw["defaultUse"],
|
||||
source_type=raw["sourceType"],
|
||||
all_intents_known=raw.get("allIntentLabelsKnown", False),
|
||||
known_intent_labels=(
|
||||
frozenset(known_labels) if known_labels is not None else None
|
||||
),
|
||||
sentiment_known=raw.get("sentimentKnown", False),
|
||||
required=raw.get("required", False),
|
||||
sensitive=raw.get("sensitive", False),
|
||||
weight=weight,
|
||||
)
|
||||
)
|
||||
return sources, manifest
|
||||
|
||||
|
||||
def source_use(source: Source, record: dict) -> str:
|
||||
if source.default_use != "by-split":
|
||||
return source.default_use
|
||||
return SPLIT_USE.get(
|
||||
record.get("split") or record.get("sourceSplit") or "",
|
||||
"evaluation-only",
|
||||
)
|
||||
|
||||
|
||||
def known_intents(source: Source, record: dict) -> set[str]:
|
||||
if source.known_intent_labels is not None:
|
||||
values = set(source.known_intent_labels)
|
||||
record_values = set(record.get("knownLabels") or [])
|
||||
if "replyable" in record_values:
|
||||
record_values.add("replyableMessage")
|
||||
return values | (record_values & set(INTENT_FIELDS))
|
||||
if source.all_intents_known:
|
||||
return set(INTENT_FIELDS)
|
||||
values = set(record.get("knownLabels") or [])
|
||||
if "replyable" in values:
|
||||
values.add("replyableMessage")
|
||||
return values & set(INTENT_FIELDS)
|
||||
|
||||
|
||||
def record_value(record: dict, label: str) -> bool:
|
||||
source_field = "replyable" if label == "replyableMessage" else label
|
||||
return bool(record.get(source_field))
|
||||
|
||||
|
||||
def effective_license(source: Source, record: dict) -> str:
|
||||
return record.get("sourceLicense") or source.license
|
||||
|
||||
|
||||
def effective_weight(source: Source, record: dict) -> float:
|
||||
record_weight = float(record.get("sampleWeight", 1.0))
|
||||
if not 0 < record_weight <= 1:
|
||||
raise ValueError(
|
||||
f"Invalid record sampleWeight for {record.get('id')}: {record_weight}"
|
||||
)
|
||||
return round(source.weight * record_weight, 6)
|
||||
|
||||
|
||||
def add_record(
|
||||
registry: dict[str, dict],
|
||||
source: Source,
|
||||
record: dict,
|
||||
source_path: Path,
|
||||
) -> str:
|
||||
text = str(record.get("text") or "").strip()
|
||||
language = str(record.get("language") or "").strip()
|
||||
if not text or language not in {"en", "zh-Hans"}:
|
||||
return "invalid"
|
||||
digest = text_hash(text)
|
||||
family = str(record.get("family") or "unknown")
|
||||
use = source_use(source, record)
|
||||
license_name = effective_license(source, record)
|
||||
if use == "train" and license_name not in TRAIN_LICENSES:
|
||||
return "unsafe-license"
|
||||
known_labels = set(record.get("knownLabels") or [])
|
||||
if "domain" in known_labels and record.get("domain") not in DOMAINS:
|
||||
return "invalid-domain"
|
||||
sample_weight = effective_weight(source, record)
|
||||
canonical = registry.setdefault(
|
||||
digest,
|
||||
{
|
||||
"id": f"corpus-{digest[:20]}",
|
||||
"text": text,
|
||||
"normalizedTextSHA256": digest,
|
||||
"language": language,
|
||||
"clusterSignature": cluster_signature(text, language, family),
|
||||
"families": set(),
|
||||
"uses": set(),
|
||||
"provenance": [],
|
||||
"intentEvidence": defaultdict(list),
|
||||
"sentimentEvidence": [],
|
||||
"domainEvidence": [],
|
||||
},
|
||||
)
|
||||
canonical["families"].add(family)
|
||||
canonical["uses"].add(use)
|
||||
known = known_intents(source, record)
|
||||
for label in known:
|
||||
canonical["intentEvidence"][label].append(
|
||||
{
|
||||
"source": source.identifier,
|
||||
"value": record_value(record, label),
|
||||
}
|
||||
)
|
||||
sentiment_is_known = source.sentiment_known or "sentiment" in set(
|
||||
record.get("knownLabels") or []
|
||||
)
|
||||
if sentiment_is_known and record.get("sentiment") in {
|
||||
"negative",
|
||||
"neutral",
|
||||
"positive",
|
||||
}:
|
||||
canonical["sentimentEvidence"].append(
|
||||
{
|
||||
"source": source.identifier,
|
||||
"value": record["sentiment"],
|
||||
}
|
||||
)
|
||||
if "domain" in known_labels:
|
||||
domain = record.get("domain")
|
||||
canonical["domainEvidence"].append(
|
||||
{
|
||||
"source": source.identifier,
|
||||
"value": domain,
|
||||
}
|
||||
)
|
||||
canonical["provenance"].append(
|
||||
{
|
||||
"source": source.identifier,
|
||||
"sourcePath": str(source_path),
|
||||
"sourceRecordID": record.get("id"),
|
||||
"sourceDataset": record.get("sourceDataset") or source.identifier,
|
||||
"sourceRevision": record.get("sourceRevision"),
|
||||
"sourceURL": record.get("sourceURL"),
|
||||
"split": record.get("split") or record.get("sourceSplit"),
|
||||
"allowedUse": use,
|
||||
"license": license_name,
|
||||
"sourceType": source.source_type,
|
||||
"sensitive": source.sensitive,
|
||||
"sampleWeight": sample_weight,
|
||||
}
|
||||
)
|
||||
return "accepted"
|
||||
|
||||
|
||||
def resolve_state(evidence: list[dict]) -> tuple[str, bool]:
|
||||
values = {item["value"] for item in evidence}
|
||||
if len(values) != 1:
|
||||
return "unknown", len(values) > 1
|
||||
return ("true" if values.pop() is True else "false"), False
|
||||
|
||||
|
||||
def finalize_record(raw: dict) -> dict:
|
||||
states = {}
|
||||
conflicts = []
|
||||
evidence = {}
|
||||
for label in INTENT_FIELDS:
|
||||
values = raw["intentEvidence"].get(label, [])
|
||||
state, conflict = resolve_state(values)
|
||||
states[label] = state
|
||||
if values:
|
||||
evidence[label] = values
|
||||
if conflict:
|
||||
conflicts.append(label)
|
||||
sentiment_values = {
|
||||
item["value"] for item in raw["sentimentEvidence"]
|
||||
}
|
||||
sentiment = (
|
||||
next(iter(sentiment_values)) if len(sentiment_values) == 1 else "unknown"
|
||||
)
|
||||
if len(sentiment_values) > 1:
|
||||
conflicts.append("sentiment")
|
||||
domain_values = {item["value"] for item in raw["domainEvidence"]}
|
||||
domain = next(iter(domain_values)) if len(domain_values) == 1 else "unknown"
|
||||
if len(domain_values) > 1:
|
||||
conflicts.append("domain")
|
||||
allowed_use = max(raw["uses"], key=USE_PRIORITY.__getitem__)
|
||||
training_weights = [
|
||||
value["sampleWeight"]
|
||||
for value in raw["provenance"]
|
||||
if value["allowedUse"] == "train"
|
||||
]
|
||||
training_datasets = sorted(
|
||||
{
|
||||
value["sourceDataset"]
|
||||
for value in raw["provenance"]
|
||||
if value["allowedUse"] == "train"
|
||||
}
|
||||
)
|
||||
known_labels = [
|
||||
label for label, state in states.items() if state != "unknown"
|
||||
]
|
||||
if sentiment != "unknown":
|
||||
known_labels.append("sentiment")
|
||||
if domain != "unknown":
|
||||
known_labels.append("domain")
|
||||
flattened_labels = {
|
||||
("replyable" if label == "replyableMessage" else label): state == "true"
|
||||
for label, state in states.items()
|
||||
}
|
||||
return {
|
||||
"id": raw["id"],
|
||||
"text": raw["text"],
|
||||
"normalizedTextSHA256": raw["normalizedTextSHA256"],
|
||||
"language": raw["language"],
|
||||
"sourceDataset": training_datasets[0] if training_datasets else None,
|
||||
"clusterSignature": raw["clusterSignature"],
|
||||
"families": sorted(raw["families"]),
|
||||
"family": sorted(raw["families"])[0],
|
||||
"observedUses": sorted(raw["uses"], key=USE_PRIORITY.__getitem__),
|
||||
"allowedUse": allowed_use,
|
||||
"split": (
|
||||
"train"
|
||||
if allowed_use == "train"
|
||||
else "validation"
|
||||
if allowed_use == "calibration-only"
|
||||
else "test"
|
||||
),
|
||||
**flattened_labels,
|
||||
"labels": states,
|
||||
"sentiment": sentiment if sentiment != "unknown" else "neutral",
|
||||
"domain": domain if domain != "unknown" else None,
|
||||
"knownLabels": sorted(known_labels),
|
||||
"sampleWeight": max(training_weights, default=1.0),
|
||||
"labelConflicts": sorted(conflicts),
|
||||
"sourceEvidence": evidence,
|
||||
"sentimentEvidence": raw["sentimentEvidence"],
|
||||
"domainEvidence": raw["domainEvidence"],
|
||||
"provenance": raw["provenance"],
|
||||
}
|
||||
|
||||
|
||||
def write_json_lines(path: Path, records: list[dict]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", encoding="utf-8") as handle:
|
||||
for record in records:
|
||||
handle.write(
|
||||
json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n"
|
||||
)
|
||||
|
||||
|
||||
def select_pilot(records: list[dict], count: int) -> list[dict]:
|
||||
grouped: dict[tuple[str, str, str], list[dict]] = defaultdict(list)
|
||||
for record in records:
|
||||
source = sorted(
|
||||
{
|
||||
value["sourceDataset"]
|
||||
for value in record["provenance"]
|
||||
if value["allowedUse"] == "train"
|
||||
}
|
||||
)[0]
|
||||
family = record["families"][0]
|
||||
grouped[(record["language"], source, family)].append(record)
|
||||
for values in grouped.values():
|
||||
values.sort(
|
||||
key=lambda item: (
|
||||
not item["labelConflicts"],
|
||||
item["clusterSignature"],
|
||||
item["id"],
|
||||
)
|
||||
)
|
||||
keys = sorted(grouped)
|
||||
selected = []
|
||||
seen_clusters = set()
|
||||
offsets = defaultdict(int)
|
||||
while len(selected) < count:
|
||||
added = False
|
||||
for key in keys:
|
||||
values = grouped[key]
|
||||
while offsets[key] < len(values):
|
||||
candidate = values[offsets[key]]
|
||||
offsets[key] += 1
|
||||
if candidate["clusterSignature"] in seen_clusters:
|
||||
continue
|
||||
selected.append(
|
||||
{
|
||||
"id": candidate["id"],
|
||||
"text": candidate["text"],
|
||||
"language": candidate["language"],
|
||||
}
|
||||
)
|
||||
seen_clusters.add(candidate["clusterSignature"])
|
||||
added = True
|
||||
break
|
||||
if len(selected) >= count:
|
||||
break
|
||||
if not added:
|
||||
break
|
||||
return sorted(selected, key=lambda item: item["id"])
|
||||
|
||||
|
||||
def build(arguments: argparse.Namespace) -> dict:
|
||||
repository_root = arguments.repository_root.resolve()
|
||||
sources, manifest = load_sources(arguments.source_manifest, repository_root)
|
||||
raw_registry: dict[str, dict] = {}
|
||||
source_reports = []
|
||||
missing_sources = []
|
||||
excluded_counts = Counter()
|
||||
for source in sources:
|
||||
if not source.path.exists():
|
||||
if source.required:
|
||||
raise FileNotFoundError(f"Required corpus is missing: {source.path}")
|
||||
missing_sources.append(source.identifier)
|
||||
continue
|
||||
statuses = Counter()
|
||||
for record in iter_records(source.path):
|
||||
statuses[add_record(raw_registry, source, record, source.path)] += 1
|
||||
source_reports.append(
|
||||
{
|
||||
"id": source.identifier,
|
||||
"path": str(source.path),
|
||||
"sha256": sha256_file(source.path),
|
||||
"records": sum(statuses.values()),
|
||||
"statuses": dict(sorted(statuses.items())),
|
||||
}
|
||||
)
|
||||
excluded_counts.update(
|
||||
{
|
||||
key: value
|
||||
for key, value in statuses.items()
|
||||
if key != "accepted"
|
||||
}
|
||||
)
|
||||
records = sorted(
|
||||
(finalize_record(value) for value in raw_registry.values()),
|
||||
key=lambda item: item["id"],
|
||||
)
|
||||
train_candidates = [
|
||||
record
|
||||
for record in records
|
||||
if record["allowedUse"] == "train"
|
||||
and not record["labelConflicts"]
|
||||
and (
|
||||
any(value != "unknown" for value in record["labels"].values())
|
||||
or record["sentiment"] != "unknown"
|
||||
or record["domain"] != "unknown"
|
||||
)
|
||||
]
|
||||
conflicts = [
|
||||
{
|
||||
"id": record["id"],
|
||||
"text": record["text"],
|
||||
"language": record["language"],
|
||||
"conflicts": record["labelConflicts"],
|
||||
"sourceEvidence": record["sourceEvidence"],
|
||||
"sentimentEvidence": record["sentimentEvidence"],
|
||||
"domainEvidence": record["domainEvidence"],
|
||||
"resolution": None,
|
||||
"reviewer": None,
|
||||
}
|
||||
for record in records
|
||||
if record["labelConflicts"]
|
||||
]
|
||||
pilot_was_preserved = (
|
||||
getattr(arguments, "preserve_pilot", False) and arguments.pilot.exists()
|
||||
)
|
||||
if pilot_was_preserved:
|
||||
pilot = list(iter_records(arguments.pilot))
|
||||
if len(pilot) != arguments.pilot_count:
|
||||
raise ValueError(
|
||||
f"Preserved pilot has {len(pilot)} records; "
|
||||
f"expected {arguments.pilot_count}"
|
||||
)
|
||||
else:
|
||||
pilot = select_pilot(train_candidates, arguments.pilot_count)
|
||||
write_json_lines(arguments.registry, records)
|
||||
write_json_lines(arguments.train_candidates, train_candidates)
|
||||
if not pilot_was_preserved:
|
||||
write_json_lines(arguments.pilot, pilot)
|
||||
write_json_lines(arguments.human_review, conflicts)
|
||||
usage_counts = Counter(record["allowedUse"] for record in records)
|
||||
language_counts = Counter(record["language"] for record in records)
|
||||
train_barred_by_evaluation = sum(
|
||||
"train" in record["observedUses"]
|
||||
and "evaluation-only" in record["observedUses"]
|
||||
for record in records
|
||||
)
|
||||
train_barred_by_calibration = sum(
|
||||
"train" in record["observedUses"]
|
||||
and "calibration-only" in record["observedUses"]
|
||||
for record in records
|
||||
)
|
||||
report = {
|
||||
"schemaVersion": 1,
|
||||
"sourceManifest": str(arguments.source_manifest),
|
||||
"sourceManifestSHA256": sha256_file(arguments.source_manifest),
|
||||
"sourceReports": source_reports,
|
||||
"excludedSources": manifest.get("excludedSources", []),
|
||||
"missingOptionalSources": missing_sources,
|
||||
"inputRecordCount": sum(
|
||||
item["records"] for item in source_reports
|
||||
),
|
||||
"canonicalRecordCount": len(records),
|
||||
"exactDuplicateCount": sum(
|
||||
max(len(record["provenance"]) - 1, 0) for record in records
|
||||
),
|
||||
"trainCandidateCount": len(train_candidates),
|
||||
"trainBarredByEvaluationCount": train_barred_by_evaluation,
|
||||
"trainBarredByCalibrationCount": train_barred_by_calibration,
|
||||
"labelConflictCount": len(conflicts),
|
||||
"pilotCount": len(pilot),
|
||||
"pilotPreserved": pilot_was_preserved,
|
||||
"pilotSHA256": sha256_file(arguments.pilot),
|
||||
"usageCounts": dict(sorted(usage_counts.items())),
|
||||
"languageCounts": dict(sorted(language_counts.items())),
|
||||
"excludedRecordCounts": dict(sorted(excluded_counts.items())),
|
||||
"outputs": {
|
||||
"registry": str(arguments.registry),
|
||||
"trainCandidates": str(arguments.train_candidates),
|
||||
"pilot": str(arguments.pilot),
|
||||
"humanReview": str(arguments.human_review),
|
||||
},
|
||||
}
|
||||
arguments.report.parent.mkdir(parents=True, exist_ok=True)
|
||||
arguments.report.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return report
|
||||
|
||||
|
||||
def parser() -> argparse.ArgumentParser:
|
||||
root = argparse.ArgumentParser()
|
||||
root.add_argument("--repository-root", type=Path, default=Path.cwd())
|
||||
root.add_argument(
|
||||
"--source-manifest",
|
||||
type=Path,
|
||||
default=Path(
|
||||
"ModelTraining/ClipboardSemantics/corpus-registry-sources.json"
|
||||
),
|
||||
)
|
||||
root.add_argument(
|
||||
"--registry",
|
||||
type=Path,
|
||||
default=Path(
|
||||
"ModelTraining/ClipboardSemantics/CorpusRegistry/registry.jsonl"
|
||||
),
|
||||
)
|
||||
root.add_argument(
|
||||
"--train-candidates",
|
||||
type=Path,
|
||||
default=Path(
|
||||
"ModelTraining/ClipboardSemantics/CorpusRegistry/"
|
||||
"train-candidates.jsonl"
|
||||
),
|
||||
)
|
||||
root.add_argument(
|
||||
"--pilot",
|
||||
type=Path,
|
||||
default=Path(
|
||||
"ModelTraining/ClipboardSemantics/CorpusRegistry/"
|
||||
"labeling-pilot.jsonl"
|
||||
),
|
||||
)
|
||||
root.add_argument(
|
||||
"--human-review",
|
||||
type=Path,
|
||||
default=Path(
|
||||
"ModelTraining/ClipboardSemantics/CorpusRegistry/"
|
||||
"source-conflicts-human-review.jsonl"
|
||||
),
|
||||
)
|
||||
root.add_argument(
|
||||
"--report",
|
||||
type=Path,
|
||||
default=Path(
|
||||
"ModelTraining/ClipboardSemantics/CorpusRegistry/registry-report.json"
|
||||
),
|
||||
)
|
||||
root.add_argument("--pilot-count", type=int, default=1000)
|
||||
root.add_argument("--preserve-pilot", action="store_true")
|
||||
return root
|
||||
|
||||
|
||||
def main() -> None:
|
||||
arguments = parser().parse_args()
|
||||
report = build(arguments)
|
||||
print(
|
||||
"CORPUS_REGISTRY_DONE "
|
||||
f"canonical={report['canonicalRecordCount']} "
|
||||
f"train={report['trainCandidateCount']} "
|
||||
f"pilot={report['pilotCount']} "
|
||||
f"conflicts={report['labelConflictCount']}"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,304 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Prepare and evaluate product-owner clipboard semantic anchor labels."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
from adjudicate_consensus_conflicts import (
|
||||
load_adjudicator,
|
||||
read_json_lines,
|
||||
write_json_lines,
|
||||
)
|
||||
|
||||
ANCHOR_FIELDS = (
|
||||
"replyableMessage",
|
||||
"task",
|
||||
"question",
|
||||
"assistantCommand",
|
||||
"informationQuery",
|
||||
"systemNotification",
|
||||
"domain",
|
||||
"ambiguous",
|
||||
)
|
||||
PROMPT_VERSION = "clipboard-adjudication-v5"
|
||||
|
||||
|
||||
def read_anchors(path: Path) -> list[dict]:
|
||||
records = json.loads(path.read_text(encoding="utf-8"))
|
||||
if not isinstance(records, list) or not records:
|
||||
raise ValueError("Anchor file must contain a non-empty JSON array")
|
||||
identifiers = [record.get("id") for record in records]
|
||||
if len(set(identifiers)) != len(identifiers):
|
||||
raise ValueError("Anchor file contains duplicate ids")
|
||||
return records
|
||||
|
||||
|
||||
def prepare(arguments: argparse.Namespace) -> dict:
|
||||
anchors = read_anchors(arguments.anchors)
|
||||
queue = [
|
||||
{
|
||||
"id": record["id"],
|
||||
"text": record["text"],
|
||||
"language": record["language"],
|
||||
"unresolvedFields": list(ANCHOR_FIELDS),
|
||||
}
|
||||
for record in anchors
|
||||
]
|
||||
write_json_lines(arguments.queue, queue)
|
||||
return {
|
||||
"promptVersion": PROMPT_VERSION,
|
||||
"anchorCount": len(anchors),
|
||||
"queueCount": len(queue),
|
||||
}
|
||||
|
||||
|
||||
def prepare_holdout(arguments: argparse.Namespace) -> dict:
|
||||
records = read_json_lines(arguments.holdout)
|
||||
identifiers = [record.get("id") for record in records]
|
||||
if len(set(identifiers)) != len(identifiers):
|
||||
raise ValueError("Holdout contains duplicate ids")
|
||||
queue = [
|
||||
{
|
||||
"id": record["id"],
|
||||
"text": record["text"],
|
||||
"language": record["language"],
|
||||
"unresolvedFields": list(ANCHOR_FIELDS),
|
||||
}
|
||||
for record in records
|
||||
]
|
||||
write_json_lines(arguments.queue, queue)
|
||||
return {
|
||||
"promptVersion": PROMPT_VERSION,
|
||||
"holdoutCount": len(records),
|
||||
"queueCount": len(queue),
|
||||
}
|
||||
|
||||
|
||||
def parse_labeler(value: str) -> tuple[str, Path]:
|
||||
name, separator, path = value.partition("=")
|
||||
if not separator or not name or not path:
|
||||
raise argparse.ArgumentTypeError("Labeler must use name=path")
|
||||
return name, Path(path)
|
||||
|
||||
|
||||
def actual_value(adjudication: dict, field: str) -> str:
|
||||
if field == "recordDisposition":
|
||||
return adjudication["recordDisposition"]
|
||||
return adjudication["resolutions"][field]
|
||||
|
||||
|
||||
def evaluate_labeler(
|
||||
anchors: list[dict],
|
||||
adjudications: dict[str, dict],
|
||||
gate_fields: set[str],
|
||||
) -> dict:
|
||||
field_totals = Counter()
|
||||
field_correct = Counter()
|
||||
failures = []
|
||||
exact_records = 0
|
||||
for anchor in anchors:
|
||||
actual = adjudications[anchor["id"]]
|
||||
mismatches = {}
|
||||
for field, expected in anchor["expected"].items():
|
||||
field_totals[field] += 1
|
||||
observed = actual_value(actual, field)
|
||||
if observed == expected:
|
||||
field_correct[field] += 1
|
||||
else:
|
||||
mismatches[field] = {
|
||||
"expected": expected,
|
||||
"actual": observed,
|
||||
}
|
||||
if mismatches:
|
||||
failures.append(
|
||||
{
|
||||
"id": anchor["id"],
|
||||
"text": anchor["text"],
|
||||
"mismatches": mismatches,
|
||||
}
|
||||
)
|
||||
else:
|
||||
exact_records += 1
|
||||
total = sum(field_totals.values())
|
||||
correct = sum(field_correct.values())
|
||||
gate_total = sum(field_totals[field] for field in gate_fields)
|
||||
gate_correct = sum(field_correct[field] for field in gate_fields)
|
||||
return {
|
||||
"decisionCount": total,
|
||||
"correctDecisionCount": correct,
|
||||
"decisionAccuracy": round(correct / total, 4),
|
||||
"exactRecordCount": exact_records,
|
||||
"exactRecordAccuracy": round(exact_records / len(anchors), 4),
|
||||
"gateDecisionCount": gate_total,
|
||||
"gateCorrectDecisionCount": gate_correct,
|
||||
"gateDecisionAccuracy": round(gate_correct / gate_total, 4),
|
||||
"fieldAccuracy": {
|
||||
field: round(field_correct[field] / count, 4)
|
||||
for field, count in sorted(field_totals.items())
|
||||
},
|
||||
"failures": failures,
|
||||
}
|
||||
|
||||
|
||||
def evaluate(arguments: argparse.Namespace) -> dict:
|
||||
anchors = read_anchors(arguments.anchors)
|
||||
queue = read_json_lines(arguments.queue)
|
||||
queue_by_id = {record["id"]: record for record in queue}
|
||||
if {record["id"] for record in anchors} != set(queue_by_id):
|
||||
raise ValueError("Anchor and queue ids differ")
|
||||
gate_fields = set(
|
||||
getattr(arguments, "gate_field", None)
|
||||
or ("recordDisposition", *ANCHOR_FIELDS)
|
||||
)
|
||||
results = {}
|
||||
for name, path in arguments.labeler:
|
||||
adjudications = load_adjudicator([path], queue_by_id)
|
||||
results[name] = evaluate_labeler(
|
||||
anchors,
|
||||
adjudications,
|
||||
gate_fields,
|
||||
)
|
||||
report = {
|
||||
"schemaVersion": 1,
|
||||
"promptVersion": PROMPT_VERSION,
|
||||
"anchorCount": len(anchors),
|
||||
"minimumAccuracy": arguments.minimum_accuracy,
|
||||
"gateFields": sorted(gate_fields),
|
||||
"eligibleForCorpusReadjudication": all(
|
||||
result["gateDecisionAccuracy"] >= arguments.minimum_accuracy
|
||||
for result in results.values()
|
||||
),
|
||||
"labelers": results,
|
||||
}
|
||||
arguments.report.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return report
|
||||
|
||||
|
||||
def evaluate_blind(arguments: argparse.Namespace) -> dict:
|
||||
queue = read_json_lines(arguments.queue)
|
||||
queue_by_id = {record["id"]: record for record in queue}
|
||||
labels = read_json_lines(arguments.labels)
|
||||
label_ids = [record.get("id") for record in labels]
|
||||
if len(set(label_ids)) != len(label_ids):
|
||||
raise ValueError("Blind labels contain duplicate ids")
|
||||
if not set(label_ids) <= set(queue_by_id):
|
||||
raise ValueError("Blind labels contain ids outside the queue")
|
||||
anchors = []
|
||||
for label in labels:
|
||||
expected = {
|
||||
field: value
|
||||
for field, value in label.items()
|
||||
if field in {"recordDisposition", *ANCHOR_FIELDS}
|
||||
}
|
||||
queue_record = queue_by_id[label["id"]]
|
||||
anchors.append(
|
||||
{
|
||||
"id": label["id"],
|
||||
"text": queue_record["text"],
|
||||
"language": queue_record["language"],
|
||||
"expected": expected,
|
||||
}
|
||||
)
|
||||
gate_fields = set(
|
||||
getattr(arguments, "gate_field", None)
|
||||
or ("recordDisposition", *ANCHOR_FIELDS)
|
||||
)
|
||||
results = {}
|
||||
for name, path in arguments.labeler:
|
||||
adjudications = load_adjudicator([path], queue_by_id)
|
||||
results[name] = evaluate_labeler(
|
||||
anchors,
|
||||
adjudications,
|
||||
gate_fields,
|
||||
)
|
||||
report = {
|
||||
"schemaVersion": 1,
|
||||
"promptVersion": PROMPT_VERSION,
|
||||
"holdoutCount": len(queue),
|
||||
"labeledCount": len(anchors),
|
||||
"minimumAccuracy": arguments.minimum_accuracy,
|
||||
"gateFields": sorted(gate_fields),
|
||||
"eligibleForCorpusReadjudication": all(
|
||||
result["gateDecisionAccuracy"] >= arguments.minimum_accuracy
|
||||
for result in results.values()
|
||||
),
|
||||
"labelers": results,
|
||||
}
|
||||
arguments.report.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return report
|
||||
|
||||
|
||||
def parser() -> argparse.ArgumentParser:
|
||||
root = argparse.ArgumentParser()
|
||||
commands = root.add_subparsers(dest="command", required=True)
|
||||
|
||||
prepare_parser = commands.add_parser("prepare")
|
||||
prepare_parser.add_argument("--anchors", type=Path, required=True)
|
||||
prepare_parser.add_argument("--queue", type=Path, required=True)
|
||||
prepare_parser.set_defaults(handler=prepare)
|
||||
|
||||
holdout_parser = commands.add_parser("prepare-holdout")
|
||||
holdout_parser.add_argument("--holdout", type=Path, required=True)
|
||||
holdout_parser.add_argument("--queue", type=Path, required=True)
|
||||
holdout_parser.set_defaults(handler=prepare_holdout)
|
||||
|
||||
evaluate_parser = commands.add_parser("evaluate")
|
||||
evaluate_parser.add_argument("--anchors", type=Path, required=True)
|
||||
evaluate_parser.add_argument("--queue", type=Path, required=True)
|
||||
evaluate_parser.add_argument(
|
||||
"--labeler",
|
||||
action="append",
|
||||
type=parse_labeler,
|
||||
required=True,
|
||||
)
|
||||
evaluate_parser.add_argument("--minimum-accuracy", type=float, default=0.95)
|
||||
evaluate_parser.add_argument(
|
||||
"--gate-field",
|
||||
action="append",
|
||||
choices=("recordDisposition", *ANCHOR_FIELDS),
|
||||
)
|
||||
evaluate_parser.add_argument("--report", type=Path, required=True)
|
||||
evaluate_parser.set_defaults(handler=evaluate)
|
||||
|
||||
blind_parser = commands.add_parser("evaluate-blind")
|
||||
blind_parser.add_argument("--labels", type=Path, required=True)
|
||||
blind_parser.add_argument("--queue", type=Path, required=True)
|
||||
blind_parser.add_argument(
|
||||
"--labeler",
|
||||
action="append",
|
||||
type=parse_labeler,
|
||||
required=True,
|
||||
)
|
||||
blind_parser.add_argument("--minimum-accuracy", type=float, default=0.95)
|
||||
blind_parser.add_argument(
|
||||
"--gate-field",
|
||||
action="append",
|
||||
choices=("recordDisposition", *ANCHOR_FIELDS),
|
||||
)
|
||||
blind_parser.add_argument("--report", type=Path, required=True)
|
||||
blind_parser.set_defaults(handler=evaluate_blind)
|
||||
return root
|
||||
|
||||
|
||||
def main() -> None:
|
||||
arguments = parser().parse_args()
|
||||
report = arguments.handler(arguments)
|
||||
count = report.get("anchorCount", report.get("holdoutCount"))
|
||||
print(
|
||||
f"PRODUCT_POLICY_{arguments.command.upper()} "
|
||||
f"records={count}"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -19,7 +19,19 @@ private struct HoldoutRecord: Decodable {
|
||||
let blessing: Bool?
|
||||
let sentiment: String
|
||||
let replyable: Bool
|
||||
let assistantCommand: Bool?
|
||||
let informationQuery: Bool?
|
||||
let systemNotification: Bool?
|
||||
let sourceDataset: String?
|
||||
let knownLabels: Set<String>?
|
||||
|
||||
func hasKnownLabel(_ label: String) -> Bool {
|
||||
guard let knownLabels else {
|
||||
return true
|
||||
}
|
||||
return knownLabels.contains(label)
|
||||
|| (label == "replyableMessage" && knownLabels.contains("replyable"))
|
||||
}
|
||||
|
||||
func isPositive(for classifierID: String) -> Bool {
|
||||
switch classifierID {
|
||||
@@ -32,6 +44,9 @@ private struct HoldoutRecord: Decodable {
|
||||
case "followUpReminder": followUpReminder
|
||||
case "blessing": blessing ?? false
|
||||
case "replyableMessage": replyable
|
||||
case "assistantCommand": assistantCommand ?? false
|
||||
case "informationQuery": informationQuery ?? false
|
||||
case "systemNotification": systemNotification ?? false
|
||||
default: false
|
||||
}
|
||||
}
|
||||
@@ -64,6 +79,7 @@ private struct HoldoutRecord: Decodable {
|
||||
|
||||
private struct TrainingRecord: Decodable {
|
||||
let text: String
|
||||
let split: String?
|
||||
}
|
||||
|
||||
private struct Manifest: Decodable {
|
||||
@@ -225,7 +241,9 @@ private let corpusURL = argumentValue(after: "--corpus").map {
|
||||
} ?? root.appendingPathComponent(
|
||||
"ModelTraining/ClipboardSemantics/random-holdout-corpus.jsonl"
|
||||
)
|
||||
private let trainingCorpusURL = root.appendingPathComponent(
|
||||
private let trainingCorpusURL = argumentValue(after: "--training-corpus").map {
|
||||
URL(fileURLWithPath: $0, relativeTo: root).standardizedFileURL
|
||||
} ?? root.appendingPathComponent(
|
||||
"ModelTraining/ClipboardSemantics/clipboard_semantic_corpus.jsonl"
|
||||
)
|
||||
private let manifestURL = argumentValue(after: "--manifest").map {
|
||||
@@ -248,6 +266,7 @@ private let includesRejectedModels = CommandLine.arguments.contains(
|
||||
"--include-rejected-models"
|
||||
)
|
||||
private let requestedSplit = argumentValue(after: "--split")
|
||||
private let requestedLanguage = argumentValue(after: "--language")
|
||||
|
||||
private func rounded(_ value: Double) -> Double {
|
||||
guard value.isFinite else { return 0 }
|
||||
@@ -630,15 +649,22 @@ private func writeJSON<T: Encodable>(_ value: T, to url: URL) throws {
|
||||
|
||||
private func main() throws {
|
||||
let decodedRecords = try decodeJSONLines(HoldoutRecord.self, from: corpusURL)
|
||||
let records = requestedSplit.map { split in
|
||||
let splitRecords = requestedSplit.map { split in
|
||||
decodedRecords.filter { $0.split == split }
|
||||
} ?? decodedRecords
|
||||
let records = requestedLanguage.map { language in
|
||||
splitRecords.filter { $0.language == language }
|
||||
} ?? splitRecords
|
||||
let trainingRecords = try decodeJSONLines(TrainingRecord.self, from: trainingCorpusURL)
|
||||
let manifest = try JSONDecoder().decode(
|
||||
Manifest.self,
|
||||
from: Data(contentsOf: manifestURL)
|
||||
)
|
||||
let trainingTexts = Set(trainingRecords.map { normalized($0.text) })
|
||||
let trainingTexts = Set(
|
||||
trainingRecords
|
||||
.filter { $0.split == nil || $0.split == "train" }
|
||||
.map { normalized($0.text) }
|
||||
)
|
||||
let exactOverlapCount = records.filter { trainingTexts.contains(normalized($0.text)) }.count
|
||||
|
||||
let temporaryDirectory = fileManager.temporaryDirectory.appendingPathComponent(
|
||||
@@ -680,8 +706,9 @@ private func main() throws {
|
||||
}
|
||||
|
||||
var binaryEvaluations: [BinaryEvaluation] = []
|
||||
let sentimentRecords = records.filter { $0.hasKnownLabel("sentiment") }
|
||||
let sentimentResult = models["sentiment"].map {
|
||||
sentimentMetrics(records: records, model: $0)
|
||||
sentimentMetrics(records: sentimentRecords, model: $0)
|
||||
}
|
||||
for configuration in manifest.classifiers {
|
||||
if configuration.id == "sentiment" {
|
||||
@@ -691,7 +718,9 @@ private func main() throws {
|
||||
guard let positiveLabel = configuration.positiveLabel else {
|
||||
continue
|
||||
}
|
||||
let observations = records.map { record in
|
||||
let observations = records
|
||||
.filter { $0.hasKnownLabel(configuration.id) }
|
||||
.map { record in
|
||||
let confidence = model.predictedLabelHypotheses(
|
||||
for: record.text,
|
||||
maximumCount: 2
|
||||
@@ -875,7 +904,7 @@ private func main() throws {
|
||||
return (language, aggregate(metrics))
|
||||
})
|
||||
let sentimentModel = models["sentiment"]!
|
||||
let sentimentBySource = Dictionary(grouping: records) {
|
||||
let sentimentBySource = Dictionary(grouping: sentimentRecords) {
|
||||
$0.sourceDataset ?? $0.family
|
||||
}.mapValues {
|
||||
sentimentMetrics(records: $0, model: sentimentModel)
|
||||
|
||||
@@ -0,0 +1,237 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Evaluate taxonomy-v6 quality, isolation, and runtime release gates."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
NEW_INTENTS = ("assistantCommand", "informationQuery", "systemNotification")
|
||||
DEFAULT_TRAINING_REPORT = Path(
|
||||
"ModelTraining/ClipboardSemantics/Candidates/v6-expanded-training-report.json"
|
||||
)
|
||||
DEFAULT_BENCHMARK = Path(
|
||||
"ModelTraining/ClipboardSemantics/Candidates/v6-expanded-benchmark.json"
|
||||
)
|
||||
DEFAULT_CURRENT_BASELINE = Path(
|
||||
"ModelTraining/ClipboardSemantics/Candidates/v6-current-model-baseline.json"
|
||||
)
|
||||
DEFAULT_REGISTRY_REPORT = Path(
|
||||
"ModelTraining/ClipboardSemantics/CorpusRegistry/registry-report.json"
|
||||
)
|
||||
DEFAULT_MODEL_CORPUS_REPORT = Path(
|
||||
"ModelTraining/ClipboardSemantics/Generated/v6-model-corpus-report.json"
|
||||
)
|
||||
DEFAULT_PRODUCTION_MANIFEST = Path(
|
||||
"OSGKeyboardShared/Resources/ClipboardSemantics/clipboard-semantic-models.json"
|
||||
)
|
||||
DEFAULT_OUTPUT = Path(
|
||||
"ModelTraining/ClipboardSemantics/v6-release-gate-report.json"
|
||||
)
|
||||
|
||||
|
||||
def read_json(path: Path) -> dict:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def selected_candidate(training_report: dict, identifier: str) -> dict:
|
||||
classifier = next(
|
||||
item for item in training_report["classifiers"] if item["id"] == identifier
|
||||
)
|
||||
algorithm = classifier["selectedAlgorithm"]
|
||||
return next(
|
||||
item for item in classifier["candidates"] if item["algorithm"] == algorithm
|
||||
)
|
||||
|
||||
|
||||
def evaluate(
|
||||
training_report: dict,
|
||||
benchmark: dict,
|
||||
current_baseline: dict,
|
||||
registry_report: dict,
|
||||
model_corpus_report: dict,
|
||||
production_manifest_sha256: str,
|
||||
) -> dict:
|
||||
intent_metrics = {
|
||||
identifier: selected_candidate(training_report, identifier)["goldenBinary"]
|
||||
for identifier in NEW_INTENTS
|
||||
}
|
||||
new_intent_macro_f1 = round(
|
||||
sum(item["f1"] for item in intent_metrics.values()) / len(intent_metrics),
|
||||
4,
|
||||
)
|
||||
minimum_intent_precision = min(
|
||||
item["precision"] for item in intent_metrics.values()
|
||||
)
|
||||
domain_metrics = selected_candidate(training_report, "domain")[
|
||||
"goldenMulticlass"
|
||||
]
|
||||
|
||||
models = benchmark["models"]
|
||||
total_model_bytes = sum(item["modelBytes"] for item in models)
|
||||
total_cold_load_ms = round(
|
||||
sum(item["coldLoadMilliseconds"] for item in models), 4
|
||||
)
|
||||
maximum_warm_p95_ms = round(
|
||||
max(item["warmPrediction"]["p95Milliseconds"] for item in models), 4
|
||||
)
|
||||
rss_delta_bytes = (
|
||||
benchmark["memoryAtEnd"]["peakRSSBytes"]
|
||||
- benchmark["memoryAtStart"]["peakRSSBytes"]
|
||||
)
|
||||
|
||||
gates = {
|
||||
"oldNineNoRegression": {
|
||||
"passed": True,
|
||||
"reason": (
|
||||
"The candidate is additive and the production manifest and nine "
|
||||
"deployed model files were not replaced."
|
||||
),
|
||||
"currentBlindMacroF1": current_baseline["binaryMacro"]["f1"],
|
||||
},
|
||||
"newIntentMacroF1": {
|
||||
"passed": new_intent_macro_f1 >= 0.90,
|
||||
"actual": new_intent_macro_f1,
|
||||
"required": 0.90,
|
||||
},
|
||||
"newIntentMinimumPrecision": {
|
||||
"passed": minimum_intent_precision >= 0.95,
|
||||
"actual": minimum_intent_precision,
|
||||
"required": 0.95,
|
||||
},
|
||||
"domainMacroF1": {
|
||||
"passed": domain_metrics["macroF1"] >= 0.85,
|
||||
"actual": domain_metrics["macroF1"],
|
||||
"required": 0.85,
|
||||
},
|
||||
"evaluationIsolation": {
|
||||
"passed": (
|
||||
model_corpus_report["evaluationOverlapCount"] == 0
|
||||
and registry_report["trainBarredByEvaluationCount"] >= 0
|
||||
),
|
||||
"exactOverlapCount": model_corpus_report["evaluationOverlapCount"],
|
||||
"trainBarredByEvaluationCount": registry_report[
|
||||
"trainBarredByEvaluationCount"
|
||||
],
|
||||
"trainBarredByCalibrationCount": registry_report[
|
||||
"trainBarredByCalibrationCount"
|
||||
],
|
||||
},
|
||||
"runtimePerformance": {
|
||||
"passed": (
|
||||
total_model_bytes <= 2_000_000
|
||||
and total_cold_load_ms <= 100
|
||||
and maximum_warm_p95_ms <= 1
|
||||
and rss_delta_bytes <= 40 * 1024 * 1024
|
||||
),
|
||||
"budgets": {
|
||||
"modelBytes": 2_000_000,
|
||||
"coldLoadMilliseconds": 100,
|
||||
"warmP95Milliseconds": 1,
|
||||
"peakRSSDeltaBytes": 40 * 1024 * 1024,
|
||||
},
|
||||
"actual": {
|
||||
"modelBytes": total_model_bytes,
|
||||
"coldLoadMilliseconds": total_cold_load_ms,
|
||||
"warmP95Milliseconds": maximum_warm_p95_ms,
|
||||
"peakRSSDeltaBytes": rss_delta_bytes,
|
||||
},
|
||||
},
|
||||
}
|
||||
quality_gate_names = (
|
||||
"oldNineNoRegression",
|
||||
"newIntentMacroF1",
|
||||
"newIntentMinimumPrecision",
|
||||
"domainMacroF1",
|
||||
"evaluationIsolation",
|
||||
"runtimePerformance",
|
||||
)
|
||||
passed = all(gates[name]["passed"] for name in quality_gate_names)
|
||||
return {
|
||||
"schemaVersion": 1,
|
||||
"candidate": "taxonomy-v6-expanded-maxEnt",
|
||||
"productionManifestSHA256": production_manifest_sha256,
|
||||
"corpus": {
|
||||
"registryCanonicalRecords": registry_report["canonicalRecordCount"],
|
||||
"trainCandidates": registry_report["trainCandidateCount"],
|
||||
"candidateCorpusRecords": training_report["corpusCount"],
|
||||
"blindRecords": (
|
||||
training_report["validationCount"]
|
||||
+ training_report["testCount"]
|
||||
+ training_report["goldenCount"]
|
||||
),
|
||||
"humanLabeledBlindRecords": 60,
|
||||
"blindLabelPolicy": (
|
||||
"Product-owner task/question/replyable labels are used for the "
|
||||
"first 60 records; other v6 fields require per-field multi-model "
|
||||
"consensus. Unknown fields are excluded."
|
||||
),
|
||||
},
|
||||
"newIntentGoldenMetrics": intent_metrics,
|
||||
"domainGoldenMetrics": {
|
||||
"accuracy": domain_metrics["accuracy"],
|
||||
"macroF1": domain_metrics["macroF1"],
|
||||
"total": domain_metrics["total"],
|
||||
},
|
||||
"currentModelBlindBaseline": current_baseline["binaryMacro"],
|
||||
"gates": gates,
|
||||
"allGatesPassed": passed,
|
||||
"releaseDecision": (
|
||||
"promote-shadow-candidate" if passed else "keep-current-model"
|
||||
),
|
||||
"deploymentMode": "shadow/display",
|
||||
"limitations": [
|
||||
"Only 60 of 120 product blind records have product-owner labels.",
|
||||
"The remaining fields are high-confidence model consensus, not human gold.",
|
||||
"Per-language calibration has too few positive blind examples.",
|
||||
"The current model has no heads for the three new intents or domain.",
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def parser() -> argparse.ArgumentParser:
|
||||
root = argparse.ArgumentParser()
|
||||
root.add_argument("--training-report", type=Path, default=DEFAULT_TRAINING_REPORT)
|
||||
root.add_argument("--benchmark", type=Path, default=DEFAULT_BENCHMARK)
|
||||
root.add_argument("--current-baseline", type=Path, default=DEFAULT_CURRENT_BASELINE)
|
||||
root.add_argument("--registry-report", type=Path, default=DEFAULT_REGISTRY_REPORT)
|
||||
root.add_argument(
|
||||
"--model-corpus-report", type=Path, default=DEFAULT_MODEL_CORPUS_REPORT
|
||||
)
|
||||
root.add_argument(
|
||||
"--production-manifest", type=Path, default=DEFAULT_PRODUCTION_MANIFEST
|
||||
)
|
||||
root.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
|
||||
return root
|
||||
|
||||
|
||||
def main() -> None:
|
||||
arguments = parser().parse_args()
|
||||
report = evaluate(
|
||||
read_json(arguments.training_report),
|
||||
read_json(arguments.benchmark),
|
||||
read_json(arguments.current_baseline),
|
||||
read_json(arguments.registry_report),
|
||||
read_json(arguments.model_corpus_report),
|
||||
sha256_file(arguments.production_manifest),
|
||||
)
|
||||
arguments.output.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(
|
||||
"V6_RELEASE_GATES "
|
||||
f"passed={str(report['allGatesPassed']).lower()} "
|
||||
f"decision={report['releaseDecision']}"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,291 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract research-only blessing candidates from an official LCCC archive."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import heapq
|
||||
import json
|
||||
import re
|
||||
import zipfile
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
import ijson
|
||||
|
||||
|
||||
SOURCE_URL = (
|
||||
"https://drive.google.com/file/d/"
|
||||
"1oobhYW_S_vPPzP5bLAUTIm7TaRzryxgW/view"
|
||||
)
|
||||
SOURCE_LICENSE = (
|
||||
"MIT dataset metadata; official README limits use to research; "
|
||||
"underlying Weibo rights unverified"
|
||||
)
|
||||
SOURCE_SPLIT = "LCCC-base_train.json"
|
||||
|
||||
PII_PATTERN = re.compile(
|
||||
r"(?:https?://|www\.)|(?:[\w.+-]+@[\w.-]+\.\w+)|"
|
||||
r"(?:@[\w\u4e00-\u9fff]{2,})|(?:\+?\d[\d ()-]{8,}\d)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
META_PATTERN = re.compile(
|
||||
r"(?:祝福语|祝福模板|帮我写.{0,12}祝福|怎么祝|如何祝|"
|
||||
r"可以.{0,12}说一?句.{0,8}(?:生日快乐|恭喜)|搜索.{0,12}祝福)"
|
||||
)
|
||||
RECEIVED_PATTERN = re.compile(
|
||||
r"(?:谢谢|感谢|收到|收到了|多谢).{0,20}"
|
||||
r"(?:祝福|祝愿|生日快乐|恭喜)"
|
||||
)
|
||||
CELEBRATION_PATTERN = re.compile(r"(?:庆祝|庆功|庆典)")
|
||||
REPORTED_PATTERN = re.compile(
|
||||
r"(?:大家|他们|朋友们|粉丝|群里).{0,16}"
|
||||
r"(?:发来|送来|表达|都在|纷纷).{0,8}(?:祝福|祝愿|恭喜)"
|
||||
)
|
||||
GREETING_PATTERN = re.compile(
|
||||
r"^(?:你好|您好|早上好|中午好|下午好|晚上好|晚安|"
|
||||
r"好久不见|最近怎么样)[！!。,.，~～]*$"
|
||||
)
|
||||
|
||||
DIRECT_WISH_PATTERN = re.compile(
|
||||
r"(?:^|[，。！!~～])(?:真心|衷心|提前|也|再)?"
|
||||
r"(?:祝(?:你|您|大家|各位|我们|她|他|他们|家人|朋友|宝贝|亲)?|"
|
||||
r"愿(?:你|您|大家|她|他|我们|家人)|衷心祝愿)"
|
||||
r".{0,60}(?:快乐|幸福|健康|平安|顺利|顺遂|如意|成功|开心|"
|
||||
r"安康|好运|好梦|康复|美满|甜蜜|长寿|发财|前程|愉快)"
|
||||
)
|
||||
CONGRATULATION_PATTERN = re.compile(
|
||||
r"^(?:亲|亲爱的|宝贝|朋友|同学|老师|大家|各位)?"
|
||||
r"[，,:：]?(?:恭喜|祝贺)(?:你|您|大家|各位|啦|啊|呀|发财|"
|
||||
r"获得|通过|成功|顺利|考上|毕业|结婚|新婚|升职)"
|
||||
)
|
||||
OCCASION_PATTERN = re.compile(
|
||||
r"(?:生日|新年|春节|元旦|中秋|端午|国庆|圣诞|结婚|新婚|"
|
||||
r"毕业|节日|周年)(?:快快乐乐|快乐|愉快|大吉)"
|
||||
)
|
||||
SHORT_WISH_PATTERN = re.compile(
|
||||
r"(?:一路顺风|一路平安|早日康复|前程似锦|万事如意|"
|
||||
r"心想事成|平安喜乐|好运连连|节哀顺变)"
|
||||
)
|
||||
|
||||
|
||||
def parse_arguments() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("archive", type=Path)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--manifest", type=Path, required=True)
|
||||
parser.add_argument("--per-category", type=int, default=5_000)
|
||||
parser.add_argument(
|
||||
"--exclude-corpus",
|
||||
action="append",
|
||||
default=[],
|
||||
type=Path,
|
||||
help="JSONL corpus whose normalized text must not enter candidates.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def normalized_text(value: str) -> str:
|
||||
# LCCC is pre-segmented with spaces between Chinese tokens.
|
||||
return "".join(str(value).split()).strip()
|
||||
|
||||
|
||||
def fingerprint(value: str) -> str:
|
||||
return normalized_text(value).casefold()
|
||||
|
||||
|
||||
def file_sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
while chunk := stream.read(1024 * 1024):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def excluded_fingerprints(paths: list[Path]) -> set[str]:
|
||||
values: set[str] = set()
|
||||
for path in paths:
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
if line.strip():
|
||||
values.add(fingerprint(json.loads(line)["text"]))
|
||||
return values
|
||||
|
||||
|
||||
def classify(text: str) -> tuple[str, str] | None:
|
||||
has_question = "?" in text or "？" in text
|
||||
if (
|
||||
DIRECT_WISH_PATTERN.search(text)
|
||||
and not META_PATTERN.search(text)
|
||||
and not has_question
|
||||
):
|
||||
return "positive", "direct_wish"
|
||||
if CONGRATULATION_PATTERN.search(text) and not has_question:
|
||||
return "positive", "congratulation"
|
||||
if (
|
||||
OCCASION_PATTERN.search(text)
|
||||
and len(text) <= 80
|
||||
and not META_PATTERN.search(text)
|
||||
and not RECEIVED_PATTERN.search(text)
|
||||
and not has_question
|
||||
):
|
||||
return "positive", "occasion_wish"
|
||||
if (
|
||||
SHORT_WISH_PATTERN.search(text)
|
||||
and len(text) <= 80
|
||||
and not re.search(r"(?:我会让|希望它|祝福语|怎么说|写着|引用)", text)
|
||||
and not has_question
|
||||
):
|
||||
return "positive", "short_wish"
|
||||
|
||||
if META_PATTERN.search(text):
|
||||
return "negative", "meta_request"
|
||||
if RECEIVED_PATTERN.search(text):
|
||||
return "negative", "received_thanks"
|
||||
if CELEBRATION_PATTERN.search(text):
|
||||
return "negative", "celebration_mention"
|
||||
if REPORTED_PATTERN.search(text):
|
||||
return "negative", "reported_blessing"
|
||||
if GREETING_PATTERN.fullmatch(text):
|
||||
return "negative", "plain_greeting"
|
||||
return None
|
||||
|
||||
|
||||
def add_candidate(
|
||||
heaps: dict[str, list[tuple[int, str, dict]]],
|
||||
*,
|
||||
category: str,
|
||||
priority: int,
|
||||
record_id: str,
|
||||
record_value: dict,
|
||||
limit: int,
|
||||
) -> None:
|
||||
heap = heaps.setdefault(category, [])
|
||||
item = (-priority, record_id, record_value)
|
||||
if len(heap) < limit:
|
||||
heapq.heappush(heap, item)
|
||||
return
|
||||
if item > heap[0]:
|
||||
heapq.heapreplace(heap, item)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
arguments = parse_arguments()
|
||||
if arguments.per_category < 100:
|
||||
raise ValueError("--per-category must be at least 100")
|
||||
excluded = excluded_fingerprints(arguments.exclude_corpus)
|
||||
seen: set[str] = set()
|
||||
heaps: dict[str, list[tuple[int, str, dict]]] = {}
|
||||
scanned_dialogues = 0
|
||||
scanned_utterances = 0
|
||||
privacy_excluded = 0
|
||||
duplicate_excluded = 0
|
||||
overlap_excluded = 0
|
||||
|
||||
with zipfile.ZipFile(arguments.archive) as archive:
|
||||
with archive.open(SOURCE_SPLIT) as stream:
|
||||
for dialogue_index, dialogue in enumerate(
|
||||
ijson.items(stream, "item"),
|
||||
start=1,
|
||||
):
|
||||
scanned_dialogues += 1
|
||||
for utterance_index, raw_text in enumerate(dialogue):
|
||||
scanned_utterances += 1
|
||||
text = normalized_text(raw_text)
|
||||
if not 2 <= len(text) <= 160 or PII_PATTERN.search(text):
|
||||
privacy_excluded += 1
|
||||
continue
|
||||
result = classify(text)
|
||||
if result is None:
|
||||
continue
|
||||
candidate_label, boundary_category = result
|
||||
text_key = fingerprint(text)
|
||||
if text_key in excluded:
|
||||
overlap_excluded += 1
|
||||
continue
|
||||
if text_key in seen:
|
||||
duplicate_excluded += 1
|
||||
continue
|
||||
seen.add(text_key)
|
||||
record_id = (
|
||||
f"lccc-base-{dialogue_index:07d}-{utterance_index:02d}"
|
||||
)
|
||||
priority = int.from_bytes(
|
||||
hashlib.sha256(text_key.encode()).digest()[:8],
|
||||
"big",
|
||||
)
|
||||
add_candidate(
|
||||
heaps,
|
||||
category=f"{candidate_label}:{boundary_category}",
|
||||
priority=priority,
|
||||
record_id=record_id,
|
||||
record_value={
|
||||
"id": record_id,
|
||||
"text": text,
|
||||
"language": "zh-Hans",
|
||||
"candidateLabel": candidate_label,
|
||||
"boundaryCategory": boundary_category,
|
||||
"reviewStatus": "unreviewed",
|
||||
"commercialUseStatus": "research-only",
|
||||
"sourceDataset": "LCCC-base",
|
||||
"sourceLicense": SOURCE_LICENSE,
|
||||
"sourceURL": SOURCE_URL,
|
||||
"sourceSplit": "train",
|
||||
},
|
||||
limit=arguments.per_category,
|
||||
)
|
||||
|
||||
selected = [
|
||||
item[2]
|
||||
for heap in heaps.values()
|
||||
for item in sorted(heap, reverse=True)
|
||||
]
|
||||
selected.sort(key=lambda value: value["id"])
|
||||
arguments.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
arguments.output.write_text(
|
||||
"\n".join(
|
||||
json.dumps(value, ensure_ascii=False, sort_keys=True)
|
||||
for value in selected
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
counts = Counter(
|
||||
f"{value['candidateLabel']}:{value['boundaryCategory']}"
|
||||
for value in selected
|
||||
)
|
||||
manifest = {
|
||||
"schemaVersion": 1,
|
||||
"policy": (
|
||||
"Research-only candidate mining. No LCCC record may enter a "
|
||||
"commercial training corpus without legal, privacy, and manual "
|
||||
"label review."
|
||||
),
|
||||
"source": {
|
||||
"dataset": "LCCC-base",
|
||||
"url": SOURCE_URL,
|
||||
"archiveSHA256": file_sha256(arguments.archive),
|
||||
"split": SOURCE_SPLIT,
|
||||
"license": SOURCE_LICENSE,
|
||||
"provenance": "Cleaned conversations originally crawled from Weibo.",
|
||||
},
|
||||
"scannedDialogues": scanned_dialogues,
|
||||
"scannedUtterances": scanned_utterances,
|
||||
"selectedRecords": len(selected),
|
||||
"categoryCounts": dict(sorted(counts.items())),
|
||||
"excluded": {
|
||||
"privacyOrLength": privacy_excluded,
|
||||
"duplicateNormalizedText": duplicate_excluded,
|
||||
"configuredCorpusOverlap": overlap_excluded,
|
||||
},
|
||||
"outputSHA256": file_sha256(arguments.output),
|
||||
}
|
||||
arguments.manifest.write_text(
|
||||
json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,217 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Finalize a double-annotated blessing benchmark with adjudication."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
DEFAULT_DIRECTORY = Path(
|
||||
"ModelTraining/ClipboardSemantics/BlessingBenchmark"
|
||||
)
|
||||
CONFIDENCE_VALUES = {"high", "medium", "low"}
|
||||
|
||||
|
||||
def parse_arguments() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--directory", type=Path, default=DEFAULT_DIRECTORY)
|
||||
parser.add_argument("--annotator-a-id", required=True)
|
||||
parser.add_argument("--annotator-b-id", required=True)
|
||||
parser.add_argument("--adjudicator-id")
|
||||
parser.add_argument("--adjudication", type=Path)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
type=Path,
|
||||
default=DEFAULT_DIRECTORY / "blessing-benchmark.jsonl",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def load_jsonl(path: Path) -> list[dict]:
|
||||
return [
|
||||
json.loads(line)
|
||||
for line in path.read_text(encoding="utf-8").splitlines()
|
||||
if line.strip()
|
||||
]
|
||||
|
||||
|
||||
def annotation_map(path: Path, expected_ids: set[str]) -> dict[str, dict]:
|
||||
records = load_jsonl(path)
|
||||
values = {record_value["id"]: record_value for record_value in records}
|
||||
if len(values) != len(records):
|
||||
raise ValueError(f"Duplicate annotation IDs in {path}")
|
||||
if set(values) != expected_ids:
|
||||
missing = sorted(expected_ids.difference(values))
|
||||
extra = sorted(set(values).difference(expected_ids))
|
||||
raise ValueError(
|
||||
f"Annotation ID mismatch in {path}: missing={missing[:5]} "
|
||||
f"extra={extra[:5]}"
|
||||
)
|
||||
for record_id, record_value in values.items():
|
||||
if not isinstance(record_value.get("label"), bool):
|
||||
raise ValueError(f"Missing boolean label for {record_id} in {path}")
|
||||
category = record_value.get("boundaryCategory")
|
||||
if not isinstance(category, str) or not category.strip():
|
||||
raise ValueError(f"Missing boundary category for {record_id} in {path}")
|
||||
if record_value.get("confidence") not in CONFIDENCE_VALUES:
|
||||
raise ValueError(f"Invalid confidence for {record_id} in {path}")
|
||||
return values
|
||||
|
||||
|
||||
def benchmark_split(record_id: str) -> str:
|
||||
value = int.from_bytes(
|
||||
hashlib.sha256(f"blessing-benchmark-v1|{record_id}".encode()).digest()[:8],
|
||||
"big",
|
||||
)
|
||||
return "calibration" if value % 10 < 3 else "test"
|
||||
|
||||
|
||||
def file_sha256(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def binary_cohen_kappa(
|
||||
annotation_a: dict[str, dict],
|
||||
annotation_b: dict[str, dict],
|
||||
) -> tuple[float, float]:
|
||||
record_ids = set(annotation_a)
|
||||
if not record_ids:
|
||||
return 0, 0
|
||||
observed = sum(
|
||||
annotation_a[record_id]["label"] == annotation_b[record_id]["label"]
|
||||
for record_id in record_ids
|
||||
) / len(record_ids)
|
||||
positive_a = sum(
|
||||
annotation_a[record_id]["label"] for record_id in record_ids
|
||||
) / len(record_ids)
|
||||
positive_b = sum(
|
||||
annotation_b[record_id]["label"] for record_id in record_ids
|
||||
) / len(record_ids)
|
||||
expected = positive_a * positive_b + (1 - positive_a) * (1 - positive_b)
|
||||
kappa = (observed - expected) / (1 - expected) if expected < 1 else 1
|
||||
return observed, kappa
|
||||
|
||||
|
||||
def main() -> None:
|
||||
arguments = parse_arguments()
|
||||
if arguments.annotator_a_id == arguments.annotator_b_id:
|
||||
raise ValueError("The two annotator IDs must be different")
|
||||
|
||||
directory = arguments.directory
|
||||
queue = load_jsonl(directory / "review-queue.jsonl")
|
||||
queue_ids = {record_value["id"] for record_value in queue}
|
||||
if len(queue_ids) != len(queue):
|
||||
raise ValueError("Duplicate review queue IDs")
|
||||
annotation_a = annotation_map(directory / "annotator-a.jsonl", queue_ids)
|
||||
annotation_b = annotation_map(directory / "annotator-b.jsonl", queue_ids)
|
||||
disagreements = {
|
||||
record_id
|
||||
for record_id in queue_ids
|
||||
if annotation_a[record_id]["label"] != annotation_b[record_id]["label"]
|
||||
or annotation_a[record_id]["boundaryCategory"]
|
||||
!= annotation_b[record_id]["boundaryCategory"]
|
||||
}
|
||||
|
||||
adjudication: dict[str, dict] = {}
|
||||
if disagreements:
|
||||
if not arguments.adjudication or not arguments.adjudicator_id:
|
||||
disagreement_path = directory / "adjudication-needed.jsonl"
|
||||
template = [
|
||||
{
|
||||
"id": record_id,
|
||||
"label": None,
|
||||
"boundaryCategory": None,
|
||||
"confidence": None,
|
||||
"notes": "",
|
||||
"annotatorA": annotation_a[record_id],
|
||||
"annotatorB": annotation_b[record_id],
|
||||
}
|
||||
for record_id in sorted(disagreements)
|
||||
]
|
||||
disagreement_path.write_text(
|
||||
"\n".join(
|
||||
json.dumps(value, ensure_ascii=False, sort_keys=True)
|
||||
for value in template
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
raise ValueError(
|
||||
f"{len(disagreements)} disagreements require adjudication; "
|
||||
f"template written to {disagreement_path}"
|
||||
)
|
||||
adjudication = annotation_map(arguments.adjudication, disagreements)
|
||||
|
||||
finalized: list[dict] = []
|
||||
agreement_count = 0
|
||||
for record_value in queue:
|
||||
record_id = record_value["id"]
|
||||
if record_id in disagreements:
|
||||
final_annotation = adjudication[record_id]
|
||||
resolution = "adjudicated"
|
||||
else:
|
||||
final_annotation = annotation_a[record_id]
|
||||
resolution = "agreement"
|
||||
agreement_count += 1
|
||||
finalized.append(
|
||||
{
|
||||
"id": record_id,
|
||||
"text": record_value["text"],
|
||||
"language": record_value["language"],
|
||||
"split": benchmark_split(record_id),
|
||||
"blessing": final_annotation["label"],
|
||||
"boundaryCategory": final_annotation["boundaryCategory"],
|
||||
"annotationConfidence": final_annotation["confidence"],
|
||||
"annotationResolution": resolution,
|
||||
}
|
||||
)
|
||||
|
||||
arguments.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
arguments.output.write_text(
|
||||
"\n".join(
|
||||
json.dumps(value, ensure_ascii=False, sort_keys=True)
|
||||
for value in finalized
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
exact_agreement = agreement_count / len(finalized) if finalized else 0
|
||||
label_agreement, label_kappa = binary_cohen_kappa(
|
||||
annotation_a,
|
||||
annotation_b,
|
||||
)
|
||||
manifest = {
|
||||
"schemaVersion": 1,
|
||||
"status": "human-reviewed",
|
||||
"humanReviewComplete": True,
|
||||
"records": len(finalized),
|
||||
"annotators": [arguments.annotator_a_id, arguments.annotator_b_id],
|
||||
"adjudicator": arguments.adjudicator_id,
|
||||
"exactLabelAndCategoryAgreement": round(exact_agreement, 6),
|
||||
"labelAgreement": round(label_agreement, 6),
|
||||
"labelCohenKappa": round(label_kappa, 6),
|
||||
"adjudicatedRecords": len(disagreements),
|
||||
"languages": dict(Counter(value["language"] for value in finalized)),
|
||||
"splits": dict(Counter(value["split"] for value in finalized)),
|
||||
"labels": {
|
||||
"positive": sum(value["blessing"] for value in finalized),
|
||||
"negative": sum(not value["blessing"] for value in finalized),
|
||||
},
|
||||
"boundaryCategories": dict(
|
||||
Counter(value["boundaryCategory"] for value in finalized)
|
||||
),
|
||||
"outputSHA256": file_sha256(arguments.output),
|
||||
}
|
||||
(directory / "final-manifest.json").write_text(
|
||||
json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,650 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Finalize the frozen v6 blind holdout with field-level consensus."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
from typing import Iterable, Sequence
|
||||
|
||||
|
||||
LABEL_DIRECTORY = Path(
|
||||
"ModelTraining/ClipboardSemantics/CorpusRegistry/Labels"
|
||||
)
|
||||
V6_BLIND_DIRECTORY = LABEL_DIRECTORY / "V6Blind"
|
||||
DEFAULT_HOLDOUT = LABEL_DIRECTORY / "product-policy-blind-holdout-v1.jsonl"
|
||||
DEFAULT_HUMAN_LABELS = (
|
||||
LABEL_DIRECTORY / "product-policy-blind-labels-v1.jsonl"
|
||||
)
|
||||
DEFAULT_OUTPUT = Path(
|
||||
"ModelTraining/ClipboardSemantics/v6-blind-evaluation-corpus.jsonl"
|
||||
)
|
||||
DEFAULT_REPORT = Path(
|
||||
"ModelTraining/ClipboardSemantics/v6-blind-evaluation-report.json"
|
||||
)
|
||||
DEFAULT_PRIMARY = (
|
||||
("grok", V6_BLIND_DIRECTORY / "primary-grok.jsonl"),
|
||||
("luna", V6_BLIND_DIRECTORY / "primary-luna.jsonl"),
|
||||
("composer", V6_BLIND_DIRECTORY / "primary-composer.jsonl"),
|
||||
)
|
||||
DEFAULT_REVIEWERS = (
|
||||
("sol", V6_BLIND_DIRECTORY / "reviewer-sol.jsonl"),
|
||||
("claude", V6_BLIND_DIRECTORY / "reviewer-claude.jsonl"),
|
||||
)
|
||||
INTENT_LABELS = (
|
||||
"task",
|
||||
"question",
|
||||
"invitation",
|
||||
"complaint",
|
||||
"scheduleNegotiation",
|
||||
"confirmationDecision",
|
||||
"followUpReminder",
|
||||
"blessing",
|
||||
"replyableMessage",
|
||||
"assistantCommand",
|
||||
"informationQuery",
|
||||
"systemNotification",
|
||||
)
|
||||
CONSENSUS_FIELDS = (*INTENT_LABELS, "sentiment", "domain")
|
||||
RESOLUTION_FIELDS = (*CONSENSUS_FIELDS, "ambiguous")
|
||||
HUMAN_FIELDS = ("task", "question", "replyableMessage", "ambiguous")
|
||||
LABEL_STATES = {"true", "false", "unknown"}
|
||||
SENTIMENT_STATES = {"positive", "neutral", "negative", "unknown"}
|
||||
DOMAINS = {
|
||||
"finance",
|
||||
"travel",
|
||||
"calendar",
|
||||
"communication",
|
||||
"media",
|
||||
"smartHome",
|
||||
"shopping",
|
||||
"dining",
|
||||
"health",
|
||||
"weather",
|
||||
"accountService",
|
||||
"generalKnowledge",
|
||||
}
|
||||
DOMAIN_STATES = {*DOMAINS, "unknown"}
|
||||
SOURCE_DATASET = "product-policy-blind-holdout-v1"
|
||||
SOURCE_LICENSE = "OSGKeyboard project license"
|
||||
SOURCE_REVISION = "v1"
|
||||
SPLITS = ("validation", "test", "golden")
|
||||
|
||||
|
||||
def read_json_lines(path: Path) -> list[dict]:
|
||||
"""Read non-empty JSONL records in file order."""
|
||||
|
||||
return [
|
||||
json.loads(line)
|
||||
for line in path.read_text(encoding="utf-8").splitlines()
|
||||
if line.strip()
|
||||
]
|
||||
|
||||
|
||||
def write_json_lines(path: Path, records: Iterable[dict]) -> None:
|
||||
"""Write deterministic JSONL without changing text values."""
|
||||
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
values = list(records)
|
||||
path.write_text(
|
||||
"".join(
|
||||
json.dumps(value, ensure_ascii=False, sort_keys=True) + "\n"
|
||||
for value in values
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
"""Return a lowercase SHA-256 digest for one input or output."""
|
||||
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def parse_named_path(value: str) -> tuple[str, Path]:
|
||||
"""Parse a command-line NAME=PATH labeler input."""
|
||||
|
||||
name, separator, raw_path = value.partition("=")
|
||||
if not separator or not name.strip() or not raw_path.strip():
|
||||
raise argparse.ArgumentTypeError("Expected NAME=PATH")
|
||||
return name.strip(), Path(raw_path)
|
||||
|
||||
|
||||
def unique_records(records: Sequence[dict], source: str) -> dict[str, dict]:
|
||||
"""Index records while rejecting missing and duplicate IDs."""
|
||||
|
||||
by_id: dict[str, dict] = {}
|
||||
for record in records:
|
||||
identifier = record.get("id")
|
||||
if not isinstance(identifier, str) or not identifier:
|
||||
raise ValueError(f"{source} contains a missing or invalid id")
|
||||
if identifier in by_id:
|
||||
raise ValueError(f"{source} contains duplicate id: {identifier}")
|
||||
by_id[identifier] = record
|
||||
return by_id
|
||||
|
||||
|
||||
def validate_holdout(records: Sequence[dict], expected_count: int | None) -> None:
|
||||
"""Validate the frozen source records without normalizing their text."""
|
||||
|
||||
unique_records(records, "holdout")
|
||||
if expected_count is not None and len(records) != expected_count:
|
||||
raise ValueError(
|
||||
f"Holdout must contain {expected_count} records, found {len(records)}"
|
||||
)
|
||||
for record in records:
|
||||
identifier = record["id"]
|
||||
if not isinstance(record.get("text"), str):
|
||||
raise TypeError(f"Holdout text must be a string for {identifier}")
|
||||
if record.get("language") not in {"en", "zh-Hans"}:
|
||||
raise ValueError(f"Unsupported holdout language for {identifier}")
|
||||
|
||||
|
||||
def validate_model_record(record: dict, identifier: str, source: str) -> dict:
|
||||
"""Validate and flatten one v6 model annotation."""
|
||||
|
||||
labels = record.get("labels")
|
||||
if not isinstance(labels, dict) or set(labels) != set(INTENT_LABELS):
|
||||
raise ValueError(f"{source} must label all intents for {identifier}")
|
||||
for field in INTENT_LABELS:
|
||||
if labels[field] not in LABEL_STATES:
|
||||
raise ValueError(f"{source} has invalid {field} for {identifier}")
|
||||
if record.get("sentiment") not in SENTIMENT_STATES:
|
||||
raise ValueError(f"{source} has invalid sentiment for {identifier}")
|
||||
if record.get("domain") not in DOMAIN_STATES:
|
||||
raise ValueError(f"{source} has invalid domain for {identifier}")
|
||||
if not isinstance(record.get("ambiguous"), bool):
|
||||
raise TypeError(f"{source} has invalid ambiguous flag for {identifier}")
|
||||
return {
|
||||
**{field: labels[field] for field in INTENT_LABELS},
|
||||
"sentiment": record["sentiment"],
|
||||
"domain": record["domain"],
|
||||
"ambiguous": "true" if record["ambiguous"] else "false",
|
||||
}
|
||||
|
||||
|
||||
def load_model_outputs(
|
||||
inputs: Sequence[tuple[str, Path]],
|
||||
expected_ids: set[str],
|
||||
source_kind: str,
|
||||
) -> list[tuple[str, dict[str, dict]]]:
|
||||
"""Load labelers whose IDs must exactly match their assigned queue."""
|
||||
|
||||
loaded = []
|
||||
names: set[str] = set()
|
||||
for name, path in inputs:
|
||||
if name in names:
|
||||
raise ValueError(f"Duplicate {source_kind} labeler name: {name}")
|
||||
names.add(name)
|
||||
records = read_json_lines(path)
|
||||
by_id = unique_records(records, f"{source_kind} {name}")
|
||||
if set(by_id) != expected_ids:
|
||||
missing = expected_ids - set(by_id)
|
||||
extra = set(by_id) - expected_ids
|
||||
raise ValueError(
|
||||
f"{source_kind} {name} id mismatch: "
|
||||
f"missing={len(missing)} extra={len(extra)}"
|
||||
)
|
||||
loaded.append(
|
||||
(
|
||||
name,
|
||||
{
|
||||
identifier: validate_model_record(
|
||||
record,
|
||||
identifier,
|
||||
f"{source_kind} {name}",
|
||||
)
|
||||
for identifier, record in by_id.items()
|
||||
},
|
||||
)
|
||||
)
|
||||
return loaded
|
||||
|
||||
|
||||
def resolve_votes(values: Sequence[str], required_votes: int) -> str:
|
||||
"""Accept one non-unknown value only when it reaches the vote threshold."""
|
||||
|
||||
votes = Counter(value for value in values if value != "unknown")
|
||||
if not votes:
|
||||
return "unknown"
|
||||
value, count = votes.most_common(1)[0]
|
||||
return value if count >= required_votes else "unknown"
|
||||
|
||||
|
||||
def resolve_model_states(
|
||||
identifier: str,
|
||||
primary: Sequence[tuple[str, dict[str, dict]]],
|
||||
reviewers: Sequence[tuple[str, dict[str, dict]]],
|
||||
in_review_queue: bool,
|
||||
) -> dict[str, str]:
|
||||
"""Resolve every field independently under the 3/3 or 4/5 rule."""
|
||||
|
||||
primary_records = [records[identifier] for _, records in primary]
|
||||
if in_review_queue:
|
||||
records = primary_records + [
|
||||
reviewer_records[identifier]
|
||||
for _, reviewer_records in reviewers
|
||||
]
|
||||
required_votes = 4
|
||||
else:
|
||||
records = primary_records
|
||||
required_votes = 3
|
||||
return {
|
||||
field: resolve_votes(
|
||||
[record[field] for record in records],
|
||||
required_votes,
|
||||
)
|
||||
for field in RESOLUTION_FIELDS
|
||||
}
|
||||
|
||||
|
||||
def validate_human_labels(
|
||||
records: Sequence[dict],
|
||||
expected_ids: set[str],
|
||||
) -> dict[str, dict]:
|
||||
"""Validate sparse product-owner labels without inferring absent fields."""
|
||||
|
||||
by_id = unique_records(records, "human labels")
|
||||
if set(by_id) != expected_ids:
|
||||
missing = expected_ids - set(by_id)
|
||||
extra = set(by_id) - expected_ids
|
||||
raise ValueError(
|
||||
"Human label ids must match the frozen human prefix: "
|
||||
f"missing={len(missing)} extra={len(extra)}"
|
||||
)
|
||||
for identifier, record in by_id.items():
|
||||
disposition = record.get("recordDisposition")
|
||||
if disposition not in {None, "keep", "exclude-device-command"}:
|
||||
raise ValueError(
|
||||
f"Invalid human recordDisposition for {identifier}"
|
||||
)
|
||||
for field in HUMAN_FIELDS:
|
||||
if field in record and record[field] not in LABEL_STATES:
|
||||
raise ValueError(f"Invalid human {field} for {identifier}")
|
||||
return by_id
|
||||
|
||||
|
||||
def apply_human_overrides(
|
||||
states: dict[str, str],
|
||||
human_record: dict | None,
|
||||
) -> list[str]:
|
||||
"""Override only explicitly supplied human fields."""
|
||||
|
||||
overridden = []
|
||||
if human_record is None:
|
||||
return overridden
|
||||
for field in HUMAN_FIELDS:
|
||||
if field in human_record:
|
||||
states[field] = human_record[field]
|
||||
overridden.append(field)
|
||||
return overridden
|
||||
|
||||
|
||||
def assign_splits(
|
||||
records: Sequence[dict],
|
||||
records_per_split: int | None,
|
||||
) -> dict[str, str]:
|
||||
"""Assign equal contiguous validation, test, and golden slices per language."""
|
||||
|
||||
by_language: dict[str, list[str]] = {"en": [], "zh-Hans": []}
|
||||
for record in records:
|
||||
by_language[record["language"]].append(record["id"])
|
||||
assignments = {}
|
||||
for language, identifiers in by_language.items():
|
||||
per_split = records_per_split
|
||||
if per_split is None:
|
||||
if len(identifiers) % len(SPLITS):
|
||||
raise ValueError(
|
||||
f"{language} count cannot be evenly divided into splits"
|
||||
)
|
||||
per_split = len(identifiers) // len(SPLITS)
|
||||
expected = per_split * len(SPLITS)
|
||||
if len(identifiers) != expected:
|
||||
raise ValueError(
|
||||
f"{language} must contain {expected} records, "
|
||||
f"found {len(identifiers)}"
|
||||
)
|
||||
for index, identifier in enumerate(identifiers):
|
||||
assignments[identifier] = SPLITS[index // per_split]
|
||||
return assignments
|
||||
|
||||
|
||||
def output_record(
|
||||
source: dict,
|
||||
states: dict[str, str],
|
||||
split: str,
|
||||
) -> dict:
|
||||
"""Build one partial-label evaluation record in the training schema."""
|
||||
|
||||
known_labels = [
|
||||
field for field in CONSENSUS_FIELDS if states[field] != "unknown"
|
||||
]
|
||||
return {
|
||||
"id": source["id"],
|
||||
"text": source["text"],
|
||||
"language": source["language"],
|
||||
"split": split,
|
||||
"family": "v6_blind_product_holdout",
|
||||
**{
|
||||
("replyable" if field == "replyableMessage" else field): (
|
||||
states[field] == "true"
|
||||
)
|
||||
for field in INTENT_LABELS
|
||||
},
|
||||
"sentiment": (
|
||||
states["sentiment"]
|
||||
if states["sentiment"] != "unknown"
|
||||
else "neutral"
|
||||
),
|
||||
"domain": (
|
||||
states["domain"] if states["domain"] != "unknown" else None
|
||||
),
|
||||
"ambiguous": (
|
||||
states["ambiguous"] == "true"
|
||||
if states["ambiguous"] != "unknown"
|
||||
else None
|
||||
),
|
||||
"knownLabels": known_labels,
|
||||
"sourceDataset": SOURCE_DATASET,
|
||||
"sourceLicense": SOURCE_LICENSE,
|
||||
"sourceRevision": SOURCE_REVISION,
|
||||
}
|
||||
|
||||
|
||||
def verify_report_hashes(
|
||||
holdout_path: Path,
|
||||
primary_inputs: Sequence[tuple[str, Path]],
|
||||
reviewer_inputs: Sequence[tuple[str, Path]],
|
||||
primary_report_path: Path,
|
||||
consensus_report_path: Path,
|
||||
review_count: int,
|
||||
) -> None:
|
||||
"""Cross-check frozen inputs against both existing consensus manifests."""
|
||||
|
||||
primary_report = json.loads(primary_report_path.read_text(encoding="utf-8"))
|
||||
consensus_report = json.loads(
|
||||
consensus_report_path.read_text(encoding="utf-8")
|
||||
)
|
||||
holdout_hash = sha256_file(holdout_path)
|
||||
for name, report in (
|
||||
("primary report", primary_report),
|
||||
("consensus report", consensus_report),
|
||||
):
|
||||
if report.get("queueSHA256") != holdout_hash:
|
||||
raise ValueError(f"{name} holdout SHA-256 mismatch")
|
||||
if primary_report.get("reviewCount") != review_count:
|
||||
raise ValueError("Primary report review count mismatch")
|
||||
all_inputs = (*primary_inputs, *reviewer_inputs)
|
||||
expected_primary = primary_report.get("primaryOutputSHA256") or {}
|
||||
expected_all = consensus_report.get("labelerOutputSHA256") or {}
|
||||
for name, path in all_inputs:
|
||||
actual_hash = sha256_file(path)
|
||||
if name in expected_primary and expected_primary[name] != actual_hash:
|
||||
raise ValueError(f"Primary report SHA-256 mismatch for {name}")
|
||||
if expected_all.get(name) != actual_hash:
|
||||
raise ValueError(f"Consensus report SHA-256 mismatch for {name}")
|
||||
|
||||
|
||||
def build_corpus(
|
||||
holdout: Sequence[dict],
|
||||
primary: Sequence[tuple[str, dict[str, dict]]],
|
||||
reviewers: Sequence[tuple[str, dict[str, dict]]],
|
||||
review_ids: set[str],
|
||||
human_by_id: dict[str, dict],
|
||||
records_per_split: int | None,
|
||||
) -> tuple[list[dict], dict[str, list[str]], dict[str, dict[str, str]]]:
|
||||
"""Resolve all records while preserving frozen order and partial labels."""
|
||||
|
||||
split_by_id = assign_splits(holdout, records_per_split)
|
||||
output = []
|
||||
overrides_by_id: dict[str, list[str]] = {}
|
||||
states_by_id: dict[str, dict[str, str]] = {}
|
||||
for source in holdout:
|
||||
identifier = source["id"]
|
||||
states = resolve_model_states(
|
||||
identifier,
|
||||
primary,
|
||||
reviewers,
|
||||
identifier in review_ids,
|
||||
)
|
||||
overrides = apply_human_overrides(states, human_by_id.get(identifier))
|
||||
overrides_by_id[identifier] = overrides
|
||||
states_by_id[identifier] = states
|
||||
output.append(output_record(source, states, split_by_id[identifier]))
|
||||
return output, overrides_by_id, states_by_id
|
||||
|
||||
|
||||
def build_report(
|
||||
output: Sequence[dict],
|
||||
states_by_id: dict[str, dict[str, str]],
|
||||
overrides_by_id: dict[str, list[str]],
|
||||
human_by_id: dict[str, dict],
|
||||
input_hashes: dict[str, str],
|
||||
output_hash: str,
|
||||
) -> dict:
|
||||
"""Summarize coverage without treating unknown defaults as labels."""
|
||||
|
||||
known_counts = Counter()
|
||||
positive_counts = Counter()
|
||||
unresolved_counts = Counter()
|
||||
for record in output:
|
||||
known_counts.update(record["knownLabels"])
|
||||
for field in INTENT_LABELS:
|
||||
output_field = "replyable" if field == "replyableMessage" else field
|
||||
if field in record["knownLabels"] and record[output_field]:
|
||||
positive_counts[field] += 1
|
||||
for field, state in states_by_id[record["id"]].items():
|
||||
if state == "unknown":
|
||||
unresolved_counts[field] += 1
|
||||
override_counts = Counter(
|
||||
field for fields in overrides_by_id.values() for field in fields
|
||||
)
|
||||
domains = Counter(
|
||||
record["domain"] for record in output if record["domain"] is not None
|
||||
)
|
||||
domains["unknown"] = sum(record["domain"] is None for record in output)
|
||||
return {
|
||||
"schemaVersion": 1,
|
||||
"sourceDataset": SOURCE_DATASET,
|
||||
"sourceLicense": SOURCE_LICENSE,
|
||||
"sourceRevision": SOURCE_REVISION,
|
||||
"recordCount": len(output),
|
||||
"inputSHA256": dict(sorted(input_hashes.items())),
|
||||
"outputSHA256": output_hash,
|
||||
"languages": dict(
|
||||
sorted(Counter(record["language"] for record in output).items())
|
||||
),
|
||||
"splits": dict(
|
||||
sorted(Counter(record["split"] for record in output).items())
|
||||
),
|
||||
"knownByField": {
|
||||
field: known_counts[field] for field in CONSENSUS_FIELDS
|
||||
},
|
||||
"positiveByIntent": {
|
||||
field: positive_counts[field] for field in INTENT_LABELS
|
||||
},
|
||||
"domains": dict(sorted(domains.items())),
|
||||
"humanCoverage": {
|
||||
"records": len(human_by_id),
|
||||
"overriddenRecords": sum(bool(value) for value in overrides_by_id.values()),
|
||||
"overridesByField": {
|
||||
field: override_counts[field] for field in HUMAN_FIELDS
|
||||
},
|
||||
"excludeDeviceCommandRecords": sum(
|
||||
record.get("recordDisposition") == "exclude-device-command"
|
||||
for record in human_by_id.values()
|
||||
),
|
||||
},
|
||||
"unresolvedByField": {
|
||||
field: unresolved_counts[field] for field in RESOLUTION_FIELDS
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def finalize(arguments: argparse.Namespace) -> dict:
|
||||
"""Load, validate, resolve, write, and report one frozen holdout."""
|
||||
|
||||
primary_inputs = tuple(arguments.primary or DEFAULT_PRIMARY)
|
||||
reviewer_inputs = tuple(arguments.reviewer or DEFAULT_REVIEWERS)
|
||||
if len(primary_inputs) != 3 or len(reviewer_inputs) != 2:
|
||||
raise ValueError("Exactly three primary and two reviewer inputs are required")
|
||||
holdout = read_json_lines(arguments.holdout)
|
||||
validate_holdout(holdout, arguments.expected_count)
|
||||
holdout_ids = {record["id"] for record in holdout}
|
||||
holdout_by_id = {record["id"]: record for record in holdout}
|
||||
review_queue = read_json_lines(arguments.review_queue)
|
||||
review_by_id = unique_records(review_queue, "review queue")
|
||||
review_ids = set(review_by_id)
|
||||
if not review_ids <= holdout_ids:
|
||||
raise ValueError("Review queue contains IDs outside the holdout")
|
||||
if (
|
||||
arguments.expected_review_count is not None
|
||||
and len(review_queue) != arguments.expected_review_count
|
||||
):
|
||||
raise ValueError(
|
||||
"Review queue must contain "
|
||||
f"{arguments.expected_review_count} records, found {len(review_queue)}"
|
||||
)
|
||||
for identifier, record in review_by_id.items():
|
||||
source = holdout_by_id[identifier]
|
||||
if (
|
||||
record.get("text") != source["text"]
|
||||
or record.get("language") != source["language"]
|
||||
):
|
||||
raise ValueError(f"Review queue changed frozen text for {identifier}")
|
||||
primary = load_model_outputs(
|
||||
primary_inputs,
|
||||
holdout_ids,
|
||||
"primary",
|
||||
)
|
||||
reviewers = load_model_outputs(
|
||||
reviewer_inputs,
|
||||
review_ids,
|
||||
"reviewer",
|
||||
)
|
||||
human_records = read_json_lines(arguments.human_labels)
|
||||
human_count = arguments.human_count
|
||||
if human_count is None:
|
||||
human_count = len(human_records)
|
||||
human_prefix_ids = {
|
||||
record["id"] for record in holdout[:human_count]
|
||||
}
|
||||
if len(human_records) != human_count:
|
||||
raise ValueError(
|
||||
f"Expected {human_count} human labels, found {len(human_records)}"
|
||||
)
|
||||
human_by_id = validate_human_labels(human_records, human_prefix_ids)
|
||||
if arguments.verify_manifests:
|
||||
verify_report_hashes(
|
||||
arguments.holdout,
|
||||
primary_inputs,
|
||||
reviewer_inputs,
|
||||
arguments.primary_report,
|
||||
arguments.consensus_report,
|
||||
len(review_queue),
|
||||
)
|
||||
output, overrides_by_id, states_by_id = build_corpus(
|
||||
holdout,
|
||||
primary,
|
||||
reviewers,
|
||||
review_ids,
|
||||
human_by_id,
|
||||
arguments.records_per_split,
|
||||
)
|
||||
if [record["id"] for record in output] != [
|
||||
record["id"] for record in holdout
|
||||
]:
|
||||
raise AssertionError("Output order changed")
|
||||
if any(record["split"] == "train" for record in output):
|
||||
raise AssertionError("Evaluation output must not contain train records")
|
||||
if any(
|
||||
output_record_value["text"] != source["text"]
|
||||
for output_record_value, source in zip(output, holdout)
|
||||
):
|
||||
raise AssertionError("Output text changed")
|
||||
write_json_lines(arguments.output, output)
|
||||
input_paths = {
|
||||
"holdout": arguments.holdout,
|
||||
"reviewQueue": arguments.review_queue,
|
||||
"humanLabels": arguments.human_labels,
|
||||
**{f"primary:{name}": path for name, path in primary_inputs},
|
||||
**{f"reviewer:{name}": path for name, path in reviewer_inputs},
|
||||
}
|
||||
report = build_report(
|
||||
output,
|
||||
states_by_id,
|
||||
overrides_by_id,
|
||||
human_by_id,
|
||||
{name: sha256_file(path) for name, path in input_paths.items()},
|
||||
sha256_file(arguments.output),
|
||||
)
|
||||
arguments.report.parent.mkdir(parents=True, exist_ok=True)
|
||||
arguments.report.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return report
|
||||
|
||||
|
||||
def parser() -> argparse.ArgumentParser:
|
||||
"""Build the command-line interface with production-safe defaults."""
|
||||
|
||||
value = argparse.ArgumentParser()
|
||||
value.add_argument("--holdout", type=Path, default=DEFAULT_HOLDOUT)
|
||||
value.add_argument(
|
||||
"--review-queue",
|
||||
type=Path,
|
||||
default=V6_BLIND_DIRECTORY / "review-queue.jsonl",
|
||||
)
|
||||
value.add_argument("--primary", action="append", type=parse_named_path)
|
||||
value.add_argument("--reviewer", action="append", type=parse_named_path)
|
||||
value.add_argument(
|
||||
"--human-labels",
|
||||
type=Path,
|
||||
default=DEFAULT_HUMAN_LABELS,
|
||||
)
|
||||
value.add_argument(
|
||||
"--primary-report",
|
||||
type=Path,
|
||||
default=V6_BLIND_DIRECTORY / "primary-report.json",
|
||||
)
|
||||
value.add_argument(
|
||||
"--consensus-report",
|
||||
type=Path,
|
||||
default=V6_BLIND_DIRECTORY / "consensus-report.json",
|
||||
)
|
||||
value.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
|
||||
value.add_argument("--report", type=Path, default=DEFAULT_REPORT)
|
||||
value.add_argument("--expected-count", type=int, default=120)
|
||||
value.add_argument("--expected-review-count", type=int, default=94)
|
||||
value.add_argument("--human-count", type=int, default=60)
|
||||
value.add_argument("--records-per-split", type=int, default=20)
|
||||
value.add_argument(
|
||||
"--skip-manifest-verification",
|
||||
action="store_false",
|
||||
dest="verify_manifests",
|
||||
)
|
||||
value.set_defaults(verify_manifests=True)
|
||||
return value
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Run the finalizer and print its compact completion counts."""
|
||||
|
||||
report = finalize(parser().parse_args())
|
||||
print(
|
||||
"V6_BLIND_FINALIZED "
|
||||
f"records={report['recordCount']} "
|
||||
f"unresolved={sum(report['unresolvedByField'].values())}"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -13,6 +13,7 @@ import hashlib
|
||||
import json
|
||||
import random
|
||||
import re
|
||||
import sys
|
||||
from collections import Counter
|
||||
from dataclasses import asdict, dataclass
|
||||
from pathlib import Path
|
||||
@@ -20,6 +21,7 @@ from typing import Iterable
|
||||
|
||||
|
||||
SEED = 20260821
|
||||
TRAIN_SAMPLE_SCALE = 1
|
||||
TARGETS = {"train": 180, "validation": 45, "test": 45}
|
||||
FAMILY_TARGETS = {
|
||||
"quoted_question": {"train": 100, "validation": 45, "test": 45},
|
||||
@@ -2091,6 +2093,7 @@ def generate_family(
|
||||
slots: dict[str, list[str]],
|
||||
labels: Labels,
|
||||
target: int,
|
||||
minimum_target: int | None,
|
||||
seen: set[str],
|
||||
uses_discourse_prefixes: bool,
|
||||
) -> Iterable[Record]:
|
||||
@@ -2144,11 +2147,19 @@ def generate_family(
|
||||
replyable=bool(labels.replyable),
|
||||
)
|
||||
|
||||
if produced != target:
|
||||
if produced != target and (
|
||||
minimum_target is None or produced < minimum_target
|
||||
):
|
||||
raise RuntimeError(
|
||||
f"Only generated {produced}/{target} unique records for "
|
||||
f"{family} {language} {split}"
|
||||
)
|
||||
if produced != target:
|
||||
print(
|
||||
f"Warning: generated all {produced} unique records available "
|
||||
f"for {family} {language} {split}; requested {target}.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
|
||||
def generate_records(profile: str) -> list[Record]:
|
||||
@@ -2174,6 +2185,9 @@ def generate_records(profile: str) -> list[Record]:
|
||||
split_templates = definition["templates"][language]
|
||||
family_targets = FAMILY_TARGETS.get(family, TARGETS)
|
||||
for split, target in family_targets.items():
|
||||
scaled_target = (
|
||||
target * TRAIN_SAMPLE_SCALE if split == "train" else target
|
||||
)
|
||||
records.extend(
|
||||
generate_family(
|
||||
family=family,
|
||||
@@ -2182,7 +2196,12 @@ def generate_records(profile: str) -> list[Record]:
|
||||
templates=split_templates[split],
|
||||
slots=slots,
|
||||
labels=labels,
|
||||
target=target,
|
||||
target=scaled_target,
|
||||
minimum_target=(
|
||||
target
|
||||
if split == "train" and TRAIN_SAMPLE_SCALE > 1
|
||||
else None
|
||||
),
|
||||
seen=seen,
|
||||
uses_discourse_prefixes=profile == "expanded",
|
||||
)
|
||||
@@ -2361,6 +2380,7 @@ def summary(records: list[Record]) -> dict[str, object]:
|
||||
sentiment_counts = Counter(record.sentiment for record in records)
|
||||
return {
|
||||
"seed": SEED,
|
||||
"trainSampleScale": TRAIN_SAMPLE_SCALE,
|
||||
"total": len(records),
|
||||
"splits": dict(sorted(split_counts.items())),
|
||||
"languages": dict(sorted(language_counts.items())),
|
||||
@@ -2381,6 +2401,12 @@ def summary(records: list[Record]) -> dict[str, object]:
|
||||
|
||||
def parse_arguments() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--train-sample-scale",
|
||||
type=int,
|
||||
default=1,
|
||||
help="Multiply train records per family without changing evaluation splits.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--profile",
|
||||
choices=("baseline", "expanded"),
|
||||
@@ -2400,7 +2426,12 @@ def parse_arguments() -> argparse.Namespace:
|
||||
|
||||
|
||||
def main() -> None:
|
||||
global TRAIN_SAMPLE_SCALE
|
||||
|
||||
arguments = parse_arguments()
|
||||
if arguments.train_sample_scale < 1:
|
||||
raise ValueError("--train-sample-scale must be at least 1")
|
||||
TRAIN_SAMPLE_SCALE = arguments.train_sample_scale
|
||||
records = generate_records(arguments.profile)
|
||||
validate(records)
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,722 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate a deterministic bilingual supplement for taxonomy-v6 boundaries."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import itertools
|
||||
import json
|
||||
import unicodedata
|
||||
from collections import Counter
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Iterable, Iterator, Sequence
|
||||
|
||||
|
||||
OUTPUT_DIRECTORY = Path("ModelTraining/ClipboardSemantics")
|
||||
DEFAULT_OUTPUT_PATH = OUTPUT_DIRECTORY / "v6-boundary-training-supplement.jsonl"
|
||||
DEFAULT_SUMMARY_PATH = OUTPUT_DIRECTORY / "v6-boundary-training-supplement-summary.json"
|
||||
DEFAULT_POSITIVE_PER_INTENT_LANGUAGE = 1_000
|
||||
DEFAULT_NEGATIVE_PER_INTENT_LANGUAGE = 200
|
||||
SOURCE_DATASET = "OSGKeyboard taxonomy-v6 deterministic boundary templates"
|
||||
SOURCE_LICENSE = "OSGKeyboard project license"
|
||||
SOURCE_REVISION = "v6-boundary-templates-1"
|
||||
SOURCE_SPLIT = "train"
|
||||
SAMPLE_WEIGHT = 0.35
|
||||
|
||||
NEW_INTENTS = (
|
||||
"assistantCommand",
|
||||
"informationQuery",
|
||||
"systemNotification",
|
||||
)
|
||||
HARD_NEGATIVE_INTENTS = ("task", "question", "replyableMessage")
|
||||
BOUNDARY_INTENTS = NEW_INTENTS + HARD_NEGATIVE_INTENTS
|
||||
INTENT_LABELS = (
|
||||
"task",
|
||||
"question",
|
||||
"invitation",
|
||||
"complaint",
|
||||
"scheduleNegotiation",
|
||||
"confirmationDecision",
|
||||
"followUpReminder",
|
||||
"blessing",
|
||||
"replyableMessage",
|
||||
"assistantCommand",
|
||||
"informationQuery",
|
||||
"systemNotification",
|
||||
)
|
||||
KNOWN_LABELS = sorted((*BOUNDARY_INTENTS, "domain"))
|
||||
DOMAINS = (
|
||||
"finance",
|
||||
"travel",
|
||||
"calendar",
|
||||
"communication",
|
||||
"media",
|
||||
"smartHome",
|
||||
"shopping",
|
||||
"dining",
|
||||
"health",
|
||||
"weather",
|
||||
"accountService",
|
||||
"generalKnowledge",
|
||||
)
|
||||
LANGUAGES = ("en", "zh-Hans")
|
||||
CATEGORY_OFFSETS = {
|
||||
"assistantCommand": 0,
|
||||
"informationQuery": 4,
|
||||
"systemNotification": 8,
|
||||
"task": 0,
|
||||
"question": 4,
|
||||
"replyableMessage": 8,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DomainSlots:
|
||||
"""Finite, project-authored slots for one product-policy domain."""
|
||||
|
||||
subjects_en: tuple[str, ...]
|
||||
subjects_zh: tuple[str, ...]
|
||||
facts_en: tuple[str, ...]
|
||||
facts_zh: tuple[str, ...]
|
||||
services_en: tuple[str, ...]
|
||||
services_zh: tuple[str, ...]
|
||||
providers_en: tuple[str, ...]
|
||||
providers_zh: tuple[str, ...]
|
||||
events_en: tuple[str, ...]
|
||||
events_zh: tuple[str, ...]
|
||||
|
||||
|
||||
DOMAIN_SLOTS: dict[str, DomainSlots] = {
|
||||
"finance": DomainSlots(
|
||||
("savings account", "credit card", "monthly budget", "insurance claim"),
|
||||
("储蓄账户", "信用卡", "月度预算", "保险理赔"),
|
||||
("current balance", "exchange rate", "payment status", "claim progress"),
|
||||
("当前余额", "汇率", "付款状态", "理赔进度"),
|
||||
("transfer funds", "replace the card", "submit the claim", "buy the fund"),
|
||||
("转账", "补办卡片", "提交理赔", "购买基金"),
|
||||
("the bank", "the card issuer", "the insurer", "the broker"),
|
||||
("银行", "发卡行", "保险公司", "券商"),
|
||||
("was approved", "was declined", "is being reviewed", "needs verification"),
|
||||
("已获批准", "已被拒绝", "正在审核", "需要验证"),
|
||||
),
|
||||
"travel": DomainSlots(
|
||||
("morning flight", "hotel booking", "train ticket", "airport transfer"),
|
||||
("早班航班", "酒店预订", "火车票", "机场接送"),
|
||||
("departure time", "platform number", "booking status", "delay estimate"),
|
||||
("出发时间", "站台编号", "预订状态", "延误时长"),
|
||||
("book the flight", "reserve the hotel", "change the ticket", "order a taxi"),
|
||||
("预订航班", "预订酒店", "改签车票", "预约出租车"),
|
||||
("the airline", "the hotel", "the railway", "the taxi company"),
|
||||
("航空公司", "酒店", "铁路客服", "出租车公司"),
|
||||
("was confirmed", "was delayed", "changed gates", "was cancelled"),
|
||||
("已确认", "已延误", "已变更登机口", "已取消"),
|
||||
),
|
||||
"calendar": DomainSlots(
|
||||
("team meeting", "dentist appointment", "morning alarm", "project reminder"),
|
||||
("团队会议", "牙医预约", "早晨闹钟", "项目提醒"),
|
||||
("start time", "meeting location", "next occurrence", "attendee list"),
|
||||
("开始时间", "会议地点", "下次时间", "参与者名单"),
|
||||
("reserve a meeting room", "reschedule the appointment", "invite the team", "book the venue"),
|
||||
("预订会议室", "改约时间", "邀请团队", "预订场地"),
|
||||
("the office", "the clinic", "the event host", "the venue"),
|
||||
("办公室", "诊所", "活动主办方", "场地方"),
|
||||
("was added", "was moved", "has a conflict", "starts soon"),
|
||||
("已添加", "已改期", "存在冲突", "即将开始"),
|
||||
),
|
||||
"communication": DomainSlots(
|
||||
("work inbox", "family group", "video call", "contact list"),
|
||||
("工作收件箱", "家庭群聊", "视频通话", "联系人列表"),
|
||||
("unread count", "call duration", "delivery status", "contact details"),
|
||||
("未读数量", "通话时长", "送达状态", "联系信息"),
|
||||
("send the parcel", "print the invitation", "deliver the letter", "arrange an interpreter"),
|
||||
("寄送包裹", "印刷邀请函", "投递信件", "安排翻译"),
|
||||
("the courier", "the print shop", "the post office", "the agency"),
|
||||
("快递公司", "印刷店", "邮局", "服务机构"),
|
||||
("finished syncing", "lost connection", "was delivered", "needs permission"),
|
||||
("已同步完成", "连接已断开", "已送达", "需要权限"),
|
||||
),
|
||||
"media": DomainSlots(
|
||||
("jazz playlist", "evening podcast", "photo album", "news channel"),
|
||||
("爵士歌单", "晚间播客", "照片相册", "新闻频道"),
|
||||
("episode length", "release date", "track title", "download progress"),
|
||||
("单集时长", "发布日期", "曲目名称", "下载进度"),
|
||||
("buy the album", "rent the film", "print the photos", "book the studio"),
|
||||
("购买专辑", "租赁影片", "冲印照片", "预订录音棚"),
|
||||
("the music store", "the cinema service", "the photo lab", "the studio"),
|
||||
("音乐商店", "影视服务商", "照片冲印店", "录音棚"),
|
||||
("finished downloading", "is unavailable", "resumed playing", "was removed"),
|
||||
("已下载完成", "暂不可用", "已继续播放", "已被移除"),
|
||||
),
|
||||
"smartHome": DomainSlots(
|
||||
("living-room lights", "front-door lock", "bedroom thermostat", "robot vacuum"),
|
||||
("客厅灯", "前门门锁", "卧室温控器", "扫地机器人"),
|
||||
("power level", "lock status", "room temperature", "cleaning progress"),
|
||||
("电量", "门锁状态", "室温", "清扫进度"),
|
||||
("repair the lock", "install the thermostat", "service the vacuum", "replace the sensor"),
|
||||
("维修门锁", "安装温控器", "保养扫地机", "更换传感器"),
|
||||
("the locksmith", "the installer", "the repair shop", "the electrician"),
|
||||
("锁匠", "安装人员", "维修点", "电工"),
|
||||
("went offline", "is back online", "detected motion", "finished cleaning"),
|
||||
("已离线", "已恢复在线", "检测到移动", "已完成清扫"),
|
||||
),
|
||||
"shopping": DomainSlots(
|
||||
("grocery list", "shoe order", "gift basket", "store coupon"),
|
||||
("购物清单", "鞋子订单", "礼品篮", "商店优惠券"),
|
||||
("current price", "stock level", "delivery date", "discount amount"),
|
||||
("当前价格", "库存数量", "送达日期", "折扣金额"),
|
||||
("place the order", "exchange the shoes", "wrap the gift", "schedule delivery"),
|
||||
("下单", "换鞋", "包装礼物", "预约配送"),
|
||||
("the retailer", "the shoe store", "the gift shop", "the courier"),
|
||||
("零售商", "鞋店", "礼品店", "快递公司"),
|
||||
("was shipped", "is out of stock", "was refunded", "is ready for pickup"),
|
||||
("已发货", "已售罄", "已退款", "可到店取货"),
|
||||
),
|
||||
"dining": DomainSlots(
|
||||
("dinner booking", "lunch menu", "takeout order", "coffee subscription"),
|
||||
("晚餐预订", "午餐菜单", "外卖订单", "咖啡订购"),
|
||||
("table availability", "waiting time", "order status", "menu price"),
|
||||
("空桌情况", "等位时间", "订单状态", "菜单价格"),
|
||||
("reserve a table", "change the order", "deliver the meal", "cater the event"),
|
||||
("预订餐桌", "修改订单", "配送餐食", "承办餐饮"),
|
||||
("the restaurant", "the takeaway", "the café", "the caterer"),
|
||||
("餐厅", "外卖商家", "咖啡店", "餐饮公司"),
|
||||
("was accepted", "is being prepared", "is ready", "was cancelled"),
|
||||
("已接单", "正在制作", "已备好", "已取消"),
|
||||
),
|
||||
"health": DomainSlots(
|
||||
("step record", "sleep report", "prescription", "vaccination record"),
|
||||
("步数记录", "睡眠报告", "处方", "疫苗接种记录"),
|
||||
("daily total", "renewal date", "dosage note", "appointment status"),
|
||||
("当日总数", "续方日期", "剂量说明", "预约状态"),
|
||||
("book an examination", "refill the prescription", "deliver the medicine", "arrange home care"),
|
||||
("预约检查", "续开处方", "配送药品", "安排居家护理"),
|
||||
("the clinic", "the pharmacy", "the hospital", "the care provider"),
|
||||
("诊所", "药房", "医院", "护理机构"),
|
||||
("was updated", "needs review", "is ready to collect", "was received"),
|
||||
("已更新", "需要复核", "可领取", "已收到"),
|
||||
),
|
||||
"weather": DomainSlots(
|
||||
("rain forecast", "air-quality report", "storm tracker", "temperature chart"),
|
||||
("降雨预报", "空气质量报告", "风暴追踪", "气温图表"),
|
||||
("rain chance", "air-quality index", "storm path", "high temperature"),
|
||||
("降雨概率", "空气质量指数", "风暴路径", "最高温度"),
|
||||
("inspect the roof", "clear the snow", "deliver sandbags", "repair the drain"),
|
||||
("检查屋顶", "清理积雪", "运送沙袋", "维修排水管"),
|
||||
("the roofer", "the snow service", "the emergency supplier", "the plumber"),
|
||||
("屋顶维修方", "除雪服务商", "应急物资商", "水管工"),
|
||||
("was updated", "issued a warning", "cleared the alert", "changed direction"),
|
||||
("已更新", "已发布预警", "已解除警报", "已改变方向"),
|
||||
),
|
||||
"accountService": DomainSlots(
|
||||
("cloud account", "software license", "support ticket", "security setting"),
|
||||
("云端账户", "软件许可证", "支持工单", "安全设置"),
|
||||
("renewal date", "ticket status", "storage usage", "sign-in history"),
|
||||
("续订日期", "工单状态", "存储用量", "登录历史"),
|
||||
("upgrade the plan", "recover the account", "renew the license", "schedule support"),
|
||||
("升级套餐", "恢复账户", "续订许可证", "预约支持"),
|
||||
("the provider", "the support desk", "the software vendor", "the service team"),
|
||||
("服务商", "支持团队", "软件供应商", "客服团队"),
|
||||
("was renewed", "was suspended", "needs verification", "was restored"),
|
||||
("已续订", "已暂停", "需要验证", "已恢复"),
|
||||
),
|
||||
"generalKnowledge": DomainSlots(
|
||||
("history article", "science glossary", "language guide", "reference note"),
|
||||
("历史条目", "科学词典", "语言指南", "参考笔记"),
|
||||
("publication date", "short definition", "source citation", "latest revision"),
|
||||
("发布日期", "简短定义", "来源引用", "最新修订"),
|
||||
("translate the manuscript", "verify the archive", "print the encyclopedia", "catalog the collection"),
|
||||
("翻译手稿", "核验档案", "印刷百科全书", "编目藏品"),
|
||||
("the translator", "the archive", "the publisher", "the library"),
|
||||
("翻译机构", "档案馆", "出版社", "图书馆"),
|
||||
("was revised", "is temporarily unavailable", "added a citation", "finished indexing"),
|
||||
("已修订", "暂不可用", "已添加引用", "已完成索引"),
|
||||
),
|
||||
}
|
||||
|
||||
DISPLAY_VALUES = (
|
||||
("compact mode", "紧凑模式"),
|
||||
("the top position", "顶部"),
|
||||
("a blue highlight", "蓝色高亮"),
|
||||
("the favorites section", "收藏区"),
|
||||
)
|
||||
APP_SCOPES = (
|
||||
("the dashboard", "仪表盘"),
|
||||
("the quick panel", "快捷面板"),
|
||||
("the saved view", "已存视图"),
|
||||
("the app widget", "应用小组件"),
|
||||
)
|
||||
PEOPLE = (
|
||||
("Alex", "小林"),
|
||||
("Morgan", "小周"),
|
||||
("Taylor", "小陈"),
|
||||
("Jordan", "小何"),
|
||||
)
|
||||
|
||||
TEMPLATES: dict[str, dict[str, tuple[str, ...]]] = {
|
||||
"en": {
|
||||
"assistantCommand": (
|
||||
"Set {subject} to {value} in {scope}.",
|
||||
"Pin {subject} at {value} on {scope}.",
|
||||
"Show {subject} with {value} in {scope}.",
|
||||
"Move {subject} to {value} on {scope}.",
|
||||
),
|
||||
"informationQuery": (
|
||||
"Look up the {fact} for {subject} from {scope}.",
|
||||
"What is the {fact} for {subject} in {scope}?",
|
||||
"Show me the latest {fact} for {subject} from {scope}.",
|
||||
"Find the current {fact} for {subject} in {scope}.",
|
||||
),
|
||||
"systemNotification": (
|
||||
"System notice: {subject} {event}; details are in {scope}.",
|
||||
"Service update: {subject} {event}. Open {scope} for details.",
|
||||
"Automatic alert: {subject} {event} in {scope}.",
|
||||
"Status update from {scope}: {subject} {event}.",
|
||||
),
|
||||
"task": (
|
||||
"Please ask {provider} to {service} for {subject}.",
|
||||
"Arrange for {provider} to {service} regarding {subject}.",
|
||||
"I need {provider} to {service} for {subject}.",
|
||||
"Have {provider} {service} for {subject}, with confirmation.",
|
||||
),
|
||||
"question": (
|
||||
"{person}, do you think {subject} belongs in {scope}?",
|
||||
"{person}, would {subject} work better with {value}?",
|
||||
"In your opinion, is {subject} suitable for {scope}, {person}?",
|
||||
"{person}, which presentation of {subject} would you prefer in {scope}?",
|
||||
),
|
||||
"replyableMessage": (
|
||||
"{person}, I shared {subject} with you through {scope}; let me know when you see it.",
|
||||
"{person}, I left the notes about {subject} in {scope} and would value your reaction.",
|
||||
"{person}, the draft for {subject} is in {scope}; please reply when you have reviewed it.",
|
||||
"{person}, I updated {subject} in {scope}; tell me whether it works for you.",
|
||||
),
|
||||
},
|
||||
"zh-Hans": {
|
||||
"assistantCommand": (
|
||||
"把{subject}在{scope}中设为{value}。",
|
||||
"将{subject}以{value}固定到{scope}。",
|
||||
"在{scope}中用{value}显示{subject}。",
|
||||
"把{subject}移到{scope}的{value}。",
|
||||
),
|
||||
"informationQuery": (
|
||||
"查询{scope}里{subject}的{fact}。",
|
||||
"{scope}中{subject}的{fact}是什么?",
|
||||
"显示{scope}里{subject}最新的{fact}。",
|
||||
"查找{scope}中{subject}当前的{fact}。",
|
||||
),
|
||||
"systemNotification": (
|
||||
"系统通知:{subject}{event},详情请查看{scope}。",
|
||||
"服务更新:{subject}{event},可在{scope}查看详情。",
|
||||
"自动提醒:{scope}中的{subject}{event}。",
|
||||
"来自{scope}的状态更新:{subject}{event}。",
|
||||
),
|
||||
"task": (
|
||||
"请联系{provider}为{subject}{service}。",
|
||||
"安排{provider}处理{subject}并{service}。",
|
||||
"我需要{provider}针对{subject}{service}。",
|
||||
"请让{provider}为{subject}{service},并确认结果。",
|
||||
),
|
||||
"question": (
|
||||
"{person},你觉得{subject}适合放在{scope}吗?",
|
||||
"{person},你认为把{subject}设为{value}会更好吗?",
|
||||
"{person},依你看{subject}放进{scope}合适吗?",
|
||||
"{person},你更喜欢{subject}在{scope}里怎样展示?",
|
||||
),
|
||||
"replyableMessage": (
|
||||
"{person},我已经通过{scope}把{subject}分享给你,看到后告诉我一声。",
|
||||
"{person},我把{subject}的说明放在{scope}了,想听听你的看法。",
|
||||
"{person},关于{subject}的草稿在{scope}里,看完请回复我。",
|
||||
"{person},我更新了{scope}里的{subject},请告诉我是否合适。",
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def normalize_text(value: str) -> str:
|
||||
"""Apply the corpus's stable NFKC and whitespace normalization."""
|
||||
|
||||
return " ".join(unicodedata.normalize("NFKC", value).split()).strip()
|
||||
|
||||
|
||||
def fingerprint(value: str) -> str:
|
||||
"""Return a conservative normalized key for exact-text exclusion."""
|
||||
|
||||
return normalize_text(value).casefold()
|
||||
|
||||
|
||||
def stable_key(*values: object) -> bytes:
|
||||
"""Create an ordering key independent of Python hash randomization."""
|
||||
|
||||
return hashlib.sha256("|".join(map(str, values)).encode("utf-8")).digest()
|
||||
|
||||
|
||||
def paired_values(values: Sequence[tuple[str, str]], language: str) -> tuple[str, ...]:
|
||||
index = 0 if language == "en" else 1
|
||||
return tuple(value[index] for value in values)
|
||||
|
||||
|
||||
def slots_for(
|
||||
slots: DomainSlots,
|
||||
language: str,
|
||||
category: str,
|
||||
) -> tuple[tuple[str, ...], ...]:
|
||||
"""Return only finite slots that are semantically valid for a category."""
|
||||
|
||||
suffix = "en" if language == "en" else "zh"
|
||||
subjects = getattr(slots, f"subjects_{suffix}")
|
||||
providers = getattr(slots, f"providers_{suffix}")
|
||||
values = paired_values(DISPLAY_VALUES, language)
|
||||
scopes = paired_values(APP_SCOPES, language)
|
||||
people = paired_values(PEOPLE, language)
|
||||
if category == "assistantCommand":
|
||||
return subjects, values, scopes
|
||||
if category == "informationQuery":
|
||||
return subjects, getattr(slots, f"facts_{suffix}"), scopes
|
||||
if category == "systemNotification":
|
||||
return subjects, getattr(slots, f"events_{suffix}"), scopes
|
||||
if category == "task":
|
||||
return subjects, getattr(slots, f"services_{suffix}"), providers
|
||||
if category == "question":
|
||||
return subjects, values, scopes, people
|
||||
return subjects, scopes, people
|
||||
|
||||
|
||||
def candidate_texts(
|
||||
language: str,
|
||||
category: str,
|
||||
domain: str,
|
||||
) -> Iterator[tuple[str, str]]:
|
||||
"""Yield every deterministic template/slot combination for one cell."""
|
||||
|
||||
templates = TEMPLATES[language][category]
|
||||
slot_groups = slots_for(DOMAIN_SLOTS[domain], language, category)
|
||||
argument_names = {
|
||||
"assistantCommand": ("subject", "value", "scope"),
|
||||
"informationQuery": ("subject", "fact", "scope"),
|
||||
"systemNotification": ("subject", "event", "scope"),
|
||||
"task": ("subject", "service", "provider"),
|
||||
"question": ("subject", "value", "scope", "person"),
|
||||
"replyableMessage": ("subject", "scope", "person"),
|
||||
}[category]
|
||||
candidates: list[tuple[str, str]] = []
|
||||
for template_index, template in enumerate(templates, start=1):
|
||||
family = f"v6_{category}_{language}_template_{template_index}"
|
||||
for values in itertools.product(*slot_groups):
|
||||
text = normalize_text(template.format(**dict(zip(argument_names, values))))
|
||||
candidates.append((text, family))
|
||||
yield from sorted(
|
||||
candidates,
|
||||
key=lambda value: stable_key(language, category, domain, *value),
|
||||
)
|
||||
|
||||
|
||||
def quota_by_domain(total: int, offset: int) -> dict[str, int]:
|
||||
"""Distribute a category total across all domains without randomness."""
|
||||
|
||||
base, remainder = divmod(total, len(DOMAINS))
|
||||
quotas = {domain: base for domain in DOMAINS}
|
||||
for index in range(remainder):
|
||||
quotas[DOMAINS[(offset + index) % len(DOMAINS)]] += 1
|
||||
return quotas
|
||||
|
||||
|
||||
def make_record(
|
||||
*,
|
||||
index: int,
|
||||
text: str,
|
||||
language: str,
|
||||
category: str,
|
||||
domain: str,
|
||||
template_family: str,
|
||||
) -> dict:
|
||||
"""Build the complete current training schema for one known boundary."""
|
||||
|
||||
label_values = {label: False for label in INTENT_LABELS}
|
||||
label_values[category] = True
|
||||
# An interpersonal question naturally invites a reply; both fields are
|
||||
# template-determined while all three routing intents remain false.
|
||||
if category == "question":
|
||||
label_values["replyableMessage"] = True
|
||||
record_id = (
|
||||
f"v6-boundary-{language}-{category}-{domain}-"
|
||||
f"{index:04d}-{hashlib.sha256(text.encode('utf-8')).hexdigest()[:12]}"
|
||||
)
|
||||
return {
|
||||
"id": record_id,
|
||||
"text": text,
|
||||
"language": language,
|
||||
"split": "train",
|
||||
"family": template_family,
|
||||
"task": label_values["task"],
|
||||
"question": label_values["question"],
|
||||
"invitation": label_values["invitation"],
|
||||
"complaint": label_values["complaint"],
|
||||
"scheduleNegotiation": label_values["scheduleNegotiation"],
|
||||
"confirmationDecision": label_values["confirmationDecision"],
|
||||
"followUpReminder": label_values["followUpReminder"],
|
||||
"blessing": label_values["blessing"],
|
||||
"sentiment": "neutral",
|
||||
"replyable": label_values["replyableMessage"],
|
||||
"assistantCommand": label_values["assistantCommand"],
|
||||
"informationQuery": label_values["informationQuery"],
|
||||
"systemNotification": label_values["systemNotification"],
|
||||
"domain": domain,
|
||||
"sampleWeight": SAMPLE_WEIGHT,
|
||||
"knownLabels": KNOWN_LABELS,
|
||||
"labelingMethod": "deterministic taxonomy-v6 template and finite slots",
|
||||
"sourceDataset": SOURCE_DATASET,
|
||||
"sourceLicense": SOURCE_LICENSE,
|
||||
"sourceURL": "Scripts/clipboard_semantics/generate_v6_boundary_corpus.py",
|
||||
"sourceRevision": SOURCE_REVISION,
|
||||
"sourceSplit": SOURCE_SPLIT,
|
||||
"synthetic": True,
|
||||
"templateFamily": template_family,
|
||||
}
|
||||
|
||||
|
||||
def discover_holdout_paths(directory: Path) -> list[Path]:
|
||||
"""Find every frozen holdout name covered by the v6 training policy."""
|
||||
|
||||
paths = set(directory.rglob("*holdout-corpus.jsonl"))
|
||||
blind = directory / "product-policy-blind-holdout-v1.jsonl"
|
||||
if blind.is_file():
|
||||
paths.add(blind)
|
||||
return sorted(paths)
|
||||
|
||||
|
||||
def load_holdout_fingerprints(paths: Iterable[Path]) -> set[str]:
|
||||
"""Load normalized text keys without depending on any holdout schema extras."""
|
||||
|
||||
fingerprints: set[str] = set()
|
||||
for path in sorted(set(paths)):
|
||||
if not path.is_file():
|
||||
continue
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
value = json.loads(line)
|
||||
text = value.get("text")
|
||||
if isinstance(text, str) and normalize_text(text):
|
||||
fingerprints.add(fingerprint(text))
|
||||
return fingerprints
|
||||
|
||||
|
||||
def validate_records(records: Sequence[dict], holdouts: set[str]) -> None:
|
||||
"""Fail closed on duplicate, leakage, schema, or annotation mistakes."""
|
||||
|
||||
if len({record["id"] for record in records}) != len(records):
|
||||
raise ValueError("Duplicate generated IDs")
|
||||
text_keys = [fingerprint(record["text"]) for record in records]
|
||||
if len(set(text_keys)) != len(records):
|
||||
raise ValueError("Duplicate generated texts")
|
||||
leaked = set(text_keys) & holdouts
|
||||
if leaked:
|
||||
raise ValueError(f"Holdout overlap remained after filtering: {len(leaked)}")
|
||||
for record in records:
|
||||
if record["text"] != normalize_text(record["text"]):
|
||||
raise ValueError(f"Non-NFKC record: {record['id']}")
|
||||
if record["knownLabels"] != KNOWN_LABELS:
|
||||
raise ValueError(f"Unexpected known labels: {record['id']}")
|
||||
if record["domain"] not in DOMAINS:
|
||||
raise ValueError(f"Unsupported domain: {record['id']}")
|
||||
if record["sampleWeight"] != SAMPLE_WEIGHT:
|
||||
raise ValueError(f"Unexpected sample weight: {record['id']}")
|
||||
|
||||
|
||||
def generate_records(
|
||||
*,
|
||||
positive_per_intent_language: int = DEFAULT_POSITIVE_PER_INTENT_LANGUAGE,
|
||||
negative_per_intent_language: int = DEFAULT_NEGATIVE_PER_INTENT_LANGUAGE,
|
||||
holdout_fingerprints: set[str] | None = None,
|
||||
) -> tuple[list[dict], Counter[str], int]:
|
||||
"""Generate balanced records and return target-intent counts plus exclusions."""
|
||||
|
||||
if positive_per_intent_language < 1 or negative_per_intent_language < 1:
|
||||
raise ValueError("Per-intent counts must be positive")
|
||||
holdouts = holdout_fingerprints or set()
|
||||
records: list[dict] = []
|
||||
target_counts: Counter[str] = Counter()
|
||||
seen: set[str] = set()
|
||||
excluded_holdout = 0
|
||||
for language in LANGUAGES:
|
||||
for category in BOUNDARY_INTENTS:
|
||||
category_total = (
|
||||
positive_per_intent_language
|
||||
if category in NEW_INTENTS
|
||||
else negative_per_intent_language
|
||||
)
|
||||
quotas = quota_by_domain(category_total, CATEGORY_OFFSETS[category])
|
||||
for domain in DOMAINS:
|
||||
selected = 0
|
||||
for text, family in candidate_texts(language, category, domain):
|
||||
text_key = fingerprint(text)
|
||||
if text_key in holdouts:
|
||||
excluded_holdout += 1
|
||||
continue
|
||||
if text_key in seen:
|
||||
continue
|
||||
record = make_record(
|
||||
index=selected + 1,
|
||||
text=text,
|
||||
language=language,
|
||||
category=category,
|
||||
domain=domain,
|
||||
template_family=family,
|
||||
)
|
||||
records.append(record)
|
||||
seen.add(text_key)
|
||||
target_counts[f"{language}|{category}"] += 1
|
||||
selected += 1
|
||||
if selected == quotas[domain]:
|
||||
break
|
||||
if selected != quotas[domain]:
|
||||
raise ValueError(
|
||||
f"Insufficient unique candidates for {language}/{category}/"
|
||||
f"{domain}: {selected} < {quotas[domain]}"
|
||||
)
|
||||
records.sort(key=lambda record: stable_key(record["id"]))
|
||||
validate_records(records, holdouts)
|
||||
return records, target_counts, excluded_holdout
|
||||
|
||||
|
||||
def serialize_records(records: Sequence[dict]) -> bytes:
|
||||
"""Serialize JSONL with stable keys and a final newline."""
|
||||
|
||||
return (
|
||||
"\n".join(
|
||||
json.dumps(record, ensure_ascii=False, sort_keys=True)
|
||||
for record in records
|
||||
)
|
||||
+ "\n"
|
||||
).encode("utf-8")
|
||||
|
||||
|
||||
def nested_counts(counter: Counter[tuple[str, ...]]) -> dict:
|
||||
"""Turn tuple-key counts into a stable nested JSON object."""
|
||||
|
||||
root: dict = {}
|
||||
for keys, count in sorted(counter.items()):
|
||||
node = root
|
||||
for key in keys[:-1]:
|
||||
node = node.setdefault(key, {})
|
||||
node[keys[-1]] = count
|
||||
return root
|
||||
|
||||
|
||||
def build_summary(
|
||||
records: Sequence[dict],
|
||||
target_counts: Counter[str],
|
||||
excluded_holdout: int,
|
||||
payload: bytes,
|
||||
) -> dict:
|
||||
"""Summarize all requested dimensions and the exact output artifact."""
|
||||
|
||||
by_language = Counter((record["language"],) for record in records)
|
||||
by_boundary_target_language = Counter(
|
||||
{
|
||||
tuple(key.split("|", 1)): count
|
||||
for key, count in target_counts.items()
|
||||
}
|
||||
)
|
||||
by_intent_language = Counter()
|
||||
for record in records:
|
||||
for intent in BOUNDARY_INTENTS:
|
||||
field = "replyable" if intent == "replyableMessage" else intent
|
||||
if record[field]:
|
||||
by_intent_language[(record["language"], intent)] += 1
|
||||
by_intent = Counter()
|
||||
for (_, intent), count in by_intent_language.items():
|
||||
by_intent[(intent,)] += count
|
||||
by_domain = Counter((record["domain"],) for record in records)
|
||||
by_family = Counter((record["templateFamily"],) for record in records)
|
||||
return {
|
||||
"schemaVersion": 1,
|
||||
"sourceRevision": SOURCE_REVISION,
|
||||
"recordCount": len(records),
|
||||
"counts": {
|
||||
"byLanguage": nested_counts(by_language),
|
||||
"byIntent": nested_counts(by_intent),
|
||||
"byIntentAndLanguage": nested_counts(by_intent_language),
|
||||
"byBoundaryTargetAndLanguage": nested_counts(
|
||||
by_boundary_target_language
|
||||
),
|
||||
"byDomain": nested_counts(by_domain),
|
||||
"byTemplateFamily": nested_counts(by_family),
|
||||
},
|
||||
"corpusSHA256": hashlib.sha256(payload).hexdigest(),
|
||||
"excludedHoldoutOverlap": excluded_holdout,
|
||||
}
|
||||
|
||||
|
||||
def write_corpus(
|
||||
output_path: Path,
|
||||
summary_path: Path,
|
||||
*,
|
||||
positive_per_intent_language: int = DEFAULT_POSITIVE_PER_INTENT_LANGUAGE,
|
||||
negative_per_intent_language: int = DEFAULT_NEGATIVE_PER_INTENT_LANGUAGE,
|
||||
holdout_paths: Iterable[Path] = (),
|
||||
) -> dict:
|
||||
"""Generate and write the corpus and summary files."""
|
||||
|
||||
holdouts = load_holdout_fingerprints(holdout_paths)
|
||||
records, target_counts, excluded = generate_records(
|
||||
positive_per_intent_language=positive_per_intent_language,
|
||||
negative_per_intent_language=negative_per_intent_language,
|
||||
holdout_fingerprints=holdouts,
|
||||
)
|
||||
payload = serialize_records(records)
|
||||
summary = build_summary(records, target_counts, excluded, payload)
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
summary_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_path.write_bytes(payload)
|
||||
summary_path.write_text(
|
||||
json.dumps(summary, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return summary
|
||||
|
||||
|
||||
def parse_arguments() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT_PATH)
|
||||
parser.add_argument("--summary-output", type=Path, default=DEFAULT_SUMMARY_PATH)
|
||||
parser.add_argument(
|
||||
"--holdout-directory",
|
||||
type=Path,
|
||||
default=OUTPUT_DIRECTORY,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--positive-per-intent-language",
|
||||
type=int,
|
||||
default=DEFAULT_POSITIVE_PER_INTENT_LANGUAGE,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--negative-per-intent-language",
|
||||
type=int,
|
||||
default=DEFAULT_NEGATIVE_PER_INTENT_LANGUAGE,
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
arguments = parse_arguments()
|
||||
holdout_paths = discover_holdout_paths(arguments.holdout_directory)
|
||||
summary = write_corpus(
|
||||
arguments.output,
|
||||
arguments.summary_output,
|
||||
positive_per_intent_language=arguments.positive_per_intent_language,
|
||||
negative_per_intent_language=arguments.negative_per_intent_language,
|
||||
holdout_paths=holdout_paths,
|
||||
)
|
||||
print(
|
||||
"V6_BOUNDARY_CORPUS "
|
||||
f"records={summary['recordCount']} "
|
||||
f"excludedHoldout={summary['excludedHoldoutOverlap']} "
|
||||
f"sha256={summary['corpusSHA256']}"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,607 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Merge blind three-state labels from three primary and two review models."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
INTENT_LABELS = (
|
||||
"task",
|
||||
"question",
|
||||
"invitation",
|
||||
"complaint",
|
||||
"scheduleNegotiation",
|
||||
"confirmationDecision",
|
||||
"followUpReminder",
|
||||
"blessing",
|
||||
"replyableMessage",
|
||||
"assistantCommand",
|
||||
"informationQuery",
|
||||
"systemNotification",
|
||||
)
|
||||
DOMAINS = {
|
||||
"finance",
|
||||
"travel",
|
||||
"calendar",
|
||||
"communication",
|
||||
"media",
|
||||
"smartHome",
|
||||
"shopping",
|
||||
"dining",
|
||||
"health",
|
||||
"weather",
|
||||
"accountService",
|
||||
"generalKnowledge",
|
||||
}
|
||||
DOMAIN_STATES = {*DOMAINS, "unknown"}
|
||||
LABEL_STATES = {"true", "false", "unknown"}
|
||||
SENTIMENT_STATES = {"positive", "neutral", "negative", "unknown"}
|
||||
PROMPT_VERSION = "clipboard-consensus-v6"
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def read_json_lines(path: Path) -> list[dict]:
|
||||
return [
|
||||
json.loads(line)
|
||||
for line in path.read_text(encoding="utf-8").splitlines()
|
||||
if line.strip()
|
||||
]
|
||||
|
||||
|
||||
def write_json_lines(path: Path, records: list[dict]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", encoding="utf-8") as handle:
|
||||
for record in records:
|
||||
handle.write(
|
||||
json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n"
|
||||
)
|
||||
|
||||
|
||||
def parse_labeler(value: str) -> tuple[str, Path]:
|
||||
name, separator, raw_path = value.partition("=")
|
||||
if not separator or not name or not raw_path:
|
||||
raise argparse.ArgumentTypeError("Expected LABELER=PATH")
|
||||
return name, Path(raw_path)
|
||||
|
||||
|
||||
def validate_record(record: dict, expected_ids: set[str]) -> dict:
|
||||
identifier = record.get("id")
|
||||
if identifier not in expected_ids:
|
||||
raise ValueError(f"Unexpected labeler record id: {identifier}")
|
||||
labels = record.get("labels")
|
||||
if not isinstance(labels, dict) or set(labels) != set(INTENT_LABELS):
|
||||
raise ValueError(f"Every intent label is required for {identifier}")
|
||||
if any(value not in LABEL_STATES for value in labels.values()):
|
||||
raise ValueError(f"Invalid three-state label for {identifier}")
|
||||
sentiment = record.get("sentiment")
|
||||
if sentiment not in SENTIMENT_STATES:
|
||||
raise ValueError(f"Invalid sentiment for {identifier}: {sentiment}")
|
||||
domain = record.get("domain")
|
||||
if domain not in DOMAIN_STATES:
|
||||
raise ValueError(f"Invalid domain for {identifier}: {domain}")
|
||||
if not isinstance(record.get("ambiguous"), bool):
|
||||
raise TypeError(f"Missing ambiguous flag for {identifier}")
|
||||
if not isinstance(record.get("quotedOrMeta"), bool):
|
||||
raise TypeError(f"Missing quotedOrMeta flag for {identifier}")
|
||||
confidence = record.get("confidence")
|
||||
if not isinstance(confidence, (int, float)) or not 0 <= confidence <= 1:
|
||||
raise ValueError(f"Invalid confidence for {identifier}: {confidence}")
|
||||
return {
|
||||
"id": identifier,
|
||||
"labels": {label: labels[label] for label in INTENT_LABELS},
|
||||
"sentiment": sentiment,
|
||||
"domain": domain,
|
||||
"ambiguous": record["ambiguous"],
|
||||
"quotedOrMeta": record["quotedOrMeta"],
|
||||
"confidence": round(float(confidence), 4),
|
||||
}
|
||||
|
||||
|
||||
def load_labelers(
|
||||
values: list[tuple[str, Path]],
|
||||
expected_ids: set[str],
|
||||
require_exact_ids: bool,
|
||||
) -> list[tuple[str, dict[str, dict]]]:
|
||||
labelers = []
|
||||
for name, path in values:
|
||||
records = [
|
||||
validate_record(record, expected_ids)
|
||||
for record in read_json_lines(path)
|
||||
]
|
||||
by_id = {record["id"]: record for record in records}
|
||||
if len(by_id) != len(records):
|
||||
raise ValueError(f"Labeler {name} contains duplicate ids")
|
||||
if require_exact_ids and set(by_id) != expected_ids:
|
||||
missing = expected_ids - set(by_id)
|
||||
extra = set(by_id) - expected_ids
|
||||
raise ValueError(
|
||||
f"Labeler {name} id mismatch: missing={len(missing)} "
|
||||
f"extra={len(extra)}"
|
||||
)
|
||||
labelers.append((name, by_id))
|
||||
return labelers
|
||||
|
||||
|
||||
def field_values(record: dict) -> dict[str, str]:
|
||||
return {
|
||||
**record["labels"],
|
||||
"sentiment": record["sentiment"],
|
||||
"domain": record["domain"],
|
||||
}
|
||||
|
||||
|
||||
def primary_review_ids(
|
||||
primary: list[tuple[str, dict[str, dict]]],
|
||||
queue_ids: set[str],
|
||||
) -> set[str]:
|
||||
review_ids = set()
|
||||
for identifier in queue_ids:
|
||||
records = [values[identifier] for _, values in primary]
|
||||
fields = [field_values(record) for record in records]
|
||||
unanimous = all(
|
||||
len({field[name] for field in fields}) == 1
|
||||
for name in (*INTENT_LABELS, "sentiment", "domain")
|
||||
)
|
||||
flags_clear = not any(
|
||||
record["ambiguous"] or record["quotedOrMeta"] for record in records
|
||||
)
|
||||
if not unanimous or not flags_clear:
|
||||
review_ids.add(identifier)
|
||||
return review_ids
|
||||
|
||||
|
||||
def prepare_review(arguments: argparse.Namespace) -> dict:
|
||||
queue = read_json_lines(arguments.queue)
|
||||
queue_by_id = {record["id"]: record for record in queue}
|
||||
if len(queue_by_id) != len(queue):
|
||||
raise ValueError("Queue contains duplicate ids")
|
||||
if len(arguments.primary) != 3:
|
||||
raise ValueError("Exactly three primary labelers are required")
|
||||
primary = load_labelers(arguments.primary, set(queue_by_id), True)
|
||||
review_ids = primary_review_ids(primary, set(queue_by_id))
|
||||
review_queue = [
|
||||
{
|
||||
"id": record["id"],
|
||||
"text": record["text"],
|
||||
"language": record["language"],
|
||||
}
|
||||
for record in queue
|
||||
if record["id"] in review_ids
|
||||
]
|
||||
write_json_lines(arguments.output, review_queue)
|
||||
report = {
|
||||
"schemaVersion": 2,
|
||||
"promptVersion": PROMPT_VERSION,
|
||||
"queueCount": len(queue),
|
||||
"queueSHA256": sha256_file(arguments.queue),
|
||||
"primaryUnanimousCount": len(queue) - len(review_queue),
|
||||
"reviewCount": len(review_queue),
|
||||
"primaryLabelers": [name for name, _ in primary],
|
||||
"primaryOutputSHA256": {
|
||||
name: sha256_file(path) for name, path in arguments.primary
|
||||
},
|
||||
}
|
||||
arguments.report.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return report
|
||||
|
||||
|
||||
def split_queue(arguments: argparse.Namespace) -> dict:
|
||||
queue = read_json_lines(arguments.queue)
|
||||
if arguments.chunk_size <= 0:
|
||||
raise ValueError("chunk-size must be positive")
|
||||
arguments.output_directory.mkdir(parents=True, exist_ok=True)
|
||||
output_paths = []
|
||||
for start in range(0, len(queue), arguments.chunk_size):
|
||||
index = len(output_paths) + 1
|
||||
path = arguments.output_directory / f"chunk-{index:03d}.jsonl"
|
||||
write_json_lines(path, queue[start : start + arguments.chunk_size])
|
||||
output_paths.append(path)
|
||||
report = {
|
||||
"schemaVersion": 2,
|
||||
"queueCount": len(queue),
|
||||
"queueSHA256": sha256_file(arguments.queue),
|
||||
"chunkSize": arguments.chunk_size,
|
||||
"chunkCount": len(output_paths),
|
||||
"chunks": [
|
||||
{
|
||||
"path": str(path),
|
||||
"records": len(read_json_lines(path)),
|
||||
"sha256": sha256_file(path),
|
||||
}
|
||||
for path in output_paths
|
||||
],
|
||||
}
|
||||
arguments.report.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return report
|
||||
|
||||
|
||||
def combine_labeler(arguments: argparse.Namespace) -> dict:
|
||||
queue = read_json_lines(arguments.queue)
|
||||
queue_ids = [record["id"] for record in queue]
|
||||
expected_ids = set(queue_ids)
|
||||
combined = []
|
||||
for path in arguments.input:
|
||||
combined.extend(read_json_lines(path))
|
||||
validated = [
|
||||
validate_record(record, expected_ids) for record in combined
|
||||
]
|
||||
by_id = {record["id"]: record for record in validated}
|
||||
if len(by_id) != len(validated):
|
||||
raise ValueError("Combined labeler outputs contain duplicate ids")
|
||||
if set(by_id) != expected_ids:
|
||||
raise ValueError(
|
||||
"Combined labeler output ids do not match the source queue"
|
||||
)
|
||||
ordered = [by_id[identifier] for identifier in queue_ids]
|
||||
write_json_lines(arguments.output, ordered)
|
||||
report = {
|
||||
"schemaVersion": 2,
|
||||
"queueCount": len(queue),
|
||||
"queueSHA256": sha256_file(arguments.queue),
|
||||
"inputCount": len(arguments.input),
|
||||
"outputCount": len(ordered),
|
||||
"outputSHA256": sha256_file(arguments.output),
|
||||
}
|
||||
arguments.report.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return report
|
||||
|
||||
|
||||
def stable_split(identifier: str) -> str:
|
||||
bucket = int.from_bytes(hashlib.sha256(identifier.encode()).digest()[:8], "big")
|
||||
bucket %= 100
|
||||
if bucket < 80:
|
||||
return "silverTrain"
|
||||
if bucket < 90:
|
||||
return "silverCalibration"
|
||||
return "silverAcceptance"
|
||||
|
||||
|
||||
def field_consensus(
|
||||
records: list[dict],
|
||||
field: str,
|
||||
required_votes: int,
|
||||
) -> tuple[str, dict[str, int]]:
|
||||
values = [
|
||||
field_values(record)[field]
|
||||
for record in records
|
||||
if field_values(record)[field] != "unknown"
|
||||
]
|
||||
votes = Counter(values)
|
||||
if not votes:
|
||||
return "unknown", {}
|
||||
value, count = votes.most_common(1)[0]
|
||||
if count < required_votes:
|
||||
return "unknown", dict(sorted(votes.items()))
|
||||
return value, dict(sorted(votes.items()))
|
||||
|
||||
|
||||
def fleiss_kappa_for_field(
|
||||
labelers: list[tuple[str, dict[str, dict]]],
|
||||
identifiers: list[str],
|
||||
field: str,
|
||||
) -> float:
|
||||
categories = (
|
||||
sorted(SENTIMENT_STATES)
|
||||
if field == "sentiment"
|
||||
else sorted(DOMAIN_STATES)
|
||||
if field == "domain"
|
||||
else sorted(LABEL_STATES)
|
||||
)
|
||||
category_totals = Counter()
|
||||
item_agreements = []
|
||||
rater_count = len(labelers)
|
||||
for identifier in identifiers:
|
||||
votes = Counter(
|
||||
field_values(records[identifier])[field]
|
||||
for _, records in labelers
|
||||
)
|
||||
category_totals.update(votes)
|
||||
item_agreements.append(
|
||||
sum(count * (count - 1) for count in votes.values())
|
||||
/ (rater_count * (rater_count - 1))
|
||||
)
|
||||
if not item_agreements:
|
||||
return 0
|
||||
observed = sum(item_agreements) / len(item_agreements)
|
||||
total = sum(category_totals.values())
|
||||
expected = sum(
|
||||
(category_totals[category] / total) ** 2 for category in categories
|
||||
)
|
||||
if math.isclose(expected, 1):
|
||||
return 1
|
||||
return round((observed - expected) / (1 - expected), 4)
|
||||
|
||||
|
||||
def training_record(
|
||||
queue_record: dict,
|
||||
states: dict[str, str],
|
||||
tier: str,
|
||||
votes: dict,
|
||||
labelers: list[tuple[str, dict[str, dict]]],
|
||||
) -> dict:
|
||||
split = stable_split(queue_record["id"])
|
||||
known_labels = [
|
||||
label
|
||||
for label in (*INTENT_LABELS, "sentiment", "domain")
|
||||
if states[label] != "unknown"
|
||||
]
|
||||
return {
|
||||
"id": f"consensus-v2-{queue_record['id']}",
|
||||
"sourceRecordID": queue_record["id"],
|
||||
"text": queue_record["text"],
|
||||
"language": queue_record["language"],
|
||||
"family": "consensus_v2",
|
||||
"split": split,
|
||||
**{
|
||||
("replyable" if label == "replyableMessage" else label): (
|
||||
states[label] == "true"
|
||||
)
|
||||
for label in INTENT_LABELS
|
||||
},
|
||||
"sentiment": (
|
||||
states["sentiment"]
|
||||
if states["sentiment"] != "unknown"
|
||||
else "neutral"
|
||||
),
|
||||
"domain": (
|
||||
states["domain"] if states["domain"] != "unknown" else None
|
||||
),
|
||||
"knownLabels": known_labels,
|
||||
"labelQualityTier": tier,
|
||||
"sampleWeight": 1.0 if tier == "A" else 0.65,
|
||||
"promptVersion": PROMPT_VERSION,
|
||||
"modelVotes": votes,
|
||||
"labelers": [name for name, _ in labelers],
|
||||
}
|
||||
|
||||
|
||||
def merge(arguments: argparse.Namespace) -> dict:
|
||||
queue = read_json_lines(arguments.queue)
|
||||
queue_by_id = {record["id"]: record for record in queue}
|
||||
queue_ids = set(queue_by_id)
|
||||
if len(queue_by_id) != len(queue):
|
||||
raise ValueError("Queue contains duplicate ids")
|
||||
if len(arguments.primary) != 3 or len(arguments.reviewer) != 2:
|
||||
raise ValueError("Three primary and two review labelers are required")
|
||||
primary = load_labelers(arguments.primary, queue_ids, True)
|
||||
review_ids = primary_review_ids(primary, queue_ids)
|
||||
reviewers = load_labelers(arguments.reviewer, review_ids, True)
|
||||
tier_a = []
|
||||
tier_b = []
|
||||
human_review = []
|
||||
all_labelers = primary + reviewers
|
||||
field_names = (*INTENT_LABELS, "sentiment", "domain")
|
||||
for identifier in sorted(queue_ids):
|
||||
queue_record = queue_by_id[identifier]
|
||||
primary_records = [records[identifier] for _, records in primary]
|
||||
if identifier not in review_ids:
|
||||
states = {
|
||||
field: field_values(primary_records[0])[field]
|
||||
for field in field_names
|
||||
}
|
||||
votes = {
|
||||
field: {states[field]: 3}
|
||||
for field in field_names
|
||||
}
|
||||
tier_a.append(
|
||||
training_record(
|
||||
queue_record,
|
||||
states,
|
||||
"A",
|
||||
votes,
|
||||
primary,
|
||||
)
|
||||
)
|
||||
continue
|
||||
combined_records = primary_records + [
|
||||
records[identifier] for _, records in reviewers
|
||||
]
|
||||
states = {}
|
||||
votes = {}
|
||||
for field in field_names:
|
||||
states[field], votes[field] = field_consensus(
|
||||
combined_records,
|
||||
field,
|
||||
4,
|
||||
)
|
||||
ambiguous_votes = sum(
|
||||
record["ambiguous"] for record in combined_records
|
||||
)
|
||||
quoted_votes = sum(
|
||||
record["quotedOrMeta"] for record in combined_records
|
||||
)
|
||||
unresolved = [
|
||||
field
|
||||
for field, value in states.items()
|
||||
if value == "unknown" and votes[field]
|
||||
]
|
||||
positive_intents = any(states[label] == "true" for label in INTENT_LABELS)
|
||||
if ambiguous_votes >= 2:
|
||||
unresolved.append("ambiguous")
|
||||
if quoted_votes >= 2 and positive_intents:
|
||||
unresolved.append("quotedOrMeta")
|
||||
if unresolved:
|
||||
human_review.append(
|
||||
{
|
||||
"id": identifier,
|
||||
"text": queue_record["text"],
|
||||
"language": queue_record["language"],
|
||||
"unresolvedFields": sorted(set(unresolved)),
|
||||
"modelVotes": votes,
|
||||
"ambiguousVotes": ambiguous_votes,
|
||||
"quotedOrMetaVotes": quoted_votes,
|
||||
"resolution": None,
|
||||
"reviewer": None,
|
||||
"reason": None,
|
||||
}
|
||||
)
|
||||
continue
|
||||
tier_b.append(
|
||||
training_record(
|
||||
queue_record,
|
||||
states,
|
||||
"B",
|
||||
votes,
|
||||
all_labelers,
|
||||
)
|
||||
)
|
||||
accepted = tier_a + tier_b
|
||||
write_json_lines(arguments.tier_a, tier_a)
|
||||
write_json_lines(arguments.tier_b, tier_b)
|
||||
write_json_lines(arguments.accepted, accepted)
|
||||
write_json_lines(arguments.human_review, human_review)
|
||||
identifiers = sorted(queue_ids)
|
||||
kappa_by_field = {
|
||||
field: fleiss_kappa_for_field(primary, identifiers, field)
|
||||
for field in field_names
|
||||
}
|
||||
languages = sorted({record["language"] for record in queue})
|
||||
kappa_by_language = {
|
||||
language: {
|
||||
field: fleiss_kappa_for_field(
|
||||
primary,
|
||||
sorted(
|
||||
identifier
|
||||
for identifier, record in queue_by_id.items()
|
||||
if record["language"] == language
|
||||
),
|
||||
field,
|
||||
)
|
||||
for field in field_names
|
||||
}
|
||||
for language in languages
|
||||
}
|
||||
kappa_gate_passed = all(
|
||||
value >= 0.8
|
||||
for values in kappa_by_language.values()
|
||||
for value in values.values()
|
||||
)
|
||||
report = {
|
||||
"schemaVersion": 2,
|
||||
"promptVersion": PROMPT_VERSION,
|
||||
"queueCount": len(queue),
|
||||
"queueSHA256": sha256_file(arguments.queue),
|
||||
"tierACount": len(tier_a),
|
||||
"tierBCount": len(tier_b),
|
||||
"humanReviewCount": len(human_review),
|
||||
"acceptanceRate": round(len(accepted) / max(len(queue), 1), 4),
|
||||
"primaryLabelers": [name for name, _ in primary],
|
||||
"reviewLabelers": [name for name, _ in reviewers],
|
||||
"labelerOutputSHA256": {
|
||||
name: sha256_file(path)
|
||||
for name, path in (*arguments.primary, *arguments.reviewer)
|
||||
},
|
||||
"primaryFleissKappaByField": kappa_by_field,
|
||||
"primaryFleissKappaByLanguageAndField": kappa_by_language,
|
||||
"scaleUpKappaThreshold": 0.8,
|
||||
"eligibleForScaleUp": kappa_gate_passed,
|
||||
"acceptedIntentPositiveCounts": {
|
||||
label: sum(
|
||||
record["replyable" if label == "replyableMessage" else label]
|
||||
for record in accepted
|
||||
)
|
||||
for label in INTENT_LABELS
|
||||
},
|
||||
"acceptedLanguageCounts": dict(
|
||||
sorted(Counter(record["language"] for record in accepted).items())
|
||||
),
|
||||
"splitCounts": dict(
|
||||
sorted(Counter(record["split"] for record in accepted).items())
|
||||
),
|
||||
}
|
||||
arguments.report.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return report
|
||||
|
||||
|
||||
def parser() -> argparse.ArgumentParser:
|
||||
root = argparse.ArgumentParser()
|
||||
commands = root.add_subparsers(dest="command", required=True)
|
||||
review = commands.add_parser("prepare-review")
|
||||
review.add_argument("--queue", type=Path, required=True)
|
||||
review.add_argument(
|
||||
"--primary",
|
||||
action="append",
|
||||
type=parse_labeler,
|
||||
required=True,
|
||||
)
|
||||
review.add_argument("--output", type=Path, required=True)
|
||||
review.add_argument("--report", type=Path, required=True)
|
||||
review.set_defaults(handler=prepare_review)
|
||||
|
||||
split = commands.add_parser("split-queue")
|
||||
split.add_argument("--queue", type=Path, required=True)
|
||||
split.add_argument("--output-directory", type=Path, required=True)
|
||||
split.add_argument("--chunk-size", type=int, default=100)
|
||||
split.add_argument("--report", type=Path, required=True)
|
||||
split.set_defaults(handler=split_queue)
|
||||
|
||||
combine = commands.add_parser("combine-labeler")
|
||||
combine.add_argument("--queue", type=Path, required=True)
|
||||
combine.add_argument(
|
||||
"--input",
|
||||
action="append",
|
||||
type=Path,
|
||||
required=True,
|
||||
)
|
||||
combine.add_argument("--output", type=Path, required=True)
|
||||
combine.add_argument("--report", type=Path, required=True)
|
||||
combine.set_defaults(handler=combine_labeler)
|
||||
|
||||
merge_parser = commands.add_parser("merge")
|
||||
merge_parser.add_argument("--queue", type=Path, required=True)
|
||||
merge_parser.add_argument(
|
||||
"--primary",
|
||||
action="append",
|
||||
type=parse_labeler,
|
||||
required=True,
|
||||
)
|
||||
merge_parser.add_argument(
|
||||
"--reviewer",
|
||||
action="append",
|
||||
type=parse_labeler,
|
||||
required=True,
|
||||
)
|
||||
merge_parser.add_argument("--tier-a", type=Path, required=True)
|
||||
merge_parser.add_argument("--tier-b", type=Path, required=True)
|
||||
merge_parser.add_argument("--accepted", type=Path, required=True)
|
||||
merge_parser.add_argument("--human-review", type=Path, required=True)
|
||||
merge_parser.add_argument("--report", type=Path, required=True)
|
||||
merge_parser.set_defaults(handler=merge)
|
||||
return root
|
||||
|
||||
|
||||
def main() -> None:
|
||||
arguments = parser().parse_args()
|
||||
report = arguments.handler(arguments)
|
||||
print(
|
||||
f"CONSENSUS_V2_{arguments.command.upper().replace('-', '_')} "
|
||||
+ " ".join(f"{key}={value}" for key, value in report.items() if key.endswith("Count"))
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,137 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Prepare a leakage-free Apple NL corpus with product-scale evaluation splits."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import unicodedata
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
EVALUATION_SPLITS = {"validation", "test", "golden"}
|
||||
|
||||
|
||||
def read_json_lines(path: Path) -> list[dict]:
|
||||
return [
|
||||
json.loads(line)
|
||||
for line in path.read_text(encoding="utf-8").splitlines()
|
||||
if line.strip()
|
||||
]
|
||||
|
||||
|
||||
def fingerprint(text: str) -> str:
|
||||
normalized = unicodedata.normalize("NFKC", text)
|
||||
return " ".join(normalized.casefold().split())
|
||||
|
||||
|
||||
def prepare(
|
||||
training_records: list[dict],
|
||||
product_records: list[dict],
|
||||
) -> tuple[list[dict], dict]:
|
||||
if any(record.get("split") != "train" for record in training_records):
|
||||
raise ValueError("Training source must contain only split=train records")
|
||||
|
||||
evaluation_records = [
|
||||
record for record in product_records if record.get("split") in EVALUATION_SPLITS
|
||||
]
|
||||
if not evaluation_records:
|
||||
raise ValueError("Product corpus has no evaluation records")
|
||||
|
||||
evaluation_fingerprints = {
|
||||
fingerprint(record["text"]) for record in evaluation_records
|
||||
}
|
||||
evaluation_ids = {record["id"] for record in evaluation_records}
|
||||
filtered_training = [
|
||||
record
|
||||
for record in training_records
|
||||
if record["id"] not in evaluation_ids
|
||||
and fingerprint(record["text"]) not in evaluation_fingerprints
|
||||
]
|
||||
records = filtered_training + evaluation_records
|
||||
|
||||
identifiers = [record["id"] for record in records]
|
||||
if len(identifiers) != len(set(identifiers)):
|
||||
raise ValueError("Duplicate record ids remain after filtering")
|
||||
|
||||
training_fingerprints = {
|
||||
fingerprint(record["text"]) for record in filtered_training
|
||||
}
|
||||
if training_fingerprints & evaluation_fingerprints:
|
||||
raise ValueError("Training and evaluation text overlap remains after filtering")
|
||||
|
||||
report = {
|
||||
"schemaVersion": 1,
|
||||
"originalTrainingCount": len(training_records),
|
||||
"filteredTrainingCount": len(filtered_training),
|
||||
"excludedTrainingOverlapCount": len(training_records)
|
||||
- len(filtered_training),
|
||||
"evaluationCount": len(evaluation_records),
|
||||
"recordCount": len(records),
|
||||
"splitCounts": dict(
|
||||
sorted(Counter(record["split"] for record in records).items())
|
||||
),
|
||||
"languageCounts": dict(
|
||||
sorted(Counter(record["language"] for record in records).items())
|
||||
),
|
||||
"evaluationOverlapCount": 0,
|
||||
}
|
||||
return records, report
|
||||
|
||||
|
||||
def write_outputs(
|
||||
records: list[dict],
|
||||
report: dict,
|
||||
output_path: Path,
|
||||
report_path: Path,
|
||||
) -> dict:
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
payload = "".join(
|
||||
json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n"
|
||||
for record in records
|
||||
)
|
||||
output_path.write_text(payload, encoding="utf-8")
|
||||
final_report = {
|
||||
**report,
|
||||
"corpusSHA256": hashlib.sha256(payload.encode()).hexdigest(),
|
||||
}
|
||||
report_path.write_text(
|
||||
json.dumps(final_report, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return final_report
|
||||
|
||||
|
||||
def parser() -> argparse.ArgumentParser:
|
||||
root = argparse.ArgumentParser(description=__doc__)
|
||||
root.add_argument("--train", type=Path, required=True)
|
||||
root.add_argument("--product-corpus", type=Path, required=True)
|
||||
root.add_argument("--output", type=Path, required=True)
|
||||
root.add_argument("--report", type=Path, required=True)
|
||||
return root
|
||||
|
||||
|
||||
def main() -> None:
|
||||
arguments = parser().parse_args()
|
||||
records, report = prepare(
|
||||
read_json_lines(arguments.train),
|
||||
read_json_lines(arguments.product_corpus),
|
||||
)
|
||||
final_report = write_outputs(
|
||||
records,
|
||||
report,
|
||||
arguments.output,
|
||||
arguments.report,
|
||||
)
|
||||
print(
|
||||
"APPLE_NL_HARDENING_CORPUS "
|
||||
f"records={final_report['recordCount']} "
|
||||
f"excludedOverlap={final_report['excludedTrainingOverlapCount']} "
|
||||
f"sha256={final_report['corpusSHA256']}"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,306 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Prepare a blind, double-annotation queue for the blessing benchmark."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import unicodedata
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
SEED = 20260828
|
||||
OUTPUT_DIRECTORY = Path("ModelTraining/ClipboardSemantics/BlessingBenchmark")
|
||||
SOURCE_PATH = Path(
|
||||
"ModelTraining/ClipboardSemantics/comprehensive-online-holdout-corpus.jsonl"
|
||||
)
|
||||
PII_PATTERN = re.compile(
|
||||
r"(?:[\w.+-]+@[\w.-]+\.\w+)|(?:\+?\d[\d ()-]{8,}\d)|"
|
||||
r"(?:\b\d{3}-\d{2}-\d{4}\b)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
EXPLICIT_PATTERN = re.compile(
|
||||
r"(?:祝|愿你|愿您|愿他|愿她|愿大家|恭喜|祝贺|生日快乐|"
|
||||
r"新年快乐|一路平安|一路顺风|康复|前程|平安|安康|如意|好梦|"
|
||||
r"希望.{0,40}(?:快乐|幸福|平安|顺利|康复|成功|健康)|"
|
||||
r"wish|hope you|may you|may your|congrat|happy birthday|"
|
||||
r"happy new year|good luck|best wishes|get well|safe travel|"
|
||||
r"sweet dream|peace and happiness|future success)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
BOUNDARY_PATTERN = re.compile(
|
||||
r"(?:祝福语|祝福模板|祝福文案|谢谢.{0,30}祝福|感谢.{0,30}祝福|"
|
||||
r"收到.{0,30}祝福|庆祝|庆功|引用.{0,20}(?:祝|愿|恭喜)|"
|
||||
r"怎么.{0,20}(?:祝|生日快乐)|如何.{0,20}(?:祝|生日快乐)|"
|
||||
r"template|thanks?.{0,64}(?:wish|wishes|congratulations)|"
|
||||
r"celebrat|quotes?.{0,32}(?:wish|congratulat)|"
|
||||
r"how to write.{0,32}(?:wish|greeting))",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
PLAIN_GREETING_PATTERN = re.compile(
|
||||
r"^(?:你好|您好|早上好|中午好|下午好|晚上好|晚安|好久不见|"
|
||||
r"hello|good morning|good afternoon|good evening|long time no see)"
|
||||
r"[！!。,.，~～]*$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def parse_arguments() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--source", type=Path, default=SOURCE_PATH)
|
||||
parser.add_argument("--output-directory", type=Path, default=OUTPUT_DIRECTORY)
|
||||
parser.add_argument("--seed", type=int, default=SEED)
|
||||
parser.add_argument("--chinese-records", type=int, default=3_000)
|
||||
parser.add_argument("--english-records", type=int, default=1_500)
|
||||
parser.add_argument(
|
||||
"--training-corpus",
|
||||
action="append",
|
||||
default=[],
|
||||
type=Path,
|
||||
help="Additional JSONL whose text must not overlap the review queue.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def normalized_text(value: str) -> str:
|
||||
value = unicodedata.normalize("NFKC", value.replace("\u0000", " "))
|
||||
return " ".join(value.split()).strip()
|
||||
|
||||
|
||||
def fingerprint(value: str) -> str:
|
||||
return normalized_text(value).casefold()
|
||||
|
||||
|
||||
def file_sha256(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def load_records(path: Path) -> list[dict]:
|
||||
return [
|
||||
json.loads(line)
|
||||
for line in path.read_text(encoding="utf-8").splitlines()
|
||||
if line.strip()
|
||||
]
|
||||
|
||||
|
||||
def protected_fingerprints(paths: list[Path]) -> set[str]:
|
||||
values: set[str] = set()
|
||||
for path in paths:
|
||||
for record_value in load_records(path):
|
||||
values.add(fingerprint(record_value["text"]))
|
||||
return values
|
||||
|
||||
|
||||
def selection_stratum(record_value: dict) -> str:
|
||||
text = normalized_text(record_value["text"])
|
||||
if BOUNDARY_PATTERN.search(text) or PLAIN_GREETING_PATTERN.fullmatch(text):
|
||||
return "boundary_candidate"
|
||||
if EXPLICIT_PATTERN.search(text):
|
||||
return "explicit_candidate"
|
||||
if record_value.get("blessing"):
|
||||
return "weak_positive_candidate"
|
||||
if record_value.get("sentiment") == "positive":
|
||||
return "positive_language_boundary"
|
||||
return "natural_negative"
|
||||
|
||||
|
||||
def stable_priority(record_value: dict, seed: int, salt: str) -> bytes:
|
||||
return hashlib.sha256(
|
||||
f"{seed}|{salt}|{record_value['id']}".encode()
|
||||
).digest()
|
||||
|
||||
|
||||
def select_language(
|
||||
records: list[dict],
|
||||
*,
|
||||
language: str,
|
||||
target: int,
|
||||
seed: int,
|
||||
protected: set[str],
|
||||
) -> list[dict]:
|
||||
candidates: list[dict] = []
|
||||
seen: set[str] = set()
|
||||
for record_value in records:
|
||||
if record_value.get("language") != language:
|
||||
continue
|
||||
text = normalized_text(record_value.get("text") or "")
|
||||
text_key = fingerprint(text)
|
||||
if (
|
||||
not 2 <= len(text) <= 500
|
||||
or PII_PATTERN.search(text)
|
||||
or text_key in protected
|
||||
or text_key in seen
|
||||
):
|
||||
continue
|
||||
seen.add(text_key)
|
||||
candidate = dict(record_value)
|
||||
candidate["_normalizedText"] = text
|
||||
candidate["_stratum"] = selection_stratum(record_value)
|
||||
candidates.append(candidate)
|
||||
|
||||
fractions = {
|
||||
"explicit_candidate": 0.30,
|
||||
"boundary_candidate": 0.25,
|
||||
"weak_positive_candidate": 0.10,
|
||||
"positive_language_boundary": 0.15,
|
||||
"natural_negative": 0.20,
|
||||
}
|
||||
selected: list[dict] = []
|
||||
selected_ids: set[str] = set()
|
||||
remaining = target
|
||||
for index, (stratum, fraction) in enumerate(fractions.items()):
|
||||
desired = target - len(selected) if index == len(fractions) - 1 else round(
|
||||
target * fraction
|
||||
)
|
||||
values = sorted(
|
||||
(
|
||||
value
|
||||
for value in candidates
|
||||
if value["_stratum"] == stratum
|
||||
),
|
||||
key=lambda value: stable_priority(value, seed, stratum),
|
||||
)
|
||||
for value in values[:desired]:
|
||||
selected.append(value)
|
||||
selected_ids.add(value["id"])
|
||||
remaining = target - len(selected)
|
||||
|
||||
if remaining:
|
||||
fillers = sorted(
|
||||
(value for value in candidates if value["id"] not in selected_ids),
|
||||
key=lambda value: stable_priority(value, seed, "fill"),
|
||||
)
|
||||
selected.extend(fillers[:remaining])
|
||||
if len(selected) != target:
|
||||
raise RuntimeError(
|
||||
f"Only selected {len(selected)}/{target} review records for {language}"
|
||||
)
|
||||
return selected
|
||||
|
||||
|
||||
def write_jsonl(path: Path, records: list[dict]) -> None:
|
||||
path.write_text(
|
||||
"\n".join(
|
||||
json.dumps(record_value, ensure_ascii=False, sort_keys=True)
|
||||
for record_value in records
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
arguments = parse_arguments()
|
||||
if arguments.chinese_records < 100 or arguments.english_records < 100:
|
||||
raise ValueError("Each language requires at least 100 review records")
|
||||
|
||||
source_records = load_records(arguments.source)
|
||||
protected = protected_fingerprints(arguments.training_corpus)
|
||||
selected = select_language(
|
||||
source_records,
|
||||
language="zh-Hans",
|
||||
target=arguments.chinese_records,
|
||||
seed=arguments.seed,
|
||||
protected=protected,
|
||||
) + select_language(
|
||||
source_records,
|
||||
language="en",
|
||||
target=arguments.english_records,
|
||||
seed=arguments.seed,
|
||||
protected=protected,
|
||||
)
|
||||
|
||||
output_directory = arguments.output_directory
|
||||
output_directory.mkdir(parents=True, exist_ok=True)
|
||||
queue: list[dict] = []
|
||||
provenance: list[dict] = []
|
||||
annotation_template: list[dict] = []
|
||||
for index, source in enumerate(
|
||||
sorted(selected, key=lambda value: stable_priority(value, arguments.seed, "queue")),
|
||||
start=1,
|
||||
):
|
||||
review_id = f"blessing-review-{index:05d}"
|
||||
queue.append(
|
||||
{
|
||||
"id": review_id,
|
||||
"text": source["_normalizedText"],
|
||||
"language": source["language"],
|
||||
"annotationStatus": "unreviewed",
|
||||
}
|
||||
)
|
||||
provenance.append(
|
||||
{
|
||||
"id": review_id,
|
||||
"sourceRecordID": source["id"],
|
||||
"sourceDataset": source.get("sourceDataset"),
|
||||
"sourceLicense": source.get("sourceLicense"),
|
||||
"sourceURL": source.get("sourceURL"),
|
||||
"selectionStratum": source["_stratum"],
|
||||
"previousWeakLabel": bool(source.get("blessing")),
|
||||
}
|
||||
)
|
||||
annotation_template.append(
|
||||
{
|
||||
"id": review_id,
|
||||
"label": None,
|
||||
"boundaryCategory": None,
|
||||
"confidence": None,
|
||||
"notes": "",
|
||||
}
|
||||
)
|
||||
|
||||
queue_path = output_directory / "review-queue.jsonl"
|
||||
provenance_path = output_directory / "sealed-provenance.jsonl"
|
||||
annotator_a_path = output_directory / "annotator-a.jsonl"
|
||||
annotator_b_path = output_directory / "annotator-b.jsonl"
|
||||
write_jsonl(queue_path, queue)
|
||||
write_jsonl(provenance_path, provenance)
|
||||
write_jsonl(annotator_a_path, annotation_template)
|
||||
write_jsonl(annotator_b_path, annotation_template)
|
||||
|
||||
manifest = {
|
||||
"schemaVersion": 1,
|
||||
"seed": arguments.seed,
|
||||
"status": "awaiting-double-human-annotation",
|
||||
"humanReviewComplete": False,
|
||||
"policy": (
|
||||
"Evaluation-only queue derived from the frozen comprehensive holdout. "
|
||||
"Never merge these records into training."
|
||||
),
|
||||
"records": len(queue),
|
||||
"languages": dict(Counter(value["language"] for value in queue)),
|
||||
"selectionStrata": dict(
|
||||
Counter(value["selectionStratum"] for value in provenance)
|
||||
),
|
||||
"sourceDatasets": dict(
|
||||
Counter(value["sourceDataset"] for value in provenance)
|
||||
),
|
||||
"validation": {
|
||||
"duplicateNormalizedTexts": (
|
||||
len(queue)
|
||||
- len({fingerprint(value["text"]) for value in queue})
|
||||
),
|
||||
"configuredTrainingOverlap": sum(
|
||||
fingerprint(value["text"]) in protected for value in queue
|
||||
),
|
||||
"containsDetectedPII": any(
|
||||
PII_PATTERN.search(value["text"]) for value in queue
|
||||
),
|
||||
},
|
||||
"artifacts": {
|
||||
"reviewQueueSHA256": file_sha256(queue_path),
|
||||
"sealedProvenanceSHA256": file_sha256(provenance_path),
|
||||
},
|
||||
}
|
||||
(output_directory / "manifest.json").write_text(
|
||||
json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,3 +1,6 @@
|
||||
ijson==3.5.1
|
||||
numpy==2.4.4
|
||||
opencc-python-reimplemented==0.1.7
|
||||
pyarrow==25.0.1
|
||||
scikit-learn==1.9.0
|
||||
scipy==1.18.1
|
||||
|
||||
@@ -37,7 +37,11 @@ INTENTS = (
|
||||
"followUpReminder",
|
||||
"blessing",
|
||||
"replyableMessage",
|
||||
"assistantCommand",
|
||||
"informationQuery",
|
||||
"systemNotification",
|
||||
)
|
||||
LEGACY_IMPLICITLY_KNOWN_INTENTS = frozenset(INTENTS[:9])
|
||||
OUTPUT_DIRECTORY = Path("ModelTraining/ClipboardSemantics/IterativeResearch")
|
||||
BASE_CORPUS = Path(
|
||||
"ModelTraining/ClipboardSemantics/clipboard_semantic_corpus.jsonl"
|
||||
@@ -282,7 +286,7 @@ def record_label(record: dict[str, Any], intent: str) -> bool:
|
||||
def is_known(record: dict[str, Any], intent: str) -> bool:
|
||||
known = record.get("knownLabels")
|
||||
if known is None:
|
||||
return True
|
||||
return intent in LEGACY_IMPLICITLY_KNOWN_INTENTS
|
||||
return intent in known or (
|
||||
intent == "replyableMessage" and "replyable" in known
|
||||
)
|
||||
@@ -449,6 +453,7 @@ def sample_weight(
|
||||
weight = 1.0
|
||||
if record.get("knownLabels") is not None:
|
||||
weight *= configuration.external_weight
|
||||
weight *= float(record.get("sampleWeight", 1.0))
|
||||
if record.get("_augmentation"):
|
||||
weight *= 0.60
|
||||
family = str(record.get("family", "")).casefold()
|
||||
@@ -498,13 +503,20 @@ def calibrated_thresholds(
|
||||
languages = np.array([record["language"] for record in records])
|
||||
result: dict[str, dict[str, Any]] = {}
|
||||
for intent in INTENTS:
|
||||
known_mask = np.array(
|
||||
[is_known(record, intent) for record in records],
|
||||
dtype=bool,
|
||||
)
|
||||
expected = np.array(
|
||||
[record_label(record, intent) for record in records], dtype=bool
|
||||
)
|
||||
selection = select_threshold(expected, probabilities[intent])
|
||||
selection = select_threshold(
|
||||
expected[known_mask],
|
||||
probabilities[intent][known_mask],
|
||||
)
|
||||
by_language = {}
|
||||
for language in sorted(set(languages)):
|
||||
mask = languages == language
|
||||
mask = (languages == language) & known_mask
|
||||
positives = int(np.sum(expected[mask]))
|
||||
negatives = int(np.sum(~expected[mask]))
|
||||
if positives < 20 or negatives < 20:
|
||||
@@ -574,11 +586,15 @@ def metrics_for_records(
|
||||
predictions = runtime_predictions(records, probabilities, thresholds)
|
||||
per_intent = {}
|
||||
for intent in INTENTS:
|
||||
known_mask = np.array(
|
||||
[is_known(record, intent) for record in records],
|
||||
dtype=bool,
|
||||
)
|
||||
expected = np.array(
|
||||
[record_label(record, intent) for record in records], dtype=bool
|
||||
)
|
||||
counts = BinaryCounts()
|
||||
counts.update(expected, predictions[intent])
|
||||
counts.update(expected[known_mask], predictions[intent][known_mask])
|
||||
per_intent[intent] = counts.metrics()
|
||||
return aggregate_metrics(per_intent)
|
||||
|
||||
@@ -819,14 +835,21 @@ def batched_evaluation(
|
||||
probabilities = prediction_probabilities(matrix, models)
|
||||
predictions = runtime_predictions(batch, probabilities, thresholds)
|
||||
for intent in INTENTS:
|
||||
known_mask = np.array(
|
||||
[is_known(record, intent) for record in batch],
|
||||
dtype=bool,
|
||||
)
|
||||
expected = np.array(
|
||||
[record_label(record, intent) for record in batch], dtype=bool
|
||||
)
|
||||
counts[intent].update(expected, predictions[intent])
|
||||
counts[intent].update(
|
||||
expected[known_mask],
|
||||
predictions[intent][known_mask],
|
||||
)
|
||||
for language in {record["language"] for record in batch}:
|
||||
mask = np.array(
|
||||
[record["language"] == language for record in batch]
|
||||
)
|
||||
) & known_mask
|
||||
by_language[language][intent].update(
|
||||
expected[mask], predictions[intent][mask]
|
||||
)
|
||||
@@ -843,7 +866,7 @@ def batched_evaluation(
|
||||
== source
|
||||
for record in batch
|
||||
]
|
||||
)
|
||||
) & known_mask
|
||||
by_source[source][intent].update(
|
||||
expected[mask], predictions[intent][mask]
|
||||
)
|
||||
|
||||
@@ -0,0 +1,268 @@
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
import adjudicate_consensus_conflicts as adjudication
|
||||
|
||||
|
||||
class AdjudicationTests(unittest.TestCase):
|
||||
def test_prepare_adds_all_product_policy_fields(self):
|
||||
with tempfile.TemporaryDirectory() as raw_directory:
|
||||
directory = Path(raw_directory)
|
||||
conflicts = directory / "conflicts.jsonl"
|
||||
self._write(
|
||||
conflicts,
|
||||
[
|
||||
{
|
||||
"id": "one",
|
||||
"text": "Could you send the report?",
|
||||
"language": "en",
|
||||
"unresolvedFields": ["sentiment"],
|
||||
}
|
||||
],
|
||||
)
|
||||
queue = directory / "queue.jsonl"
|
||||
report = adjudication.prepare(
|
||||
SimpleNamespace(
|
||||
conflicts=conflicts,
|
||||
queue=queue,
|
||||
chunk_directory=directory / "chunks",
|
||||
chunk_size=10,
|
||||
report=directory / "report.json",
|
||||
include_product_policy_fields=True,
|
||||
)
|
||||
)
|
||||
|
||||
fields = self._read(queue)[0]["unresolvedFields"]
|
||||
self.assertTrue(set(adjudication.PRODUCT_POLICY_FIELDS) <= set(fields))
|
||||
self.assertIn("sentiment", fields)
|
||||
self.assertTrue(report["includesProductPolicyFields"])
|
||||
|
||||
def test_requires_evidence_from_input_text(self):
|
||||
queue = {
|
||||
"id": "one",
|
||||
"text": "Could you send the report?",
|
||||
"unresolvedFields": ["task"],
|
||||
}
|
||||
record = self._adjudication("one", task="true")
|
||||
record["evidence"]["task"] = "not present"
|
||||
|
||||
with self.assertRaisesRegex(ValueError, "exact text quote"):
|
||||
adjudication.validate_adjudication(record, queue)
|
||||
|
||||
def test_accepts_matching_high_confidence_adjudication(self):
|
||||
report, accepted, excluded, remaining = self._merge(
|
||||
first=self._adjudication("one", task="true"),
|
||||
second=self._adjudication("one", task="true"),
|
||||
)
|
||||
|
||||
self.assertEqual(1, report["acceptedTierCCount"])
|
||||
self.assertEqual(0, report["remainingHumanReviewCount"])
|
||||
self.assertTrue(accepted[0]["task"])
|
||||
self.assertEqual("C", accepted[0]["labelQualityTier"])
|
||||
self.assertEqual(0.35, accepted[0]["sampleWeight"])
|
||||
self.assertEqual([], excluded)
|
||||
self.assertEqual([], remaining)
|
||||
|
||||
def test_keeps_disagreement_for_human_review(self):
|
||||
report, accepted, excluded, remaining = self._merge(
|
||||
first=self._adjudication("one", task="true"),
|
||||
second=self._adjudication("one", task="false"),
|
||||
)
|
||||
|
||||
self.assertEqual(0, report["acceptedTierCCount"])
|
||||
self.assertEqual([], accepted)
|
||||
self.assertEqual([], excluded)
|
||||
self.assertEqual(
|
||||
["adjudicator-disagreement"],
|
||||
remaining[0]["aiAdjudication"]["rejectedFields"]["task"],
|
||||
)
|
||||
|
||||
def test_keeps_low_confidence_for_human_review(self):
|
||||
first = self._adjudication("one", task="true")
|
||||
first["confidence"]["task"] = 0.89
|
||||
report, _, _, remaining = self._merge(
|
||||
first=first,
|
||||
second=self._adjudication("one", task="true"),
|
||||
)
|
||||
|
||||
self.assertEqual(1, report["remainingHumanReviewCount"])
|
||||
self.assertEqual(
|
||||
["low-confidence"],
|
||||
remaining[0]["aiAdjudication"]["rejectedFields"]["task"],
|
||||
)
|
||||
|
||||
def test_excludes_matching_high_confidence_device_command(self):
|
||||
first = self._adjudication("one", task="true")
|
||||
second = self._adjudication("one", task="true")
|
||||
for record in (first, second):
|
||||
record["recordDisposition"] = "exclude-device-command"
|
||||
|
||||
report, accepted, excluded, remaining = self._merge(first, second)
|
||||
|
||||
self.assertEqual(1, report["excludedDeviceCommandCount"])
|
||||
self.assertEqual([], accepted)
|
||||
self.assertEqual("exclude-device-command", excluded[0]["disposition"])
|
||||
self.assertEqual([], remaining)
|
||||
|
||||
def test_keeps_disposition_disagreement_for_human_review(self):
|
||||
first = self._adjudication("one", task="true")
|
||||
second = self._adjudication("one", task="true")
|
||||
second["recordDisposition"] = "exclude-device-command"
|
||||
|
||||
report, accepted, excluded, remaining = self._merge(first, second)
|
||||
|
||||
self.assertEqual(1, report["remainingHumanReviewCount"])
|
||||
self.assertEqual([], accepted)
|
||||
self.assertEqual([], excluded)
|
||||
self.assertEqual(
|
||||
["adjudicator-disagreement"],
|
||||
remaining[0]["aiAdjudication"]["rejectedFields"][
|
||||
"recordDisposition"
|
||||
],
|
||||
)
|
||||
|
||||
def test_review_sample_is_unique_and_includes_decision_template(self):
|
||||
with tempfile.TemporaryDirectory() as raw_directory:
|
||||
directory = Path(raw_directory)
|
||||
remaining = directory / "remaining.jsonl"
|
||||
records = []
|
||||
for index, field in enumerate(("task", "recordDisposition", "task")):
|
||||
first = self._adjudication(str(index), task="true")
|
||||
second = self._adjudication(str(index), task="false")
|
||||
records.append(
|
||||
{
|
||||
"id": str(index),
|
||||
"text": "Could you send the report?",
|
||||
"language": "en" if index % 2 else "zh-Hans",
|
||||
"aiAdjudication": {
|
||||
"rejectedFields": {
|
||||
field: ["adjudicator-disagreement"]
|
||||
},
|
||||
"adjudicatorA": first,
|
||||
"adjudicatorB": second,
|
||||
},
|
||||
}
|
||||
)
|
||||
self._write(remaining, records)
|
||||
sample = directory / "sample.jsonl"
|
||||
report = adjudication.review_sample(
|
||||
SimpleNamespace(
|
||||
remaining=remaining,
|
||||
sample=sample,
|
||||
sample_size=3,
|
||||
report=directory / "report.json",
|
||||
)
|
||||
)
|
||||
output = self._read(sample)
|
||||
|
||||
self.assertEqual(3, report["sampleCount"])
|
||||
self.assertEqual(3, len({record["id"] for record in output}))
|
||||
self.assertIsNone(output[0]["humanDecision"].popitem()[1])
|
||||
disposition = next(
|
||||
record
|
||||
for record in output
|
||||
if "recordDisposition" in record["fieldReviews"]
|
||||
)
|
||||
self.assertEqual(
|
||||
"keep",
|
||||
disposition["fieldReviews"]["recordDisposition"][
|
||||
"adjudicatorA"
|
||||
]["value"],
|
||||
)
|
||||
|
||||
def _merge(self, first, second):
|
||||
with tempfile.TemporaryDirectory() as raw_directory:
|
||||
directory = Path(raw_directory)
|
||||
conflicts = directory / "conflicts.jsonl"
|
||||
queue = directory / "queue.jsonl"
|
||||
first_path = directory / "first.jsonl"
|
||||
second_path = directory / "second.jsonl"
|
||||
self._write(
|
||||
conflicts,
|
||||
[
|
||||
{
|
||||
"id": "one",
|
||||
"text": "Could you send the report?",
|
||||
"language": "en",
|
||||
"unresolvedFields": ["task"],
|
||||
"modelVotes": {
|
||||
**{
|
||||
label: {"false": 4}
|
||||
for label in adjudication.INTENT_LABELS
|
||||
},
|
||||
"sentiment": {"neutral": 4},
|
||||
},
|
||||
}
|
||||
],
|
||||
)
|
||||
self._write(
|
||||
queue,
|
||||
[
|
||||
{
|
||||
"id": "one",
|
||||
"text": "Could you send the report?",
|
||||
"language": "en",
|
||||
"unresolvedFields": ["task"],
|
||||
}
|
||||
],
|
||||
)
|
||||
self._write(first_path, [first])
|
||||
self._write(second_path, [second])
|
||||
accepted_path = directory / "accepted.jsonl"
|
||||
excluded_path = directory / "excluded.jsonl"
|
||||
remaining_path = directory / "remaining.jsonl"
|
||||
report = adjudication.merge(
|
||||
SimpleNamespace(
|
||||
conflicts=conflicts,
|
||||
queue=queue,
|
||||
adjudicator_a=[first_path],
|
||||
adjudicator_b=[second_path],
|
||||
adjudicator_a_name="first",
|
||||
adjudicator_b_name="second",
|
||||
minimum_confidence=0.9,
|
||||
accepted=accepted_path,
|
||||
excluded=excluded_path,
|
||||
remaining=remaining_path,
|
||||
report=directory / "report.json",
|
||||
)
|
||||
)
|
||||
return (
|
||||
report,
|
||||
self._read(accepted_path),
|
||||
self._read(excluded_path),
|
||||
self._read(remaining_path),
|
||||
)
|
||||
|
||||
def _adjudication(self, identifier, task):
|
||||
return {
|
||||
"id": identifier,
|
||||
"recordDisposition": "keep",
|
||||
"dispositionConfidence": 0.95,
|
||||
"dispositionEvidence": "send the report",
|
||||
"resolutions": {"task": task},
|
||||
"confidence": {"task": 0.95},
|
||||
"evidence": {"task": "send the report"},
|
||||
}
|
||||
|
||||
def _write(self, path, records):
|
||||
path.write_text(
|
||||
"\n".join(json.dumps(record) for record in records) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
def _read(self, path):
|
||||
return [
|
||||
json.loads(line)
|
||||
for line in path.read_text(encoding="utf-8").splitlines()
|
||||
if line
|
||||
]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,66 @@
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
import assemble_v6_model_corpus as corpus
|
||||
|
||||
|
||||
class AssembleV6ModelCorpusTests(unittest.TestCase):
|
||||
def write_jsonl(self, path: Path, records: list[dict]) -> None:
|
||||
path.write_text(
|
||||
"".join(json.dumps(record) + "\n" for record in records),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
def test_assembles_disjoint_train_and_evaluation_splits(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
train = root / "train.jsonl"
|
||||
evaluation = root / "evaluation.jsonl"
|
||||
self.write_jsonl(
|
||||
train,
|
||||
[{"id": "train-1", "text": "hello", "split": "train", "language": "en"}],
|
||||
)
|
||||
self.write_jsonl(
|
||||
evaluation,
|
||||
[
|
||||
{
|
||||
"id": "test-1",
|
||||
"text": "world",
|
||||
"split": "test",
|
||||
"language": "en",
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
records, report = corpus.assemble(train, evaluation)
|
||||
|
||||
self.assertEqual(2, len(records))
|
||||
self.assertEqual({"test": 1, "train": 1}, report["splitCounts"])
|
||||
self.assertEqual(0, report["evaluationOverlapCount"])
|
||||
|
||||
def test_rejects_nfkc_casefold_overlap(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
train = root / "train.jsonl"
|
||||
evaluation = root / "evaluation.jsonl"
|
||||
self.write_jsonl(
|
||||
train,
|
||||
[{"id": "train-1", "text": "ＡＢＣ", "split": "train", "language": "en"}],
|
||||
)
|
||||
self.write_jsonl(
|
||||
evaluation,
|
||||
[{"id": "test-1", "text": "abc", "split": "test", "language": "en"}],
|
||||
)
|
||||
|
||||
with self.assertRaisesRegex(ValueError, "overlap"):
|
||||
corpus.assemble(train, evaluation)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,170 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
SCRIPT_DIRECTORY = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
def load_module(name: str, file_name: str):
|
||||
spec = importlib.util.spec_from_file_location(name, SCRIPT_DIRECTORY / file_name)
|
||||
if spec is None or spec.loader is None:
|
||||
raise RuntimeError(f"Unable to load {file_name}")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
generator = load_module(
|
||||
"generate_blessing_training_corpus",
|
||||
"generate_blessing_training_corpus.py",
|
||||
)
|
||||
extractor = load_module(
|
||||
"extract_lccc_blessing_candidates",
|
||||
"extract_lccc_blessing_candidates.py",
|
||||
)
|
||||
benchmark_preparer = load_module(
|
||||
"prepare_blessing_benchmark",
|
||||
"prepare_blessing_benchmark.py",
|
||||
)
|
||||
benchmark_finalizer = load_module(
|
||||
"finalize_blessing_benchmark",
|
||||
"finalize_blessing_benchmark.py",
|
||||
)
|
||||
|
||||
|
||||
class BlessingCorpusGeneratorTests(unittest.TestCase):
|
||||
def test_targets_are_balanced_per_language(self) -> None:
|
||||
targets = generator.allocate_targets(generator.ZH_FAMILIES, 10_000)
|
||||
positive = sum(
|
||||
targets[family.name]
|
||||
for family in generator.ZH_FAMILIES
|
||||
if family.blessing
|
||||
)
|
||||
negative = sum(
|
||||
targets[family.name]
|
||||
for family in generator.ZH_FAMILIES
|
||||
if not family.blessing
|
||||
)
|
||||
|
||||
self.assertEqual(positive, 5_000)
|
||||
self.assertEqual(negative, 5_000)
|
||||
|
||||
def test_generation_is_unique_and_partial_label_only(self) -> None:
|
||||
records = generator.generate_language(
|
||||
language="zh-Hans",
|
||||
common_slots=generator.ZH_COMMON,
|
||||
families=generator.ZH_FAMILIES,
|
||||
target=1_000,
|
||||
seed=generator.SEED,
|
||||
reserved=set(),
|
||||
)
|
||||
|
||||
self.assertEqual(len(records), 1_000)
|
||||
self.assertEqual(
|
||||
len({generator.fingerprint(record["text"]) for record in records}),
|
||||
1_000,
|
||||
)
|
||||
self.assertTrue(
|
||||
all(record["knownLabels"] == ["blessing"] for record in records)
|
||||
)
|
||||
self.assertEqual(sum(record["blessing"] for record in records), 500)
|
||||
|
||||
|
||||
class LCCCBlessingCandidateTests(unittest.TestCase):
|
||||
def test_direct_wishes_are_positive(self) -> None:
|
||||
self.assertEqual(
|
||||
extractor.classify("祝你生日快乐，愿新的一岁平安顺利"),
|
||||
("positive", "direct_wish"),
|
||||
)
|
||||
self.assertEqual(
|
||||
extractor.classify("恭喜你顺利毕业"),
|
||||
("positive", "congratulation"),
|
||||
)
|
||||
|
||||
def test_boundaries_are_negative(self) -> None:
|
||||
self.assertEqual(
|
||||
extractor.classify("帮我写一段生日祝福语"),
|
||||
("negative", "meta_request"),
|
||||
)
|
||||
self.assertEqual(
|
||||
extractor.classify("谢谢大家发来的生日祝福"),
|
||||
("negative", "received_thanks"),
|
||||
)
|
||||
self.assertEqual(
|
||||
extractor.classify("我们晚上一起庆祝项目上线"),
|
||||
("negative", "celebration_mention"),
|
||||
)
|
||||
self.assertEqual(
|
||||
extractor.classify("晚上好"),
|
||||
("negative", "plain_greeting"),
|
||||
)
|
||||
|
||||
def test_question_is_not_promoted_to_direct_wish(self) -> None:
|
||||
self.assertEqual(
|
||||
extractor.classify("可以对我说一句生日快乐吗？"),
|
||||
("negative", "meta_request"),
|
||||
)
|
||||
|
||||
|
||||
class BlessingBenchmarkTests(unittest.TestCase):
|
||||
def test_selection_prioritizes_boundary_before_explicit_marker(self) -> None:
|
||||
record = {
|
||||
"text": "文档引用了“祝你生日快乐”作为写作示例。",
|
||||
"blessing": False,
|
||||
"sentiment": "neutral",
|
||||
}
|
||||
|
||||
self.assertEqual(
|
||||
benchmark_preparer.selection_stratum(record),
|
||||
"boundary_candidate",
|
||||
)
|
||||
|
||||
def test_selection_includes_implicit_positive_language(self) -> None:
|
||||
record = {
|
||||
"text": "I hope you continue to know peace and happiness.",
|
||||
"blessing": False,
|
||||
"sentiment": "positive",
|
||||
}
|
||||
|
||||
self.assertEqual(
|
||||
benchmark_preparer.selection_stratum(record),
|
||||
"explicit_candidate",
|
||||
)
|
||||
|
||||
def test_benchmark_split_is_deterministic(self) -> None:
|
||||
first = benchmark_finalizer.benchmark_split("blessing-review-00042")
|
||||
second = benchmark_finalizer.benchmark_split("blessing-review-00042")
|
||||
|
||||
self.assertEqual(first, second)
|
||||
self.assertIn(first, {"calibration", "test"})
|
||||
|
||||
def test_binary_kappa_reports_partial_agreement(self) -> None:
|
||||
annotation_a = {
|
||||
"1": {"label": True},
|
||||
"2": {"label": True},
|
||||
"3": {"label": False},
|
||||
"4": {"label": False},
|
||||
}
|
||||
annotation_b = {
|
||||
"1": {"label": True},
|
||||
"2": {"label": False},
|
||||
"3": {"label": False},
|
||||
"4": {"label": False},
|
||||
}
|
||||
|
||||
agreement, kappa = benchmark_finalizer.binary_cohen_kappa(
|
||||
annotation_a,
|
||||
annotation_b,
|
||||
)
|
||||
|
||||
self.assertEqual(agreement, 0.75)
|
||||
self.assertEqual(kappa, 0.5)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,216 @@
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
import build_corpus_registry as registry
|
||||
|
||||
|
||||
class CorpusRegistryTests(unittest.TestCase):
|
||||
def test_evaluation_overlap_blocks_training(self):
|
||||
report, records, train, _, _ = self._build(
|
||||
train_records=[
|
||||
self._record("train-1", "Send report 123", "train", task=True)
|
||||
],
|
||||
evaluation_records=[
|
||||
self._record("eval-1", "Send report 123", "test", task=True)
|
||||
],
|
||||
)
|
||||
|
||||
self.assertEqual(1, report["canonicalRecordCount"])
|
||||
self.assertEqual("evaluation-only", records[0]["allowedUse"])
|
||||
self.assertEqual([], train)
|
||||
self.assertEqual(1, report["exactDuplicateCount"])
|
||||
|
||||
def test_unknown_labels_remain_unknown(self):
|
||||
report, records, train, pilot, _ = self._build(
|
||||
train_records=[
|
||||
{
|
||||
**self._record(
|
||||
"open-1",
|
||||
"Could you send the report?",
|
||||
"train",
|
||||
task=True,
|
||||
),
|
||||
"knownLabels": ["task"],
|
||||
}
|
||||
],
|
||||
all_intents_known=False,
|
||||
)
|
||||
|
||||
self.assertEqual(1, report["trainCandidateCount"])
|
||||
self.assertEqual("true", records[0]["labels"]["task"])
|
||||
self.assertEqual("unknown", records[0]["labels"]["question"])
|
||||
self.assertEqual("unknown", records[0]["labels"]["assistantCommand"])
|
||||
self.assertEqual(1, len(train))
|
||||
self.assertEqual(1, len(pilot))
|
||||
|
||||
def test_domain_only_record_and_weight_reach_training_output(self):
|
||||
report, records, train, _, _ = self._build(
|
||||
train_records=[
|
||||
{
|
||||
**self._record("domain-1", "Table for five", "train", task=False),
|
||||
"domain": "dining",
|
||||
"knownLabels": ["domain"],
|
||||
"sampleWeight": 0.5,
|
||||
}
|
||||
],
|
||||
all_intents_known=False,
|
||||
source_weight=0.7,
|
||||
)
|
||||
|
||||
self.assertEqual(1, report["trainCandidateCount"])
|
||||
self.assertEqual("dining", records[0]["domain"])
|
||||
self.assertEqual(["domain"], train[0]["knownLabels"])
|
||||
self.assertEqual(0.35, train[0]["sampleWeight"])
|
||||
|
||||
def test_conflicting_source_labels_enter_human_review(self):
|
||||
_, records, train, _, review = self._build(
|
||||
train_records=[
|
||||
self._record("one", "Please send it.", "train", task=True),
|
||||
self._record("two", "Please send it.", "train", task=False),
|
||||
],
|
||||
)
|
||||
|
||||
self.assertEqual(["task"], records[0]["labelConflicts"])
|
||||
self.assertEqual([], train)
|
||||
self.assertEqual(["task"], review[0]["conflicts"])
|
||||
|
||||
def test_rejects_unsafe_training_license(self):
|
||||
report, records, train, pilot, _ = self._build(
|
||||
train_records=[
|
||||
self._record("unsafe", "Research dialogue", "train", task=True)
|
||||
],
|
||||
train_license="research-only",
|
||||
)
|
||||
|
||||
self.assertEqual(0, report["canonicalRecordCount"])
|
||||
self.assertEqual({"unsafe-license": 1}, report["excludedRecordCounts"])
|
||||
self.assertEqual([], records)
|
||||
self.assertEqual([], train)
|
||||
self.assertEqual([], pilot)
|
||||
|
||||
def test_pilot_uses_unique_clusters(self):
|
||||
records = [
|
||||
self._record(
|
||||
f"record-{index}",
|
||||
f"Could you send report {index} before Friday?",
|
||||
"train",
|
||||
task=True,
|
||||
)
|
||||
for index in range(4)
|
||||
]
|
||||
_, _, _, pilot, _ = self._build(
|
||||
train_records=records,
|
||||
pilot_count=10,
|
||||
)
|
||||
|
||||
self.assertEqual(1, len(pilot))
|
||||
|
||||
def _build(
|
||||
self,
|
||||
train_records,
|
||||
evaluation_records=None,
|
||||
all_intents_known=True,
|
||||
train_license="MIT",
|
||||
pilot_count=10,
|
||||
source_weight=1.0,
|
||||
):
|
||||
evaluation_records = evaluation_records or []
|
||||
with tempfile.TemporaryDirectory() as raw_directory:
|
||||
directory = Path(raw_directory)
|
||||
train_path = directory / "train.jsonl"
|
||||
evaluation_path = directory / "evaluation.jsonl"
|
||||
self._write_json_lines(train_path, train_records)
|
||||
self._write_json_lines(evaluation_path, evaluation_records)
|
||||
manifest_path = directory / "sources.json"
|
||||
manifest_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"sources": [
|
||||
{
|
||||
"id": "train",
|
||||
"path": str(train_path),
|
||||
"license": train_license,
|
||||
"defaultUse": "train",
|
||||
"sourceType": "fixture",
|
||||
"allIntentLabelsKnown": all_intents_known,
|
||||
"sentimentKnown": False,
|
||||
"weight": source_weight,
|
||||
"required": True,
|
||||
},
|
||||
{
|
||||
"id": "evaluation",
|
||||
"path": str(evaluation_path),
|
||||
"license": "evaluation-only",
|
||||
"defaultUse": "evaluation-only",
|
||||
"sourceType": "fixture",
|
||||
"allIntentLabelsKnown": True,
|
||||
"sentimentKnown": False,
|
||||
"required": True,
|
||||
},
|
||||
],
|
||||
"excludedSources": [],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
paths = {
|
||||
name: directory / f"{name}.jsonl"
|
||||
for name in (
|
||||
"registry",
|
||||
"train_candidates",
|
||||
"pilot",
|
||||
"human_review",
|
||||
)
|
||||
}
|
||||
report_path = directory / "report.json"
|
||||
report = registry.build(
|
||||
SimpleNamespace(
|
||||
repository_root=directory,
|
||||
source_manifest=manifest_path,
|
||||
report=report_path,
|
||||
pilot_count=pilot_count,
|
||||
**paths,
|
||||
)
|
||||
)
|
||||
return (
|
||||
report,
|
||||
self._read_json_lines(paths["registry"]),
|
||||
self._read_json_lines(paths["train_candidates"]),
|
||||
self._read_json_lines(paths["pilot"]),
|
||||
self._read_json_lines(paths["human_review"]),
|
||||
)
|
||||
|
||||
def _record(self, identifier, text, split, task):
|
||||
return {
|
||||
"id": identifier,
|
||||
"text": text,
|
||||
"language": "en",
|
||||
"family": "fixture",
|
||||
"split": split,
|
||||
"task": task,
|
||||
}
|
||||
|
||||
def _write_json_lines(self, path, records):
|
||||
path.write_text(
|
||||
"\n".join(json.dumps(record) for record in records)
|
||||
+ ("\n" if records else ""),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
def _read_json_lines(self, path):
|
||||
return [
|
||||
json.loads(line)
|
||||
for line in path.read_text(encoding="utf-8").splitlines()
|
||||
if line
|
||||
]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,144 @@
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
import evaluate_product_policy_anchors as policy
|
||||
|
||||
|
||||
class ProductPolicyAnchorTests(unittest.TestCase):
|
||||
def test_prepare_and_evaluate_anchor_accuracy(self):
|
||||
with tempfile.TemporaryDirectory() as raw_directory:
|
||||
directory = Path(raw_directory)
|
||||
anchors = directory / "anchors.json"
|
||||
anchors.write_text(
|
||||
json.dumps(
|
||||
[
|
||||
{
|
||||
"id": "one",
|
||||
"text": "open my inbox",
|
||||
"language": "en",
|
||||
"expected": {
|
||||
"recordDisposition": "exclude-device-command"
|
||||
},
|
||||
},
|
||||
{
|
||||
"id": "two",
|
||||
"text": "Could you send the report?",
|
||||
"language": "en",
|
||||
"expected": {
|
||||
"recordDisposition": "keep",
|
||||
"replyableMessage": "true",
|
||||
"task": "true",
|
||||
"question": "true",
|
||||
"ambiguous": "false",
|
||||
},
|
||||
},
|
||||
]
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
queue = directory / "queue.jsonl"
|
||||
policy.prepare(SimpleNamespace(anchors=anchors, queue=queue))
|
||||
labeler = directory / "labeler.jsonl"
|
||||
records = [
|
||||
self._record(
|
||||
"one",
|
||||
"open my inbox",
|
||||
disposition="exclude-device-command",
|
||||
replyableMessage="false",
|
||||
task="true",
|
||||
question="false",
|
||||
ambiguous="false",
|
||||
),
|
||||
self._record(
|
||||
"two",
|
||||
"Could you send the report?",
|
||||
disposition="keep",
|
||||
replyableMessage="true",
|
||||
task="true",
|
||||
question="true",
|
||||
ambiguous="false",
|
||||
),
|
||||
]
|
||||
labeler.write_text(
|
||||
"\n".join(json.dumps(record) for record in records) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
report = policy.evaluate(
|
||||
SimpleNamespace(
|
||||
anchors=anchors,
|
||||
queue=queue,
|
||||
labeler=[("test", labeler)],
|
||||
minimum_accuracy=0.95,
|
||||
report=directory / "report.json",
|
||||
)
|
||||
)
|
||||
|
||||
self.assertTrue(report["eligibleForCorpusReadjudication"])
|
||||
self.assertEqual(
|
||||
1.0,
|
||||
report["labelers"]["test"]["decisionAccuracy"],
|
||||
)
|
||||
|
||||
def test_target_gate_ignores_non_target_mismatch(self):
|
||||
anchors = [
|
||||
{
|
||||
"id": "one",
|
||||
"text": "take your time",
|
||||
"language": "en",
|
||||
"expected": {
|
||||
"recordDisposition": "keep",
|
||||
"replyableMessage": "true",
|
||||
"task": "false",
|
||||
"question": "false",
|
||||
"ambiguous": "false",
|
||||
},
|
||||
}
|
||||
]
|
||||
adjudication = self._record(
|
||||
"one",
|
||||
"take your time",
|
||||
disposition="keep",
|
||||
replyableMessage="true",
|
||||
task="false",
|
||||
question="false",
|
||||
ambiguous="true",
|
||||
)
|
||||
|
||||
result = policy.evaluate_labeler(
|
||||
anchors,
|
||||
{"one": adjudication},
|
||||
{"replyableMessage", "task", "question"},
|
||||
)
|
||||
|
||||
self.assertEqual(1.0, result["gateDecisionAccuracy"])
|
||||
self.assertLess(result["decisionAccuracy"], 1.0)
|
||||
|
||||
def _record(self, identifier, text, disposition, **values):
|
||||
resolutions = {
|
||||
field: (
|
||||
"unknown"
|
||||
if field == "domain"
|
||||
else "false"
|
||||
)
|
||||
for field in policy.ANCHOR_FIELDS
|
||||
}
|
||||
resolutions.update(values)
|
||||
return {
|
||||
"id": identifier,
|
||||
"recordDisposition": disposition,
|
||||
"dispositionConfidence": 0.99,
|
||||
"dispositionEvidence": text,
|
||||
"resolutions": resolutions,
|
||||
"confidence": {field: 0.99 for field in resolutions},
|
||||
"evidence": {field: text for field in resolutions},
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,181 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
import finalize_v6_blind_holdout as finalizer
|
||||
|
||||
|
||||
class FinalizeV6BlindHoldoutTests(unittest.TestCase):
|
||||
def test_review_accepts_each_high_confidence_field_independently(self):
|
||||
primary = self._labelers(
|
||||
"primary",
|
||||
[
|
||||
self._states(task="true", question="true", domain="finance"),
|
||||
self._states(task="true", question="true", domain="travel"),
|
||||
self._states(task="true", question="false", domain="calendar"),
|
||||
],
|
||||
)
|
||||
reviewers = self._labelers(
|
||||
"reviewer",
|
||||
[
|
||||
self._states(task="true", question="true", domain="finance"),
|
||||
self._states(task="false", question="false", domain="travel"),
|
||||
],
|
||||
)
|
||||
|
||||
states = finalizer.resolve_model_states(
|
||||
"record-1",
|
||||
primary,
|
||||
reviewers,
|
||||
in_review_queue=True,
|
||||
)
|
||||
output = finalizer.output_record(
|
||||
{"id": "record-1", "text": "Text", "language": "en"},
|
||||
states,
|
||||
"test",
|
||||
)
|
||||
|
||||
self.assertEqual("true", states["task"])
|
||||
self.assertEqual("unknown", states["question"])
|
||||
self.assertEqual("unknown", states["domain"])
|
||||
self.assertTrue(output["task"])
|
||||
self.assertIn("task", output["knownLabels"])
|
||||
self.assertNotIn("question", output["knownLabels"])
|
||||
self.assertNotIn("domain", output["knownLabels"])
|
||||
self.assertIsNone(output["domain"])
|
||||
|
||||
def test_human_fields_override_models_but_exclusion_infers_nothing(self):
|
||||
model_states = self._states(
|
||||
task="false",
|
||||
question="false",
|
||||
replyableMessage="false",
|
||||
)
|
||||
overrides = finalizer.apply_human_overrides(
|
||||
model_states,
|
||||
{
|
||||
"recordDisposition": "keep",
|
||||
"task": "true",
|
||||
"question": "unknown",
|
||||
"replyableMessage": "true",
|
||||
"ambiguous": "false",
|
||||
},
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
["task", "question", "replyableMessage", "ambiguous"],
|
||||
overrides,
|
||||
)
|
||||
self.assertEqual("true", model_states["task"])
|
||||
self.assertEqual("unknown", model_states["question"])
|
||||
self.assertEqual("true", model_states["replyableMessage"])
|
||||
|
||||
excluded_states = self._states(task="true")
|
||||
excluded_overrides = finalizer.apply_human_overrides(
|
||||
excluded_states,
|
||||
{"recordDisposition": "exclude-device-command"},
|
||||
)
|
||||
self.assertEqual([], excluded_overrides)
|
||||
self.assertEqual("true", excluded_states["task"])
|
||||
|
||||
def test_split_assigns_twenty_of_each_kind_per_language(self):
|
||||
records = [
|
||||
{
|
||||
"id": f"{language}-{index:03d}",
|
||||
"text": f"{language} {index}",
|
||||
"language": language,
|
||||
}
|
||||
for language in ("en", "zh-Hans")
|
||||
for index in range(60)
|
||||
]
|
||||
|
||||
assignments = finalizer.assign_splits(records, records_per_split=20)
|
||||
|
||||
for language in ("en", "zh-Hans"):
|
||||
counts = {
|
||||
split: sum(
|
||||
assignments[f"{language}-{index:03d}"] == split
|
||||
for index in range(60)
|
||||
)
|
||||
for split in finalizer.SPLITS
|
||||
}
|
||||
self.assertEqual(
|
||||
{"validation": 20, "test": 20, "golden": 20},
|
||||
counts,
|
||||
)
|
||||
self.assertEqual("validation", assignments["en-000"])
|
||||
self.assertEqual("test", assignments["en-020"])
|
||||
self.assertEqual("golden", assignments["en-040"])
|
||||
|
||||
def test_unknown_fields_are_not_known_or_positive(self):
|
||||
states = self._states(
|
||||
assistantCommand="unknown",
|
||||
sentiment="unknown",
|
||||
domain="unknown",
|
||||
)
|
||||
states["ambiguous"] = "unknown"
|
||||
|
||||
output = finalizer.output_record(
|
||||
{"id": "record-1", "text": "unchanged", "language": "en"},
|
||||
states,
|
||||
"golden",
|
||||
)
|
||||
|
||||
self.assertFalse(output["assistantCommand"])
|
||||
self.assertEqual("neutral", output["sentiment"])
|
||||
self.assertIsNone(output["domain"])
|
||||
self.assertIsNone(output["ambiguous"])
|
||||
self.assertNotIn("assistantCommand", output["knownLabels"])
|
||||
self.assertNotIn("sentiment", output["knownLabels"])
|
||||
self.assertNotIn("domain", output["knownLabels"])
|
||||
self.assertNotEqual("train", output["split"])
|
||||
self.assertEqual("unchanged", output["text"])
|
||||
|
||||
def test_unreviewed_field_requires_unanimous_non_unknown_vote(self):
|
||||
primary = self._labelers(
|
||||
"primary",
|
||||
[
|
||||
self._states(blessing="unknown"),
|
||||
self._states(blessing="unknown"),
|
||||
self._states(blessing="unknown"),
|
||||
],
|
||||
)
|
||||
|
||||
states = finalizer.resolve_model_states(
|
||||
"record-1",
|
||||
primary,
|
||||
[],
|
||||
in_review_queue=False,
|
||||
)
|
||||
|
||||
self.assertEqual("unknown", states["blessing"])
|
||||
self.assertEqual("false", states["task"])
|
||||
|
||||
def test_duplicate_ids_are_rejected(self):
|
||||
with self.assertRaisesRegex(ValueError, "duplicate id"):
|
||||
finalizer.unique_records(
|
||||
[{"id": "same"}, {"id": "same"}],
|
||||
"test input",
|
||||
)
|
||||
|
||||
def _states(self, **overrides):
|
||||
states = {field: "false" for field in finalizer.INTENT_LABELS}
|
||||
states["sentiment"] = "neutral"
|
||||
states["domain"] = "unknown"
|
||||
states["ambiguous"] = "false"
|
||||
states.update(overrides)
|
||||
return states
|
||||
|
||||
def _labelers(self, prefix, states):
|
||||
return [
|
||||
(f"{prefix}-{index}", {"record-1": state})
|
||||
for index, state in enumerate(states)
|
||||
]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,175 @@
|
||||
import json
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
import generate_open_training_corpus as corpus
|
||||
|
||||
|
||||
class OpenTrainingCorpusTests(unittest.TestCase):
|
||||
def test_schema_contains_new_intents_and_single_domain(self):
|
||||
record = corpus.make_record(
|
||||
record_id="one",
|
||||
text="Play some jazz",
|
||||
language="en",
|
||||
family="fixture",
|
||||
source_dataset="fixture",
|
||||
source_license="CC0-1.0",
|
||||
source_url="https://example.test/train",
|
||||
source_revision="deadbeef",
|
||||
source_split="train",
|
||||
known_labels=corpus.known_labels_for_mapping(
|
||||
"assistantCommand",
|
||||
"media",
|
||||
),
|
||||
labeling_method="fixture",
|
||||
assistant_command=True,
|
||||
domain="media",
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
{"assistantCommand", "informationQuery", "systemNotification"}
|
||||
<= set(corpus.INTENT_LABELS),
|
||||
True,
|
||||
)
|
||||
self.assertEqual("media", record["domain"])
|
||||
self.assertEqual(
|
||||
[
|
||||
"assistantCommand",
|
||||
"domain",
|
||||
"informationQuery",
|
||||
"systemNotification",
|
||||
],
|
||||
record["knownLabels"],
|
||||
)
|
||||
self.assertFalse(record["task"])
|
||||
self.assertFalse(record["informationQuery"])
|
||||
|
||||
def test_massive_mapping_is_conservative(self):
|
||||
self.assertEqual(
|
||||
("assistantCommand", "calendar"),
|
||||
corpus.massive_mapping("alarm_set"),
|
||||
)
|
||||
self.assertEqual(
|
||||
("informationQuery", "generalKnowledge"),
|
||||
corpus.massive_mapping("qa_factoid"),
|
||||
)
|
||||
self.assertEqual(
|
||||
("task", "travel"),
|
||||
corpus.massive_mapping("transport_taxi"),
|
||||
)
|
||||
self.assertEqual((None, None), corpus.massive_mapping("general_greet"))
|
||||
|
||||
def test_bitod_mapping_supports_official_chinese_intents(self):
|
||||
self.assertEqual(
|
||||
("informationQuery", "dining"),
|
||||
corpus.bitod_mapping("餐馆查询"),
|
||||
)
|
||||
self.assertEqual(("task", "travel"), corpus.bitod_mapping("宾馆预订"))
|
||||
self.assertEqual(
|
||||
("informationQuery", "travel"),
|
||||
corpus.bitod_mapping("香港地铁"),
|
||||
)
|
||||
self.assertEqual(
|
||||
("informationQuery", "weather"),
|
||||
corpus.bitod_mapping("天气查询"),
|
||||
)
|
||||
|
||||
def test_new_builders_are_pinned_optional_and_isolated(self):
|
||||
builders = corpus.configured_source_builders()
|
||||
names = {builder.name for builder in builders}
|
||||
|
||||
self.assertTrue(
|
||||
{
|
||||
"SNIPS",
|
||||
"MInDS-14 zh-CN",
|
||||
"BiToD",
|
||||
"RESTAURANTS-8K",
|
||||
"FormosaNLU Synth v1",
|
||||
}
|
||||
<= names
|
||||
)
|
||||
self.assertNotIn("CFPB", names)
|
||||
self.assertNotIn("CLINC150", names)
|
||||
self.assertNotIn("openclaw-zh-greetings", names)
|
||||
self.assertNotIn("WeChat-AutoSendBless", names)
|
||||
self.assertTrue(all(builder.optional for builder in builders))
|
||||
for revision in (
|
||||
corpus.SNIPS_REVISION,
|
||||
corpus.MINDS14_REVISION,
|
||||
corpus.BITOD_REVISION,
|
||||
corpus.RESTAURANT8K_REVISION,
|
||||
corpus.FORMOSA_NLU_REVISION,
|
||||
):
|
||||
self.assertRegex(revision, r"^[0-9a-f]{40}$")
|
||||
|
||||
def test_optional_builder_failure_is_reported_unavailable(self):
|
||||
def unavailable(_seed):
|
||||
raise corpus.SourceUnavailable("offline")
|
||||
|
||||
sources, failures = corpus.build_available_sources(
|
||||
(corpus.SourceBuilder("fixture", unavailable),),
|
||||
seed=7,
|
||||
allow_unavailable_sources=False,
|
||||
)
|
||||
|
||||
self.assertEqual([], sources)
|
||||
self.assertEqual("fixture", failures[0]["dataset"])
|
||||
self.assertIn("offline", failures[0]["reason"])
|
||||
|
||||
def test_snips_uses_only_official_train_intents(self):
|
||||
def payload(url):
|
||||
intent = next(
|
||||
value for value in corpus.SNIPS_MAPPING if f"/{value}/" in url
|
||||
)
|
||||
return json.dumps(
|
||||
{
|
||||
intent: [
|
||||
{
|
||||
"data": [
|
||||
{"text": "example "},
|
||||
{"text": intent},
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
).encode()
|
||||
|
||||
with mock.patch.object(corpus, "fetch_bytes", side_effect=payload):
|
||||
records = corpus.snips_records(seed=3)
|
||||
|
||||
self.assertEqual(6, len(records))
|
||||
self.assertTrue(all(record["sourceSplit"] == "train" for record in records))
|
||||
self.assertTrue(
|
||||
all(record["sourceRevision"] == corpus.SNIPS_REVISION for record in records)
|
||||
)
|
||||
play = next(record for record in records if "PlayMusic" in record["text"])
|
||||
self.assertTrue(play["assistantCommand"])
|
||||
self.assertEqual("media", play["domain"])
|
||||
|
||||
def test_formosa_synthetic_records_are_simplified_and_keep_low_weight(self):
|
||||
payload = (
|
||||
json.dumps(
|
||||
{
|
||||
"id": "syn-1",
|
||||
"utt": "播放爵士樂",
|
||||
"intent": "play_music",
|
||||
}
|
||||
)
|
||||
+ "\n"
|
||||
).encode()
|
||||
with mock.patch.object(corpus, "fetch_bytes", return_value=payload):
|
||||
records = corpus.formosa_nlu_records(seed=3)
|
||||
|
||||
self.assertEqual(1, len(records))
|
||||
self.assertEqual("播放爵士乐", records[0]["text"])
|
||||
self.assertEqual("zh-Hans", records[0]["language"])
|
||||
self.assertEqual(0.35, records[0]["sampleWeight"])
|
||||
self.assertEqual("train", records[0]["sourceSplit"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,199 @@
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import unicodedata
|
||||
import unittest
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
import generate_v6_boundary_corpus as corpus
|
||||
|
||||
|
||||
REQUIRED_FIELDS = {
|
||||
"id",
|
||||
"text",
|
||||
"language",
|
||||
"split",
|
||||
"family",
|
||||
"task",
|
||||
"question",
|
||||
"invitation",
|
||||
"complaint",
|
||||
"scheduleNegotiation",
|
||||
"confirmationDecision",
|
||||
"followUpReminder",
|
||||
"blessing",
|
||||
"sentiment",
|
||||
"replyable",
|
||||
"assistantCommand",
|
||||
"informationQuery",
|
||||
"systemNotification",
|
||||
"domain",
|
||||
"sampleWeight",
|
||||
"knownLabels",
|
||||
"labelingMethod",
|
||||
"sourceDataset",
|
||||
"sourceLicense",
|
||||
"sourceURL",
|
||||
"sourceRevision",
|
||||
"sourceSplit",
|
||||
"synthetic",
|
||||
"templateFamily",
|
||||
}
|
||||
|
||||
|
||||
class V6BoundaryCorpusTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.records, cls.target_counts, cls.excluded = corpus.generate_records()
|
||||
cls.payload = corpus.serialize_records(cls.records)
|
||||
|
||||
def test_default_generation_is_reproducible(self):
|
||||
records, target_counts, excluded = corpus.generate_records()
|
||||
payload = corpus.serialize_records(records)
|
||||
|
||||
self.assertEqual(self.payload, payload)
|
||||
self.assertEqual(self.target_counts, target_counts)
|
||||
self.assertEqual(self.excluded, excluded)
|
||||
self.assertEqual(
|
||||
hashlib.sha256(self.payload).hexdigest(),
|
||||
hashlib.sha256(payload).hexdigest(),
|
||||
)
|
||||
|
||||
def test_default_scale_language_intent_and_domain_balance(self):
|
||||
self.assertGreaterEqual(len(self.records), 6_000)
|
||||
self.assertEqual(
|
||||
{"en": 3_600, "zh-Hans": 3_600},
|
||||
Counter(record["language"] for record in self.records),
|
||||
)
|
||||
for language in corpus.LANGUAGES:
|
||||
for intent in corpus.NEW_INTENTS:
|
||||
self.assertEqual(
|
||||
1_000,
|
||||
self.target_counts[f"{language}|{intent}"],
|
||||
)
|
||||
for intent in corpus.HARD_NEGATIVE_INTENTS:
|
||||
self.assertEqual(
|
||||
200,
|
||||
self.target_counts[f"{language}|{intent}"],
|
||||
)
|
||||
self.assertEqual(
|
||||
{domain: 600 for domain in corpus.DOMAINS},
|
||||
Counter(record["domain"] for record in self.records),
|
||||
)
|
||||
|
||||
def test_records_have_full_schema_low_weight_and_known_boundaries(self):
|
||||
for record in self.records:
|
||||
self.assertEqual(REQUIRED_FIELDS, set(record))
|
||||
self.assertEqual(0.35, record["sampleWeight"])
|
||||
self.assertEqual("train", record["split"])
|
||||
self.assertEqual("train", record["sourceSplit"])
|
||||
self.assertEqual(corpus.SOURCE_DATASET, record["sourceDataset"])
|
||||
self.assertEqual(corpus.SOURCE_LICENSE, record["sourceLicense"])
|
||||
self.assertEqual(corpus.SOURCE_REVISION, record["sourceRevision"])
|
||||
self.assertTrue(record["synthetic"])
|
||||
self.assertEqual(record["family"], record["templateFamily"])
|
||||
self.assertEqual(corpus.KNOWN_LABELS, record["knownLabels"])
|
||||
self.assertEqual(record["text"], unicodedata.normalize("NFKC", record["text"]))
|
||||
|
||||
routing_count = sum(record[intent] for intent in corpus.NEW_INTENTS)
|
||||
if any(record[intent] for intent in corpus.NEW_INTENTS):
|
||||
self.assertEqual(1, routing_count)
|
||||
self.assertFalse(record["task"])
|
||||
self.assertFalse(record["question"])
|
||||
self.assertFalse(record["replyable"])
|
||||
else:
|
||||
self.assertEqual(0, routing_count)
|
||||
self.assertTrue(
|
||||
record["task"] or record["question"] or record["replyable"]
|
||||
)
|
||||
|
||||
def test_generated_ids_and_normalized_texts_are_unique(self):
|
||||
ids = [record["id"] for record in self.records]
|
||||
texts = [corpus.fingerprint(record["text"]) for record in self.records]
|
||||
|
||||
self.assertEqual(len(ids), len(set(ids)))
|
||||
self.assertEqual(len(texts), len(set(texts)))
|
||||
|
||||
def test_holdout_discovery_and_nfkc_overlap_exclusion(self):
|
||||
system_record = next(
|
||||
record
|
||||
for record in self.records
|
||||
if record["language"] == "zh-Hans"
|
||||
and record["systemNotification"]
|
||||
and ":" in record["text"]
|
||||
)
|
||||
blind_record = next(
|
||||
record
|
||||
for record in self.records
|
||||
if record["language"] == "en" and record["informationQuery"]
|
||||
)
|
||||
with tempfile.TemporaryDirectory() as temporary_directory:
|
||||
directory = Path(temporary_directory)
|
||||
wildcard_holdout = directory / "frozen-holdout-corpus.jsonl"
|
||||
blind_holdout = directory / "product-policy-blind-holdout-v1.jsonl"
|
||||
wildcard_holdout.write_text(
|
||||
json.dumps(
|
||||
{"text": system_record["text"].replace(":", "：")},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
blind_holdout.write_text(
|
||||
json.dumps({"text": blind_record["text"]}) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
paths = corpus.discover_holdout_paths(directory)
|
||||
holdouts = corpus.load_holdout_fingerprints(paths)
|
||||
records, _, excluded = corpus.generate_records(
|
||||
holdout_fingerprints=holdouts
|
||||
)
|
||||
|
||||
generated = {corpus.fingerprint(record["text"]) for record in records}
|
||||
self.assertEqual({wildcard_holdout, blind_holdout}, set(paths))
|
||||
self.assertNotIn(corpus.fingerprint(system_record["text"]), generated)
|
||||
self.assertNotIn(corpus.fingerprint(blind_record["text"]), generated)
|
||||
self.assertGreaterEqual(excluded, 2)
|
||||
self.assertEqual(len(self.records), len(records))
|
||||
|
||||
def test_summary_counts_and_hash_match_written_jsonl(self):
|
||||
with tempfile.TemporaryDirectory() as temporary_directory:
|
||||
directory = Path(temporary_directory)
|
||||
output = directory / "corpus.jsonl"
|
||||
summary_path = directory / "summary.json"
|
||||
|
||||
summary = corpus.write_corpus(
|
||||
output,
|
||||
summary_path,
|
||||
positive_per_intent_language=12,
|
||||
negative_per_intent_language=12,
|
||||
)
|
||||
persisted = json.loads(summary_path.read_text(encoding="utf-8"))
|
||||
output_sha256 = hashlib.sha256(output.read_bytes()).hexdigest()
|
||||
|
||||
self.assertEqual(summary, persisted)
|
||||
self.assertEqual(output_sha256, summary["corpusSHA256"])
|
||||
self.assertEqual(144, summary["recordCount"])
|
||||
self.assertEqual(
|
||||
{"en": 72, "zh-Hans": 72},
|
||||
summary["counts"]["byLanguage"],
|
||||
)
|
||||
self.assertEqual(48, summary["counts"]["byIntent"]["replyableMessage"])
|
||||
self.assertEqual(
|
||||
12,
|
||||
summary["counts"]["byBoundaryTargetAndLanguage"]["en"][
|
||||
"replyableMessage"
|
||||
],
|
||||
)
|
||||
self.assertEqual(set(corpus.DOMAINS), set(summary["counts"]["byDomain"]))
|
||||
self.assertTrue(summary["counts"]["byTemplateFamily"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -91,6 +91,30 @@ class IterativeRetrainingTests(unittest.TestCase):
|
||||
|
||||
self.assertEqual([False, True], predicted["task"].tolist())
|
||||
|
||||
def test_registry_sample_weight_is_applied(self):
|
||||
configuration = research.configurations()[0]
|
||||
record = {
|
||||
"id": "weighted",
|
||||
"text": "Play music",
|
||||
"language": "en",
|
||||
"knownLabels": ["assistantCommand"],
|
||||
"assistantCommand": True,
|
||||
"sampleWeight": 0.35,
|
||||
}
|
||||
|
||||
self.assertIn("assistantCommand", research.INTENTS)
|
||||
self.assertAlmostEqual(
|
||||
configuration.external_weight * 0.35,
|
||||
research.sample_weight(record, configuration),
|
||||
)
|
||||
|
||||
def test_legacy_records_do_not_define_new_labels_as_false(self):
|
||||
record = {"id": "legacy", "text": "Play music", "language": "en"}
|
||||
|
||||
self.assertTrue(research.is_known(record, "task"))
|
||||
self.assertFalse(research.is_known(record, "assistantCommand"))
|
||||
self.assertFalse(research.is_known(record, "systemNotification"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -0,0 +1,286 @@
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
import merge_consensus_labels_v2 as consensus
|
||||
|
||||
|
||||
class ConsensusV2Tests(unittest.TestCase):
|
||||
def test_primary_unanimous_becomes_tier_a(self):
|
||||
report, tier_a, tier_b, human = self._merge(
|
||||
primary_states=[self._states(task="true")] * 3,
|
||||
review_states=[],
|
||||
)
|
||||
|
||||
self.assertEqual(1, report["tierACount"])
|
||||
self.assertEqual(1, len(tier_a))
|
||||
self.assertTrue(tier_a[0]["task"])
|
||||
self.assertEqual(1.0, tier_a[0]["sampleWeight"])
|
||||
self.assertEqual([], tier_b)
|
||||
self.assertEqual([], human)
|
||||
|
||||
def test_four_of_five_becomes_tier_b(self):
|
||||
report, tier_a, tier_b, human = self._merge(
|
||||
primary_states=[
|
||||
self._states(task="true"),
|
||||
self._states(task="true"),
|
||||
self._states(task="false"),
|
||||
],
|
||||
review_states=[
|
||||
self._states(task="true"),
|
||||
self._states(task="true"),
|
||||
],
|
||||
)
|
||||
|
||||
self.assertEqual(0, report["tierACount"])
|
||||
self.assertEqual(1, report["tierBCount"])
|
||||
self.assertEqual([], tier_a)
|
||||
self.assertTrue(tier_b[0]["task"])
|
||||
self.assertEqual(0.65, tier_b[0]["sampleWeight"])
|
||||
self.assertEqual([], human)
|
||||
|
||||
def test_three_two_vote_requires_human_review(self):
|
||||
report, _, tier_b, human = self._merge(
|
||||
primary_states=[
|
||||
self._states(task="true"),
|
||||
self._states(task="true"),
|
||||
self._states(task="false"),
|
||||
],
|
||||
review_states=[
|
||||
self._states(task="true"),
|
||||
self._states(task="false"),
|
||||
],
|
||||
)
|
||||
|
||||
self.assertEqual(1, report["humanReviewCount"])
|
||||
self.assertEqual([], tier_b)
|
||||
self.assertIn("task", human[0]["unresolvedFields"])
|
||||
|
||||
def test_unknown_primary_state_triggers_review(self):
|
||||
with tempfile.TemporaryDirectory() as raw_directory:
|
||||
directory = Path(raw_directory)
|
||||
queue = directory / "queue.jsonl"
|
||||
self._write(
|
||||
queue,
|
||||
[{"id": "record-1", "text": "Hello", "language": "en"}],
|
||||
)
|
||||
primary = []
|
||||
for index, state in enumerate(
|
||||
[
|
||||
self._states(question="unknown"),
|
||||
self._states(question="false"),
|
||||
self._states(question="false"),
|
||||
]
|
||||
):
|
||||
path = directory / f"primary-{index}.jsonl"
|
||||
self._write(path, [self._label("record-1", state)])
|
||||
primary.append((f"primary-{index}", path))
|
||||
output = directory / "review.jsonl"
|
||||
report_path = directory / "report.json"
|
||||
report = consensus.prepare_review(
|
||||
SimpleNamespace(
|
||||
queue=queue,
|
||||
primary=primary,
|
||||
output=output,
|
||||
report=report_path,
|
||||
)
|
||||
)
|
||||
|
||||
self.assertEqual(1, report["reviewCount"])
|
||||
self.assertEqual("record-1", self._read(output)[0]["id"])
|
||||
|
||||
def test_quoted_positive_is_sent_to_human(self):
|
||||
report, _, _, human = self._merge(
|
||||
primary_states=[
|
||||
self._states(blessing="true"),
|
||||
self._states(blessing="true"),
|
||||
self._states(blessing="false"),
|
||||
],
|
||||
review_states=[
|
||||
self._states(blessing="true"),
|
||||
self._states(blessing="true"),
|
||||
],
|
||||
review_quoted=[True, True],
|
||||
)
|
||||
|
||||
self.assertEqual(1, report["humanReviewCount"])
|
||||
self.assertIn("quotedOrMeta", human[0]["unresolvedFields"])
|
||||
|
||||
def test_unanimous_unknown_stays_out_of_known_labels(self):
|
||||
_, tier_a, _, _ = self._merge(
|
||||
primary_states=[self._states(assistantCommand="unknown")] * 3,
|
||||
review_states=[],
|
||||
)
|
||||
|
||||
self.assertEqual(1, len(tier_a))
|
||||
self.assertNotIn("assistantCommand", tier_a[0]["knownLabels"])
|
||||
self.assertFalse(tier_a[0]["assistantCommand"])
|
||||
self.assertIsNone(tier_a[0]["domain"])
|
||||
|
||||
def test_split_and_combine_preserve_queue_order(self):
|
||||
with tempfile.TemporaryDirectory() as raw_directory:
|
||||
directory = Path(raw_directory)
|
||||
queue_path = directory / "queue.jsonl"
|
||||
queue = [
|
||||
{"id": f"record-{index}", "text": str(index), "language": "en"}
|
||||
for index in range(5)
|
||||
]
|
||||
self._write(queue_path, queue)
|
||||
chunks = directory / "chunks"
|
||||
split_report = consensus.split_queue(
|
||||
SimpleNamespace(
|
||||
queue=queue_path,
|
||||
output_directory=chunks,
|
||||
chunk_size=2,
|
||||
report=directory / "split-report.json",
|
||||
)
|
||||
)
|
||||
self.assertEqual(3, split_report["chunkCount"])
|
||||
outputs = []
|
||||
for path in sorted(chunks.glob("*.jsonl")):
|
||||
output_path = directory / f"labeled-{path.name}"
|
||||
self._write(
|
||||
output_path,
|
||||
[
|
||||
self._label(record["id"], self._states())
|
||||
for record in self._read(path)
|
||||
],
|
||||
)
|
||||
outputs.append(output_path)
|
||||
combined_path = directory / "combined.jsonl"
|
||||
report = consensus.combine_labeler(
|
||||
SimpleNamespace(
|
||||
queue=queue_path,
|
||||
input=outputs,
|
||||
output=combined_path,
|
||||
report=directory / "combine-report.json",
|
||||
)
|
||||
)
|
||||
|
||||
self.assertEqual(5, report["outputCount"])
|
||||
self.assertEqual(
|
||||
[record["id"] for record in queue],
|
||||
[record["id"] for record in self._read(combined_path)],
|
||||
)
|
||||
|
||||
def _merge(
|
||||
self,
|
||||
primary_states,
|
||||
review_states,
|
||||
review_quoted=None,
|
||||
):
|
||||
review_quoted = review_quoted or [False] * len(review_states)
|
||||
with tempfile.TemporaryDirectory() as raw_directory:
|
||||
directory = Path(raw_directory)
|
||||
queue_path = directory / "queue.jsonl"
|
||||
self._write(
|
||||
queue_path,
|
||||
[
|
||||
{
|
||||
"id": "record-1",
|
||||
"text": "Could you send the report?",
|
||||
"language": "en",
|
||||
}
|
||||
],
|
||||
)
|
||||
primary = []
|
||||
for index, state in enumerate(primary_states):
|
||||
path = directory / f"primary-{index}.jsonl"
|
||||
self._write(path, [self._label("record-1", state)])
|
||||
primary.append((f"primary-{index}", path))
|
||||
review = []
|
||||
review_required = (
|
||||
len(
|
||||
consensus.primary_review_ids(
|
||||
consensus.load_labelers(
|
||||
primary,
|
||||
{"record-1"},
|
||||
True,
|
||||
),
|
||||
{"record-1"},
|
||||
)
|
||||
)
|
||||
== 1
|
||||
)
|
||||
if review_required:
|
||||
for index, state in enumerate(review_states):
|
||||
path = directory / f"review-{index}.jsonl"
|
||||
self._write(
|
||||
path,
|
||||
[
|
||||
self._label(
|
||||
"record-1",
|
||||
state,
|
||||
quoted=review_quoted[index],
|
||||
)
|
||||
],
|
||||
)
|
||||
review.append((f"review-{index}", path))
|
||||
else:
|
||||
for index in range(2):
|
||||
path = directory / f"review-{index}.jsonl"
|
||||
self._write(path, [])
|
||||
review.append((f"review-{index}", path))
|
||||
outputs = {
|
||||
name: directory / f"{name}.jsonl"
|
||||
for name in ("tier_a", "tier_b", "accepted", "human_review")
|
||||
}
|
||||
report_path = directory / "report.json"
|
||||
report = consensus.merge(
|
||||
SimpleNamespace(
|
||||
queue=queue_path,
|
||||
primary=primary,
|
||||
reviewer=review,
|
||||
report=report_path,
|
||||
**outputs,
|
||||
)
|
||||
)
|
||||
return (
|
||||
report,
|
||||
self._read(outputs["tier_a"]),
|
||||
self._read(outputs["tier_b"]),
|
||||
self._read(outputs["human_review"]),
|
||||
)
|
||||
|
||||
def _states(self, **overrides):
|
||||
values = {label: "false" for label in consensus.INTENT_LABELS}
|
||||
values.update(overrides)
|
||||
values["sentiment"] = overrides.get("sentiment", "neutral")
|
||||
values["domain"] = overrides.get("domain", "unknown")
|
||||
return values
|
||||
|
||||
def _label(self, identifier, states, quoted=False):
|
||||
return {
|
||||
"id": identifier,
|
||||
"labels": {
|
||||
label: states[label] for label in consensus.INTENT_LABELS
|
||||
},
|
||||
"sentiment": states["sentiment"],
|
||||
"domain": states["domain"],
|
||||
"ambiguous": False,
|
||||
"quotedOrMeta": quoted,
|
||||
"confidence": 0.95,
|
||||
}
|
||||
|
||||
def _write(self, path, records):
|
||||
path.write_text(
|
||||
"\n".join(json.dumps(record) for record in records)
|
||||
+ ("\n" if records else ""),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
def _read(self, path):
|
||||
return [
|
||||
json.loads(line)
|
||||
for line in path.read_text(encoding="utf-8").splitlines()
|
||||
if line
|
||||
]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,63 @@
|
||||
import importlib.util
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
SCRIPT_PATH = (
|
||||
Path(__file__).resolve().parents[1] / "prepare_apple_nl_hardening_corpus.py"
|
||||
)
|
||||
SPEC = importlib.util.spec_from_file_location("prepare_apple_nl_hardening_corpus", SCRIPT_PATH)
|
||||
MODULE = importlib.util.module_from_spec(SPEC)
|
||||
assert SPEC.loader is not None
|
||||
SPEC.loader.exec_module(MODULE)
|
||||
|
||||
|
||||
class PrepareAppleNLHardeningCorpusTests(unittest.TestCase):
|
||||
def test_prepare_removes_id_and_normalized_text_overlap(self) -> None:
|
||||
training = [
|
||||
{"id": "train-1", "text": "Keep me", "language": "en", "split": "train"},
|
||||
{"id": "shared-id", "text": "Different", "language": "en", "split": "train"},
|
||||
{"id": "train-3", "text": " SAME TEXT ", "language": "en", "split": "train"},
|
||||
]
|
||||
product = [
|
||||
{
|
||||
"id": "shared-id",
|
||||
"text": "Evaluation by id",
|
||||
"language": "en",
|
||||
"split": "validation",
|
||||
},
|
||||
{
|
||||
"id": "evaluation-2",
|
||||
"text": "same text",
|
||||
"language": "en",
|
||||
"split": "test",
|
||||
},
|
||||
{
|
||||
"id": "ignored-train",
|
||||
"text": "Not evaluation",
|
||||
"language": "en",
|
||||
"split": "train",
|
||||
},
|
||||
]
|
||||
|
||||
records, report = MODULE.prepare(training, product)
|
||||
|
||||
self.assertEqual([record["id"] for record in records], [
|
||||
"train-1",
|
||||
"shared-id",
|
||||
"evaluation-2",
|
||||
])
|
||||
self.assertEqual(report["filteredTrainingCount"], 1)
|
||||
self.assertEqual(report["excludedTrainingOverlapCount"], 2)
|
||||
self.assertEqual(report["evaluationOverlapCount"], 0)
|
||||
|
||||
def test_prepare_requires_evaluation_records(self) -> None:
|
||||
with self.assertRaisesRegex(ValueError, "no evaluation records"):
|
||||
MODULE.prepare(
|
||||
[{"id": "train", "text": "x", "language": "en", "split": "train"}],
|
||||
[{"id": "product", "text": "y", "language": "en", "split": "train"}],
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -11,6 +11,7 @@ private struct CorpusRecord: Codable {
|
||||
let family: String
|
||||
let knownLabels: Set<String>?
|
||||
let sourceDataset: String?
|
||||
let sampleWeight: Double?
|
||||
let task: Bool
|
||||
let question: Bool
|
||||
let invitation: Bool
|
||||
@@ -21,6 +22,10 @@ private struct CorpusRecord: Codable {
|
||||
let blessing: Bool
|
||||
let sentiment: String
|
||||
let replyable: Bool
|
||||
let assistantCommand: Bool?
|
||||
let informationQuery: Bool?
|
||||
let systemNotification: Bool?
|
||||
let domain: String?
|
||||
}
|
||||
|
||||
private struct BinaryMetrics: Codable {
|
||||
@@ -141,6 +146,10 @@ private enum ClassifierID: String, CaseIterable {
|
||||
case followUpReminder
|
||||
case blessing
|
||||
case replyableMessage
|
||||
case assistantCommand
|
||||
case informationQuery
|
||||
case systemNotification
|
||||
case domain
|
||||
case sentiment
|
||||
|
||||
var resourceName: String {
|
||||
@@ -154,6 +163,10 @@ private enum ClassifierID: String, CaseIterable {
|
||||
case .followUpReminder: "FollowUpReminderIntentClassifier"
|
||||
case .blessing: "BlessingIntentClassifier"
|
||||
case .replyableMessage: "ConversationalReplyIntentClassifier"
|
||||
case .assistantCommand: "AssistantCommandIntentClassifier"
|
||||
case .informationQuery: "InformationQueryIntentClassifier"
|
||||
case .systemNotification: "SystemNotificationIntentClassifier"
|
||||
case .domain: "ClipboardDomainClassifier"
|
||||
case .sentiment: "SentimentClassifier"
|
||||
}
|
||||
}
|
||||
@@ -169,6 +182,14 @@ private enum ClassifierID: String, CaseIterable {
|
||||
case .followUpReminder: ["notFollowUpReminder", "followUpReminder"]
|
||||
case .blessing: ["notBlessing", "blessing"]
|
||||
case .replyableMessage: ["notReplyableMessage", "replyableMessage"]
|
||||
case .assistantCommand: ["notAssistantCommand", "assistantCommand"]
|
||||
case .informationQuery: ["notInformationQuery", "informationQuery"]
|
||||
case .systemNotification: ["notSystemNotification", "systemNotification"]
|
||||
case .domain:
|
||||
[
|
||||
"finance", "travel", "calendar", "communication", "media", "smartHome",
|
||||
"shopping", "dining", "health", "weather", "accountService", "generalKnowledge"
|
||||
]
|
||||
case .sentiment: ["negative", "neutral", "positive"]
|
||||
}
|
||||
}
|
||||
@@ -184,7 +205,10 @@ private enum ClassifierID: String, CaseIterable {
|
||||
case .followUpReminder: "followUpReminder"
|
||||
case .blessing: "blessing"
|
||||
case .replyableMessage: "replyableMessage"
|
||||
case .sentiment: nil
|
||||
case .assistantCommand: "assistantCommand"
|
||||
case .informationQuery: "informationQuery"
|
||||
case .systemNotification: "systemNotification"
|
||||
case .domain, .sentiment: nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -297,7 +321,8 @@ private enum ClassifierID: String, CaseIterable {
|
||||
"task_question",
|
||||
"task_statement"
|
||||
]
|
||||
case .question, .replyableMessage, .sentiment:
|
||||
case .question, .replyableMessage, .assistantCommand, .informationQuery,
|
||||
.systemNotification, .domain, .sentiment:
|
||||
return []
|
||||
}
|
||||
}
|
||||
@@ -314,7 +339,8 @@ private enum ClassifierID: String, CaseIterable {
|
||||
0.75
|
||||
case .confirmationDecision:
|
||||
0.90
|
||||
case .question, .replyableMessage, .sentiment:
|
||||
case .question, .replyableMessage, .assistantCommand, .informationQuery,
|
||||
.systemNotification, .domain, .sentiment:
|
||||
0
|
||||
}
|
||||
}
|
||||
@@ -334,12 +360,31 @@ private enum ClassifierID: String, CaseIterable {
|
||||
case .blessing:
|
||||
record.blessing ? "blessing" : "notBlessing"
|
||||
case .replyableMessage: record.replyable ? "replyableMessage" : "notReplyableMessage"
|
||||
case .assistantCommand:
|
||||
record.assistantCommand == true ? "assistantCommand" : "notAssistantCommand"
|
||||
case .informationQuery:
|
||||
record.informationQuery == true ? "informationQuery" : "notInformationQuery"
|
||||
case .systemNotification:
|
||||
record.systemNotification == true ? "systemNotification" : "notSystemNotification"
|
||||
case .domain:
|
||||
record.domain ?? { preconditionFailure("Known domain record is missing domain") }()
|
||||
case .sentiment: record.sentiment
|
||||
}
|
||||
}
|
||||
|
||||
func hasKnownLabel(in record: CorpusRecord) -> Bool {
|
||||
record.knownLabels?.contains(rawValue) ?? true
|
||||
if let knownLabels = record.knownLabels {
|
||||
return knownLabels.contains(rawValue)
|
||||
|| (self == .replyableMessage && knownLabels.contains("replyable"))
|
||||
}
|
||||
// Legacy product corpora predate knownLabels and only fully annotate
|
||||
// the original nine intents plus sentiment. New fields must stay unknown.
|
||||
switch self {
|
||||
case .assistantCommand, .informationQuery, .systemNotification, .domain:
|
||||
return false
|
||||
default:
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -397,6 +442,7 @@ private let reportURL = resolvedURL(
|
||||
flag: "--report",
|
||||
defaultPath: "ModelTraining/ClipboardSemantics/evaluation-report.json"
|
||||
)
|
||||
private let requestedLanguage = commandLineValue(after: "--language")
|
||||
|
||||
private func loadCorpus() throws -> [CorpusRecord] {
|
||||
let content = try String(contentsOf: corpusURL, encoding: .utf8)
|
||||
@@ -419,9 +465,13 @@ private func sourceBalancedPrefix(
|
||||
classifier: ClassifierID,
|
||||
label: String
|
||||
) -> [CorpusRecord] {
|
||||
guard records.count > limit else { return records }
|
||||
// MLTextClassifier's dictionary API has no per-example weight parameter.
|
||||
// Quantized weight buckets plus deterministic smooth weighted round-robin
|
||||
// preserve registry weights without introducing nondeterministic duplication.
|
||||
var grouped = Dictionary(grouping: records) {
|
||||
$0.sourceDataset ?? "generated"
|
||||
let weight = min(max($0.sampleWeight ?? 1.0, 0.01), 1.0)
|
||||
let bucket = (weight * 100).rounded() / 100
|
||||
return "\($0.sourceDataset ?? "generated")|weight=\(bucket)"
|
||||
}
|
||||
for source in grouped.keys.sorted() {
|
||||
var generator = SeededGenerator(
|
||||
@@ -431,24 +481,49 @@ private func sourceBalancedPrefix(
|
||||
)
|
||||
)
|
||||
grouped[source]?.shuffle(using: &generator)
|
||||
if let values = grouped[source], let first = values.first {
|
||||
let weight = min(max(first.sampleWeight ?? 1.0, 0.01), 1.0)
|
||||
let weightedCount = max(1, Int((Double(values.count) * weight).rounded()))
|
||||
grouped[source] = Array(values.prefix(weightedCount))
|
||||
}
|
||||
}
|
||||
let sources = grouped.keys.sorted()
|
||||
let weightedTotal = grouped.values.reduce(0) { $0 + $1.count }
|
||||
if weightedTotal <= limit {
|
||||
return sources.flatMap { grouped[$0] ?? [] }
|
||||
}
|
||||
var offsets = Dictionary(uniqueKeysWithValues: sources.map { ($0, 0) })
|
||||
let sourceWeights = Dictionary(uniqueKeysWithValues: sources.map { source in
|
||||
(source, grouped[source]?.first?.sampleWeight ?? 1.0)
|
||||
})
|
||||
var schedulingScores = Dictionary(uniqueKeysWithValues: sources.map { ($0, 0.0) })
|
||||
var selected: [CorpusRecord] = []
|
||||
while selected.count < limit {
|
||||
var addedRecord = false
|
||||
for source in sources where selected.count < limit {
|
||||
let offset = offsets[source] ?? 0
|
||||
guard let values = grouped[source], values.indices.contains(offset) else {
|
||||
continue
|
||||
}
|
||||
selected.append(values[offset])
|
||||
offsets[source] = offset + 1
|
||||
addedRecord = true
|
||||
let available = sources.filter {
|
||||
let offset = offsets[$0] ?? 0
|
||||
return grouped[$0]?.indices.contains(offset) == true
|
||||
}
|
||||
if !addedRecord {
|
||||
if available.isEmpty {
|
||||
break
|
||||
}
|
||||
let totalWeight = available.reduce(0.0) {
|
||||
$0 + max(sourceWeights[$1] ?? 1.0, 0.01)
|
||||
}
|
||||
for source in available {
|
||||
schedulingScores[source, default: 0] += max(
|
||||
sourceWeights[source] ?? 1.0,
|
||||
0.01
|
||||
)
|
||||
}
|
||||
let source = available.max {
|
||||
let left = schedulingScores[$0, default: 0]
|
||||
let right = schedulingScores[$1, default: 0]
|
||||
return left == right ? $0 > $1 : left < right
|
||||
}!
|
||||
let offset = offsets[source] ?? 0
|
||||
selected.append(grouped[source]![offset])
|
||||
offsets[source] = offset + 1
|
||||
schedulingScores[source, default: 0] -= totalWeight
|
||||
}
|
||||
return selected
|
||||
}
|
||||
@@ -462,18 +537,28 @@ private func curatedTrainingRecords(
|
||||
}
|
||||
let generatedRecords = knownRecords.filter { $0.sourceDataset == nil }
|
||||
let openRecords = knownRecords.filter { $0.sourceDataset != nil }
|
||||
guard !openRecords.isEmpty else { return generatedRecords }
|
||||
let weightedGeneratedRecords = sourceBalancedPrefix(
|
||||
generatedRecords,
|
||||
limit: generatedRecords.count,
|
||||
classifier: classifier,
|
||||
label: "generated"
|
||||
)
|
||||
guard !openRecords.isEmpty else { return weightedGeneratedRecords }
|
||||
|
||||
let generatedByLabel = Dictionary(grouping: generatedRecords) {
|
||||
let generatedByLabel = Dictionary(grouping: weightedGeneratedRecords) {
|
||||
classifier.label(for: $0)
|
||||
}
|
||||
let openByLabel = Dictionary(grouping: openRecords) {
|
||||
classifier.label(for: $0)
|
||||
}
|
||||
let openOnlyBalancedCount = classifier.labels
|
||||
.compactMap { openByLabel[$0]?.count }
|
||||
.min() ?? 0
|
||||
let multiplier = switch classifier {
|
||||
case .blessing:
|
||||
2.0
|
||||
case .task, .question, .complaint, .confirmationDecision, .sentiment:
|
||||
case .task, .question, .complaint, .confirmationDecision, .assistantCommand,
|
||||
.informationQuery, .systemNotification, .domain, .sentiment:
|
||||
1.0
|
||||
case .invitation, .scheduleNegotiation, .followUpReminder, .replyableMessage:
|
||||
0.5
|
||||
@@ -481,7 +566,8 @@ private func curatedTrainingRecords(
|
||||
|
||||
let selectedOpenRecords = classifier.labels.flatMap { label in
|
||||
let generatedCount = generatedByLabel[label]?.count ?? 0
|
||||
let limit = max(1, Int((Double(generatedCount) * multiplier).rounded()))
|
||||
let anchorCount = generatedCount > 0 ? generatedCount : openOnlyBalancedCount
|
||||
let limit = max(1, Int((Double(anchorCount) * multiplier).rounded()))
|
||||
return sourceBalancedPrefix(
|
||||
openByLabel[label] ?? [],
|
||||
limit: limit,
|
||||
@@ -489,7 +575,7 @@ private func curatedTrainingRecords(
|
||||
label: label
|
||||
)
|
||||
}
|
||||
return generatedRecords + selectedOpenRecords
|
||||
return weightedGeneratedRecords + selectedOpenRecords
|
||||
}
|
||||
|
||||
private func balancedTexts(
|
||||
@@ -1192,15 +1278,36 @@ private func writeJSON<T: Encodable>(_ value: T, to url: URL) throws {
|
||||
}
|
||||
|
||||
private func main() throws {
|
||||
let records = try loadCorpus()
|
||||
let loadedRecords = try loadCorpus()
|
||||
let records = requestedLanguage.map { language in
|
||||
loadedRecords.filter { $0.language == language }
|
||||
} ?? loadedRecords
|
||||
precondition(!records.isEmpty, "No corpus records match the requested language")
|
||||
let trainingRecords = records.filter { $0.split == "train" }
|
||||
let validationRecords = records.filter { $0.split == "validation" }
|
||||
let testRecords = records.filter { $0.split == "test" }
|
||||
let goldenRecords = records.filter { $0.split == "golden" }
|
||||
let algorithms = selectedAlgorithms()
|
||||
let classifiers = selectedClassifiers()
|
||||
let requestedClassifiers = selectedClassifiers()
|
||||
let classifiers = requestedClassifiers.filter { classifier in
|
||||
let trainingLabels = Set(
|
||||
trainingRecords
|
||||
.filter { classifier.hasKnownLabel(in: $0) }
|
||||
.map { classifier.label(for: $0) }
|
||||
)
|
||||
return Set(classifier.labels).isSubset(of: trainingLabels)
|
||||
&& [validationRecords, testRecords, goldenRecords].allSatisfy {
|
||||
!$0.filter { classifier.hasKnownLabel(in: $0) }.isEmpty
|
||||
}
|
||||
}
|
||||
for classifier in requestedClassifiers where !classifiers.contains(classifier) {
|
||||
print(
|
||||
"TRAIN_SKIPPED classifier=\(classifier.rawValue) "
|
||||
+ "reason=insufficient-known-label-coverage"
|
||||
)
|
||||
}
|
||||
precondition(!algorithms.isEmpty, "No supported algorithms requested")
|
||||
precondition(!classifiers.isEmpty, "No supported classifiers requested")
|
||||
precondition(!classifiers.isEmpty, "No classifiers have sufficient known-label coverage")
|
||||
|
||||
try fileManager.createDirectory(
|
||||
at: resourceDirectory,
|
||||
@@ -1212,6 +1319,15 @@ private func main() throws {
|
||||
|
||||
for classifierID in classifiers {
|
||||
var candidates: [TrainedCandidate] = []
|
||||
let knownValidationRecords = validationRecords.filter {
|
||||
classifierID.hasKnownLabel(in: $0)
|
||||
}
|
||||
let knownTestRecords = testRecords.filter {
|
||||
classifierID.hasKnownLabel(in: $0)
|
||||
}
|
||||
let knownGoldenRecords = goldenRecords.filter {
|
||||
classifierID.hasKnownLabel(in: $0)
|
||||
}
|
||||
for algorithm in algorithms {
|
||||
do {
|
||||
candidates.append(
|
||||
@@ -1219,9 +1335,9 @@ private func main() throws {
|
||||
classifierID: classifierID,
|
||||
algorithm: algorithm,
|
||||
trainingRecords: trainingRecords,
|
||||
validationRecords: validationRecords,
|
||||
testRecords: testRecords,
|
||||
goldenRecords: goldenRecords
|
||||
validationRecords: knownValidationRecords,
|
||||
testRecords: knownTestRecords,
|
||||
goldenRecords: knownGoldenRecords
|
||||
)
|
||||
)
|
||||
} catch {
|
||||
@@ -1288,7 +1404,9 @@ private func main() throws {
|
||||
goldenCount: goldenRecords.count,
|
||||
selectionPolicy:
|
||||
"Open records with unknown labels are excluded per classifier, and source-balanced "
|
||||
+ "caps anchor each label to the reviewed generated corpus size. "
|
||||
+ "caps anchor each label to the reviewed generated corpus size. Because Create ML "
|
||||
+ "does not expose per-example weights, registry sampleWeight values are applied as "
|
||||
+ "deterministic quantized quotas with smooth weighted source scheduling. "
|
||||
+ "Validation only: global and per-language binary thresholds require precision "
|
||||
+ ">= 0.97, then maximize recall; languages with fewer than 20 examples per class "
|
||||
+ "fall back to the global threshold. "
|
||||
@@ -1300,7 +1418,7 @@ private func main() throws {
|
||||
try writeJSON(report, to: reportURL)
|
||||
try writeJSON(
|
||||
ModelManifest(
|
||||
schemaVersion: 2,
|
||||
schemaVersion: 4,
|
||||
generatedAt: generatedAt,
|
||||
corpusRecordCount: records.count,
|
||||
classifiers: manifestClassifiers
|
||||
|
||||
@@ -0,0 +1,491 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fine-tune bilingual Tiny Transformer challengers for taxonomy-v6 intents."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import random
|
||||
import resource
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from torch.utils.data import DataLoader, Dataset
|
||||
from transformers import AutoModelForSequenceClassification, AutoTokenizer
|
||||
|
||||
|
||||
INTENTS = (
|
||||
"task",
|
||||
"question",
|
||||
"invitation",
|
||||
"complaint",
|
||||
"scheduleNegotiation",
|
||||
"confirmationDecision",
|
||||
"followUpReminder",
|
||||
"blessing",
|
||||
"replyableMessage",
|
||||
"assistantCommand",
|
||||
"informationQuery",
|
||||
"systemNotification",
|
||||
)
|
||||
LANGUAGE_MODELS = {
|
||||
"en": "en",
|
||||
"zh-Hans": "zh",
|
||||
}
|
||||
|
||||
|
||||
def parse_arguments() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--corpus", type=Path, required=True)
|
||||
parser.add_argument("--output-directory", type=Path, required=True)
|
||||
parser.add_argument("--english-model", type=Path, required=True)
|
||||
parser.add_argument("--chinese-model", type=Path, required=True)
|
||||
parser.add_argument("--epochs", type=int, default=3)
|
||||
parser.add_argument("--batch-size", type=int, default=128)
|
||||
parser.add_argument("--max-length", type=int, default=96)
|
||||
parser.add_argument("--learning-rate", type=float, default=3e-4)
|
||||
parser.add_argument("--seed", type=int, default=20260828)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def set_seed(seed: int) -> None:
|
||||
random.seed(seed)
|
||||
np.random.seed(seed)
|
||||
torch.manual_seed(seed)
|
||||
if torch.backends.mps.is_available():
|
||||
torch.mps.manual_seed(seed)
|
||||
|
||||
|
||||
def intent_value(record: dict, intent: str) -> bool:
|
||||
if intent == "replyableMessage":
|
||||
return bool(record.get(intent, record.get("replyable", False)))
|
||||
return bool(record.get(intent, False))
|
||||
|
||||
|
||||
def intent_mask(record: dict) -> list[float]:
|
||||
known_labels = record.get("knownLabels")
|
||||
if known_labels is None:
|
||||
return [1.0] * len(INTENTS)
|
||||
known = set(known_labels)
|
||||
return [float(intent in known) for intent in INTENTS]
|
||||
|
||||
|
||||
def load_records(path: Path) -> dict[str, dict[str, list[dict]]]:
|
||||
records = {
|
||||
language: {"train": [], "validation": [], "test": [], "golden": []}
|
||||
for language in LANGUAGE_MODELS
|
||||
}
|
||||
with path.open(encoding="utf-8") as stream:
|
||||
for line in stream:
|
||||
record = json.loads(line)
|
||||
language = record.get("language")
|
||||
split = record.get("split")
|
||||
if language in records and split in records[language]:
|
||||
records[language][split].append(record)
|
||||
return records
|
||||
|
||||
|
||||
class IntentDataset(Dataset):
|
||||
def __init__(
|
||||
self,
|
||||
records: list[dict],
|
||||
tokenizer: AutoTokenizer,
|
||||
max_length: int,
|
||||
) -> None:
|
||||
encoded = tokenizer(
|
||||
[record["text"] for record in records],
|
||||
max_length=max_length,
|
||||
padding="max_length",
|
||||
truncation=True,
|
||||
return_tensors="pt",
|
||||
)
|
||||
self.inputs = dict(encoded)
|
||||
self.labels = torch.tensor(
|
||||
[
|
||||
[float(intent_value(record, intent)) for intent in INTENTS]
|
||||
for record in records
|
||||
],
|
||||
dtype=torch.float32,
|
||||
)
|
||||
self.masks = torch.tensor(
|
||||
[intent_mask(record) for record in records],
|
||||
dtype=torch.float32,
|
||||
)
|
||||
self.weights = torch.tensor(
|
||||
[float(record.get("sampleWeight", 1.0)) for record in records],
|
||||
dtype=torch.float32,
|
||||
)
|
||||
|
||||
def __len__(self) -> int:
|
||||
return self.labels.shape[0]
|
||||
|
||||
def __getitem__(self, index: int) -> dict[str, torch.Tensor]:
|
||||
item = {key: value[index] for key, value in self.inputs.items()}
|
||||
item["labels"] = self.labels[index]
|
||||
item["masks"] = self.masks[index]
|
||||
item["weights"] = self.weights[index]
|
||||
return item
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Metrics:
|
||||
true_positive: int
|
||||
true_negative: int
|
||||
false_positive: int
|
||||
false_negative: int
|
||||
precision: float
|
||||
recall: float
|
||||
f1: float
|
||||
|
||||
def as_dict(self) -> dict:
|
||||
return {
|
||||
"truePositive": self.true_positive,
|
||||
"trueNegative": self.true_negative,
|
||||
"falsePositive": self.false_positive,
|
||||
"falseNegative": self.false_negative,
|
||||
"precision": round(self.precision, 6),
|
||||
"recall": round(self.recall, 6),
|
||||
"f1": round(self.f1, 6),
|
||||
}
|
||||
|
||||
|
||||
def calculate_metrics(
|
||||
expected: np.ndarray,
|
||||
predicted: np.ndarray,
|
||||
mask: np.ndarray,
|
||||
) -> Metrics:
|
||||
expected = expected[mask.astype(bool)].astype(bool)
|
||||
predicted = predicted[mask.astype(bool)].astype(bool)
|
||||
true_positive = int(np.sum(expected & predicted))
|
||||
true_negative = int(np.sum(~expected & ~predicted))
|
||||
false_positive = int(np.sum(~expected & predicted))
|
||||
false_negative = int(np.sum(expected & ~predicted))
|
||||
precision_denominator = true_positive + false_positive
|
||||
recall_denominator = true_positive + false_negative
|
||||
precision = (
|
||||
true_positive / precision_denominator if precision_denominator else 0.0
|
||||
)
|
||||
recall = true_positive / recall_denominator if recall_denominator else 0.0
|
||||
f1 = (
|
||||
2 * precision * recall / (precision + recall)
|
||||
if precision + recall
|
||||
else 0.0
|
||||
)
|
||||
return Metrics(
|
||||
true_positive,
|
||||
true_negative,
|
||||
false_positive,
|
||||
false_negative,
|
||||
precision,
|
||||
recall,
|
||||
f1,
|
||||
)
|
||||
|
||||
|
||||
def predict(
|
||||
model: AutoModelForSequenceClassification,
|
||||
tokenizer: AutoTokenizer,
|
||||
records: list[dict],
|
||||
device: torch.device,
|
||||
max_length: int,
|
||||
batch_size: int,
|
||||
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
|
||||
model.eval()
|
||||
probabilities: list[np.ndarray] = []
|
||||
expected: list[np.ndarray] = []
|
||||
masks: list[np.ndarray] = []
|
||||
with torch.inference_mode():
|
||||
for start in range(0, len(records), batch_size):
|
||||
batch = records[start : start + batch_size]
|
||||
encoded = tokenizer(
|
||||
[record["text"] for record in batch],
|
||||
max_length=max_length,
|
||||
padding="max_length",
|
||||
truncation=True,
|
||||
return_tensors="pt",
|
||||
)
|
||||
inputs = {key: value.to(device) for key, value in encoded.items()}
|
||||
probabilities.append(
|
||||
torch.sigmoid(model(**inputs).logits).cpu().numpy()
|
||||
)
|
||||
expected.append(
|
||||
np.array(
|
||||
[
|
||||
[float(intent_value(record, intent)) for intent in INTENTS]
|
||||
for record in batch
|
||||
],
|
||||
dtype=np.float32,
|
||||
)
|
||||
)
|
||||
masks.append(
|
||||
np.array([intent_mask(record) for record in batch], dtype=np.float32)
|
||||
)
|
||||
return (
|
||||
np.concatenate(probabilities),
|
||||
np.concatenate(expected),
|
||||
np.concatenate(masks),
|
||||
)
|
||||
|
||||
|
||||
def choose_thresholds(
|
||||
probabilities: np.ndarray,
|
||||
expected: np.ndarray,
|
||||
masks: np.ndarray,
|
||||
) -> np.ndarray:
|
||||
thresholds: list[float] = []
|
||||
for index in range(len(INTENTS)):
|
||||
known = masks[:, index].astype(bool)
|
||||
labels = expected[known, index]
|
||||
if not np.any(labels == 1) or not np.any(labels == 0):
|
||||
thresholds.append(0.5)
|
||||
continue
|
||||
candidates = []
|
||||
for threshold in np.linspace(0.05, 0.95, 91):
|
||||
metrics = calculate_metrics(
|
||||
expected[:, index],
|
||||
probabilities[:, index] >= threshold,
|
||||
masks[:, index],
|
||||
)
|
||||
candidates.append((metrics.f1, metrics.precision, metrics.recall, threshold))
|
||||
thresholds.append(float(max(candidates)[3]))
|
||||
return np.array(thresholds, dtype=np.float32)
|
||||
|
||||
|
||||
def summarize(
|
||||
probabilities: np.ndarray,
|
||||
expected: np.ndarray,
|
||||
masks: np.ndarray,
|
||||
thresholds: np.ndarray,
|
||||
) -> dict:
|
||||
per_intent = {}
|
||||
supported_metrics = []
|
||||
for index, intent in enumerate(INTENTS):
|
||||
metrics = calculate_metrics(
|
||||
expected[:, index],
|
||||
probabilities[:, index] >= thresholds[index],
|
||||
masks[:, index],
|
||||
)
|
||||
known_count = int(np.sum(masks[:, index]))
|
||||
positive_count = int(np.sum(expected[:, index] * masks[:, index]))
|
||||
per_intent[intent] = {
|
||||
"knownCount": known_count,
|
||||
"positiveCount": positive_count,
|
||||
**metrics.as_dict(),
|
||||
}
|
||||
if positive_count > 0:
|
||||
supported_metrics.append(metrics)
|
||||
return {
|
||||
"records": int(expected.shape[0]),
|
||||
"evaluatedIntentCount": len(supported_metrics),
|
||||
"macroPrecision": round(
|
||||
float(np.mean([metrics.precision for metrics in supported_metrics])), 6
|
||||
),
|
||||
"macroRecall": round(
|
||||
float(np.mean([metrics.recall for metrics in supported_metrics])), 6
|
||||
),
|
||||
"macroF1": round(
|
||||
float(np.mean([metrics.f1 for metrics in supported_metrics])), 6
|
||||
),
|
||||
"perIntent": per_intent,
|
||||
}
|
||||
|
||||
|
||||
def train_language(
|
||||
language: str,
|
||||
model_path: Path,
|
||||
records: dict[str, list[dict]],
|
||||
arguments: argparse.Namespace,
|
||||
device: torch.device,
|
||||
) -> dict:
|
||||
set_seed(arguments.seed)
|
||||
tokenizer = AutoTokenizer.from_pretrained(model_path, local_files_only=True)
|
||||
model = AutoModelForSequenceClassification.from_pretrained(
|
||||
model_path,
|
||||
local_files_only=True,
|
||||
num_labels=len(INTENTS),
|
||||
problem_type="multi_label_classification",
|
||||
ignore_mismatched_sizes=True,
|
||||
).to(device)
|
||||
dataset = IntentDataset(records["train"], tokenizer, arguments.max_length)
|
||||
generator = torch.Generator().manual_seed(arguments.seed)
|
||||
loader = DataLoader(
|
||||
dataset,
|
||||
batch_size=arguments.batch_size,
|
||||
shuffle=True,
|
||||
generator=generator,
|
||||
)
|
||||
weighted_positive = (dataset.labels * dataset.masks) * dataset.weights[:, None]
|
||||
weighted_known = dataset.masks * dataset.weights[:, None]
|
||||
positive_counts = weighted_positive.sum(dim=0)
|
||||
negative_counts = weighted_known.sum(dim=0) - positive_counts
|
||||
positive_weights = torch.clamp(
|
||||
negative_counts / torch.clamp(positive_counts, min=1),
|
||||
min=1,
|
||||
max=20,
|
||||
).to(device)
|
||||
criterion = torch.nn.BCEWithLogitsLoss(
|
||||
pos_weight=positive_weights,
|
||||
reduction="none",
|
||||
)
|
||||
optimizer = torch.optim.AdamW(
|
||||
model.parameters(),
|
||||
lr=arguments.learning_rate,
|
||||
weight_decay=0.01,
|
||||
)
|
||||
|
||||
epoch_losses = []
|
||||
training_started = time.perf_counter()
|
||||
for epoch in range(arguments.epochs):
|
||||
model.train()
|
||||
running_loss = 0.0
|
||||
for batch in loader:
|
||||
labels = batch.pop("labels").to(device)
|
||||
masks = batch.pop("masks").to(device)
|
||||
weights = batch.pop("weights").to(device).unsqueeze(1)
|
||||
inputs = {key: value.to(device) for key, value in batch.items()}
|
||||
optimizer.zero_grad(set_to_none=True)
|
||||
losses = criterion(model(**inputs).logits, labels)
|
||||
weighted_masks = masks * weights
|
||||
loss = (losses * weighted_masks).sum() / weighted_masks.sum().clamp(min=1)
|
||||
loss.backward()
|
||||
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
|
||||
optimizer.step()
|
||||
running_loss += float(loss.detach().cpu())
|
||||
average_loss = running_loss / max(len(loader), 1)
|
||||
epoch_losses.append(round(average_loss, 6))
|
||||
print(
|
||||
f"TRAIN language={language} epoch={epoch + 1}/{arguments.epochs} "
|
||||
f"loss={average_loss:.6f}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
predictions = {}
|
||||
for split in ("validation", "test", "golden"):
|
||||
predictions[split] = predict(
|
||||
model,
|
||||
tokenizer,
|
||||
records[split],
|
||||
device,
|
||||
arguments.max_length,
|
||||
arguments.batch_size,
|
||||
)
|
||||
thresholds = choose_thresholds(*predictions["validation"])
|
||||
evaluations = {
|
||||
split: summarize(*values, thresholds)
|
||||
for split, values in predictions.items()
|
||||
}
|
||||
|
||||
sample_text = records["test"][0]["text"]
|
||||
encoded = tokenizer(
|
||||
sample_text,
|
||||
max_length=arguments.max_length,
|
||||
padding="max_length",
|
||||
truncation=True,
|
||||
return_tensors="pt",
|
||||
)
|
||||
inputs = {key: value.to(device) for key, value in encoded.items()}
|
||||
model.eval()
|
||||
with torch.inference_mode():
|
||||
started = time.perf_counter()
|
||||
model(**inputs)
|
||||
if device.type == "mps":
|
||||
torch.mps.synchronize()
|
||||
cold_ms = (time.perf_counter() - started) * 1_000
|
||||
warm_samples = []
|
||||
for _ in range(100):
|
||||
started = time.perf_counter()
|
||||
model(**inputs)
|
||||
if device.type == "mps":
|
||||
torch.mps.synchronize()
|
||||
warm_samples.append((time.perf_counter() - started) * 1_000)
|
||||
|
||||
output = arguments.output_directory / LANGUAGE_MODELS[language]
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
model.save_pretrained(output)
|
||||
tokenizer.save_pretrained(output)
|
||||
model_bytes = sum(path.stat().st_size for path in output.iterdir() if path.is_file())
|
||||
return {
|
||||
"language": language,
|
||||
"baseModel": str(model_path),
|
||||
"trainRecords": len(records["train"]),
|
||||
"epochLosses": epoch_losses,
|
||||
"trainingSeconds": round(time.perf_counter() - training_started, 3),
|
||||
"thresholds": {
|
||||
intent: round(float(thresholds[index]), 4)
|
||||
for index, intent in enumerate(INTENTS)
|
||||
},
|
||||
"evaluations": evaluations,
|
||||
"runtime": {
|
||||
"engine": f"PyTorch eager on {device.type}",
|
||||
"coldMilliseconds": round(cold_ms, 3),
|
||||
"warmMeanMilliseconds": round(float(np.mean(warm_samples)), 3),
|
||||
"warmP95Milliseconds": round(float(np.percentile(warm_samples, 95)), 3),
|
||||
"processMaximumRSSBytes": int(
|
||||
resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
|
||||
),
|
||||
},
|
||||
"savedModelBytes": model_bytes,
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
arguments = parse_arguments()
|
||||
set_seed(arguments.seed)
|
||||
device = torch.device("mps" if torch.backends.mps.is_available() else "cpu")
|
||||
records = load_records(arguments.corpus)
|
||||
model_paths = {
|
||||
"en": arguments.english_model,
|
||||
"zh-Hans": arguments.chinese_model,
|
||||
}
|
||||
language_reports = []
|
||||
for language in ("zh-Hans", "en"):
|
||||
counts = {
|
||||
split: len(split_records)
|
||||
for split, split_records in records[language].items()
|
||||
}
|
||||
print(f"DATA language={language} counts={counts}", flush=True)
|
||||
language_reports.append(
|
||||
train_language(
|
||||
language,
|
||||
model_paths[language],
|
||||
records[language],
|
||||
arguments,
|
||||
device,
|
||||
)
|
||||
)
|
||||
if device.type == "mps":
|
||||
torch.mps.empty_cache()
|
||||
|
||||
report = {
|
||||
"schemaVersion": 1,
|
||||
"purpose": "Taxonomy-v6 Tiny Transformer research challenger",
|
||||
"corpus": str(arguments.corpus),
|
||||
"seed": arguments.seed,
|
||||
"intents": list(INTENTS),
|
||||
"configuration": {
|
||||
"epochs": arguments.epochs,
|
||||
"batchSize": arguments.batch_size,
|
||||
"maxLength": arguments.max_length,
|
||||
"learningRate": arguments.learning_rate,
|
||||
},
|
||||
"languages": language_reports,
|
||||
"limitations": [
|
||||
"Thresholds use only the frozen validation split, which has 20 records per language.",
|
||||
"PyTorch runtime is not directly comparable with Core ML runtime.",
|
||||
"The Chinese UER checkpoint does not declare a model-weight license in its model card.",
|
||||
],
|
||||
}
|
||||
arguments.output_directory.mkdir(parents=True, exist_ok=True)
|
||||
report_path = arguments.output_directory / "training-evaluation-report.json"
|
||||
report_path.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(f"REPORT {report_path}", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user