chore(semantics): add v6 release gate pipeline
- Add reproducible v6 boundary, blessing, and consensus-adjudication corpora, plus the tiny-transformer trainer and v6 release-gate evaluator that gate every candidate on the deployed baselines. - Wire consensus-label merging, product-policy anchor evaluation, and sealed blessing benchmark review with their pytest coverage. - Refresh open-training corpus generation, iterative retraining runner, and random-holdout evaluation so v6 candidates can be benchmarked end-to-end.
This commit is contained in:
Rocky
@@ -0,0 +1,268 @@
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
import adjudicate_consensus_conflicts as adjudication
|
||||
|
||||
|
||||
class AdjudicationTests(unittest.TestCase):
|
||||
def test_prepare_adds_all_product_policy_fields(self):
|
||||
with tempfile.TemporaryDirectory() as raw_directory:
|
||||
directory = Path(raw_directory)
|
||||
conflicts = directory / "conflicts.jsonl"
|
||||
self._write(
|
||||
conflicts,
|
||||
[
|
||||
{
|
||||
"id": "one",
|
||||
"text": "Could you send the report?",
|
||||
"language": "en",
|
||||
"unresolvedFields": ["sentiment"],
|
||||
}
|
||||
],
|
||||
)
|
||||
queue = directory / "queue.jsonl"
|
||||
report = adjudication.prepare(
|
||||
SimpleNamespace(
|
||||
conflicts=conflicts,
|
||||
queue=queue,
|
||||
chunk_directory=directory / "chunks",
|
||||
chunk_size=10,
|
||||
report=directory / "report.json",
|
||||
include_product_policy_fields=True,
|
||||
)
|
||||
)
|
||||
|
||||
fields = self._read(queue)[0]["unresolvedFields"]
|
||||
self.assertTrue(set(adjudication.PRODUCT_POLICY_FIELDS) <= set(fields))
|
||||
self.assertIn("sentiment", fields)
|
||||
self.assertTrue(report["includesProductPolicyFields"])
|
||||
|
||||
def test_requires_evidence_from_input_text(self):
|
||||
queue = {
|
||||
"id": "one",
|
||||
"text": "Could you send the report?",
|
||||
"unresolvedFields": ["task"],
|
||||
}
|
||||
record = self._adjudication("one", task="true")
|
||||
record["evidence"]["task"] = "not present"
|
||||
|
||||
with self.assertRaisesRegex(ValueError, "exact text quote"):
|
||||
adjudication.validate_adjudication(record, queue)
|
||||
|
||||
def test_accepts_matching_high_confidence_adjudication(self):
|
||||
report, accepted, excluded, remaining = self._merge(
|
||||
first=self._adjudication("one", task="true"),
|
||||
second=self._adjudication("one", task="true"),
|
||||
)
|
||||
|
||||
self.assertEqual(1, report["acceptedTierCCount"])
|
||||
self.assertEqual(0, report["remainingHumanReviewCount"])
|
||||
self.assertTrue(accepted[0]["task"])
|
||||
self.assertEqual("C", accepted[0]["labelQualityTier"])
|
||||
self.assertEqual(0.35, accepted[0]["sampleWeight"])
|
||||
self.assertEqual([], excluded)
|
||||
self.assertEqual([], remaining)
|
||||
|
||||
def test_keeps_disagreement_for_human_review(self):
|
||||
report, accepted, excluded, remaining = self._merge(
|
||||
first=self._adjudication("one", task="true"),
|
||||
second=self._adjudication("one", task="false"),
|
||||
)
|
||||
|
||||
self.assertEqual(0, report["acceptedTierCCount"])
|
||||
self.assertEqual([], accepted)
|
||||
self.assertEqual([], excluded)
|
||||
self.assertEqual(
|
||||
["adjudicator-disagreement"],
|
||||
remaining[0]["aiAdjudication"]["rejectedFields"]["task"],
|
||||
)
|
||||
|
||||
def test_keeps_low_confidence_for_human_review(self):
|
||||
first = self._adjudication("one", task="true")
|
||||
first["confidence"]["task"] = 0.89
|
||||
report, _, _, remaining = self._merge(
|
||||
first=first,
|
||||
second=self._adjudication("one", task="true"),
|
||||
)
|
||||
|
||||
self.assertEqual(1, report["remainingHumanReviewCount"])
|
||||
self.assertEqual(
|
||||
["low-confidence"],
|
||||
remaining[0]["aiAdjudication"]["rejectedFields"]["task"],
|
||||
)
|
||||
|
||||
def test_excludes_matching_high_confidence_device_command(self):
|
||||
first = self._adjudication("one", task="true")
|
||||
second = self._adjudication("one", task="true")
|
||||
for record in (first, second):
|
||||
record["recordDisposition"] = "exclude-device-command"
|
||||
|
||||
report, accepted, excluded, remaining = self._merge(first, second)
|
||||
|
||||
self.assertEqual(1, report["excludedDeviceCommandCount"])
|
||||
self.assertEqual([], accepted)
|
||||
self.assertEqual("exclude-device-command", excluded[0]["disposition"])
|
||||
self.assertEqual([], remaining)
|
||||
|
||||
def test_keeps_disposition_disagreement_for_human_review(self):
|
||||
first = self._adjudication("one", task="true")
|
||||
second = self._adjudication("one", task="true")
|
||||
second["recordDisposition"] = "exclude-device-command"
|
||||
|
||||
report, accepted, excluded, remaining = self._merge(first, second)
|
||||
|
||||
self.assertEqual(1, report["remainingHumanReviewCount"])
|
||||
self.assertEqual([], accepted)
|
||||
self.assertEqual([], excluded)
|
||||
self.assertEqual(
|
||||
["adjudicator-disagreement"],
|
||||
remaining[0]["aiAdjudication"]["rejectedFields"][
|
||||
"recordDisposition"
|
||||
],
|
||||
)
|
||||
|
||||
def test_review_sample_is_unique_and_includes_decision_template(self):
|
||||
with tempfile.TemporaryDirectory() as raw_directory:
|
||||
directory = Path(raw_directory)
|
||||
remaining = directory / "remaining.jsonl"
|
||||
records = []
|
||||
for index, field in enumerate(("task", "recordDisposition", "task")):
|
||||
first = self._adjudication(str(index), task="true")
|
||||
second = self._adjudication(str(index), task="false")
|
||||
records.append(
|
||||
{
|
||||
"id": str(index),
|
||||
"text": "Could you send the report?",
|
||||
"language": "en" if index % 2 else "zh-Hans",
|
||||
"aiAdjudication": {
|
||||
"rejectedFields": {
|
||||
field: ["adjudicator-disagreement"]
|
||||
},
|
||||
"adjudicatorA": first,
|
||||
"adjudicatorB": second,
|
||||
},
|
||||
}
|
||||
)
|
||||
self._write(remaining, records)
|
||||
sample = directory / "sample.jsonl"
|
||||
report = adjudication.review_sample(
|
||||
SimpleNamespace(
|
||||
remaining=remaining,
|
||||
sample=sample,
|
||||
sample_size=3,
|
||||
report=directory / "report.json",
|
||||
)
|
||||
)
|
||||
output = self._read(sample)
|
||||
|
||||
self.assertEqual(3, report["sampleCount"])
|
||||
self.assertEqual(3, len({record["id"] for record in output}))
|
||||
self.assertIsNone(output[0]["humanDecision"].popitem()[1])
|
||||
disposition = next(
|
||||
record
|
||||
for record in output
|
||||
if "recordDisposition" in record["fieldReviews"]
|
||||
)
|
||||
self.assertEqual(
|
||||
"keep",
|
||||
disposition["fieldReviews"]["recordDisposition"][
|
||||
"adjudicatorA"
|
||||
]["value"],
|
||||
)
|
||||
|
||||
def _merge(self, first, second):
|
||||
with tempfile.TemporaryDirectory() as raw_directory:
|
||||
directory = Path(raw_directory)
|
||||
conflicts = directory / "conflicts.jsonl"
|
||||
queue = directory / "queue.jsonl"
|
||||
first_path = directory / "first.jsonl"
|
||||
second_path = directory / "second.jsonl"
|
||||
self._write(
|
||||
conflicts,
|
||||
[
|
||||
{
|
||||
"id": "one",
|
||||
"text": "Could you send the report?",
|
||||
"language": "en",
|
||||
"unresolvedFields": ["task"],
|
||||
"modelVotes": {
|
||||
**{
|
||||
label: {"false": 4}
|
||||
for label in adjudication.INTENT_LABELS
|
||||
},
|
||||
"sentiment": {"neutral": 4},
|
||||
},
|
||||
}
|
||||
],
|
||||
)
|
||||
self._write(
|
||||
queue,
|
||||
[
|
||||
{
|
||||
"id": "one",
|
||||
"text": "Could you send the report?",
|
||||
"language": "en",
|
||||
"unresolvedFields": ["task"],
|
||||
}
|
||||
],
|
||||
)
|
||||
self._write(first_path, [first])
|
||||
self._write(second_path, [second])
|
||||
accepted_path = directory / "accepted.jsonl"
|
||||
excluded_path = directory / "excluded.jsonl"
|
||||
remaining_path = directory / "remaining.jsonl"
|
||||
report = adjudication.merge(
|
||||
SimpleNamespace(
|
||||
conflicts=conflicts,
|
||||
queue=queue,
|
||||
adjudicator_a=[first_path],
|
||||
adjudicator_b=[second_path],
|
||||
adjudicator_a_name="first",
|
||||
adjudicator_b_name="second",
|
||||
minimum_confidence=0.9,
|
||||
accepted=accepted_path,
|
||||
excluded=excluded_path,
|
||||
remaining=remaining_path,
|
||||
report=directory / "report.json",
|
||||
)
|
||||
)
|
||||
return (
|
||||
report,
|
||||
self._read(accepted_path),
|
||||
self._read(excluded_path),
|
||||
self._read(remaining_path),
|
||||
)
|
||||
|
||||
def _adjudication(self, identifier, task):
|
||||
return {
|
||||
"id": identifier,
|
||||
"recordDisposition": "keep",
|
||||
"dispositionConfidence": 0.95,
|
||||
"dispositionEvidence": "send the report",
|
||||
"resolutions": {"task": task},
|
||||
"confidence": {"task": 0.95},
|
||||
"evidence": {"task": "send the report"},
|
||||
}
|
||||
|
||||
def _write(self, path, records):
|
||||
path.write_text(
|
||||
"\n".join(json.dumps(record) for record in records) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
def _read(self, path):
|
||||
return [
|
||||
json.loads(line)
|
||||
for line in path.read_text(encoding="utf-8").splitlines()
|
||||
if line
|
||||
]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,66 @@
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
import assemble_v6_model_corpus as corpus
|
||||
|
||||
|
||||
class AssembleV6ModelCorpusTests(unittest.TestCase):
|
||||
def write_jsonl(self, path: Path, records: list[dict]) -> None:
|
||||
path.write_text(
|
||||
"".join(json.dumps(record) + "\n" for record in records),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
def test_assembles_disjoint_train_and_evaluation_splits(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
train = root / "train.jsonl"
|
||||
evaluation = root / "evaluation.jsonl"
|
||||
self.write_jsonl(
|
||||
train,
|
||||
[{"id": "train-1", "text": "hello", "split": "train", "language": "en"}],
|
||||
)
|
||||
self.write_jsonl(
|
||||
evaluation,
|
||||
[
|
||||
{
|
||||
"id": "test-1",
|
||||
"text": "world",
|
||||
"split": "test",
|
||||
"language": "en",
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
records, report = corpus.assemble(train, evaluation)
|
||||
|
||||
self.assertEqual(2, len(records))
|
||||
self.assertEqual({"test": 1, "train": 1}, report["splitCounts"])
|
||||
self.assertEqual(0, report["evaluationOverlapCount"])
|
||||
|
||||
def test_rejects_nfkc_casefold_overlap(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
train = root / "train.jsonl"
|
||||
evaluation = root / "evaluation.jsonl"
|
||||
self.write_jsonl(
|
||||
train,
|
||||
[{"id": "train-1", "text": "ＡＢＣ", "split": "train", "language": "en"}],
|
||||
)
|
||||
self.write_jsonl(
|
||||
evaluation,
|
||||
[{"id": "test-1", "text": "abc", "split": "test", "language": "en"}],
|
||||
)
|
||||
|
||||
with self.assertRaisesRegex(ValueError, "overlap"):
|
||||
corpus.assemble(train, evaluation)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,170 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
SCRIPT_DIRECTORY = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
def load_module(name: str, file_name: str):
|
||||
spec = importlib.util.spec_from_file_location(name, SCRIPT_DIRECTORY / file_name)
|
||||
if spec is None or spec.loader is None:
|
||||
raise RuntimeError(f"Unable to load {file_name}")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
generator = load_module(
|
||||
"generate_blessing_training_corpus",
|
||||
"generate_blessing_training_corpus.py",
|
||||
)
|
||||
extractor = load_module(
|
||||
"extract_lccc_blessing_candidates",
|
||||
"extract_lccc_blessing_candidates.py",
|
||||
)
|
||||
benchmark_preparer = load_module(
|
||||
"prepare_blessing_benchmark",
|
||||
"prepare_blessing_benchmark.py",
|
||||
)
|
||||
benchmark_finalizer = load_module(
|
||||
"finalize_blessing_benchmark",
|
||||
"finalize_blessing_benchmark.py",
|
||||
)
|
||||
|
||||
|
||||
class BlessingCorpusGeneratorTests(unittest.TestCase):
|
||||
def test_targets_are_balanced_per_language(self) -> None:
|
||||
targets = generator.allocate_targets(generator.ZH_FAMILIES, 10_000)
|
||||
positive = sum(
|
||||
targets[family.name]
|
||||
for family in generator.ZH_FAMILIES
|
||||
if family.blessing
|
||||
)
|
||||
negative = sum(
|
||||
targets[family.name]
|
||||
for family in generator.ZH_FAMILIES
|
||||
if not family.blessing
|
||||
)
|
||||
|
||||
self.assertEqual(positive, 5_000)
|
||||
self.assertEqual(negative, 5_000)
|
||||
|
||||
def test_generation_is_unique_and_partial_label_only(self) -> None:
|
||||
records = generator.generate_language(
|
||||
language="zh-Hans",
|
||||
common_slots=generator.ZH_COMMON,
|
||||
families=generator.ZH_FAMILIES,
|
||||
target=1_000,
|
||||
seed=generator.SEED,
|
||||
reserved=set(),
|
||||
)
|
||||
|
||||
self.assertEqual(len(records), 1_000)
|
||||
self.assertEqual(
|
||||
len({generator.fingerprint(record["text"]) for record in records}),
|
||||
1_000,
|
||||
)
|
||||
self.assertTrue(
|
||||
all(record["knownLabels"] == ["blessing"] for record in records)
|
||||
)
|
||||
self.assertEqual(sum(record["blessing"] for record in records), 500)
|
||||
|
||||
|
||||
class LCCCBlessingCandidateTests(unittest.TestCase):
|
||||
def test_direct_wishes_are_positive(self) -> None:
|
||||
self.assertEqual(
|
||||
extractor.classify("祝你生日快乐，愿新的一岁平安顺利"),
|
||||
("positive", "direct_wish"),
|
||||
)
|
||||
self.assertEqual(
|
||||
extractor.classify("恭喜你顺利毕业"),
|
||||
("positive", "congratulation"),
|
||||
)
|
||||
|
||||
def test_boundaries_are_negative(self) -> None:
|
||||
self.assertEqual(
|
||||
extractor.classify("帮我写一段生日祝福语"),
|
||||
("negative", "meta_request"),
|
||||
)
|
||||
self.assertEqual(
|
||||
extractor.classify("谢谢大家发来的生日祝福"),
|
||||
("negative", "received_thanks"),
|
||||
)
|
||||
self.assertEqual(
|
||||
extractor.classify("我们晚上一起庆祝项目上线"),
|
||||
("negative", "celebration_mention"),
|
||||
)
|
||||
self.assertEqual(
|
||||
extractor.classify("晚上好"),
|
||||
("negative", "plain_greeting"),
|
||||
)
|
||||
|
||||
def test_question_is_not_promoted_to_direct_wish(self) -> None:
|
||||
self.assertEqual(
|
||||
extractor.classify("可以对我说一句生日快乐吗？"),
|
||||
("negative", "meta_request"),
|
||||
)
|
||||
|
||||
|
||||
class BlessingBenchmarkTests(unittest.TestCase):
|
||||
def test_selection_prioritizes_boundary_before_explicit_marker(self) -> None:
|
||||
record = {
|
||||
"text": "文档引用了“祝你生日快乐”作为写作示例。",
|
||||
"blessing": False,
|
||||
"sentiment": "neutral",
|
||||
}
|
||||
|
||||
self.assertEqual(
|
||||
benchmark_preparer.selection_stratum(record),
|
||||
"boundary_candidate",
|
||||
)
|
||||
|
||||
def test_selection_includes_implicit_positive_language(self) -> None:
|
||||
record = {
|
||||
"text": "I hope you continue to know peace and happiness.",
|
||||
"blessing": False,
|
||||
"sentiment": "positive",
|
||||
}
|
||||
|
||||
self.assertEqual(
|
||||
benchmark_preparer.selection_stratum(record),
|
||||
"explicit_candidate",
|
||||
)
|
||||
|
||||
def test_benchmark_split_is_deterministic(self) -> None:
|
||||
first = benchmark_finalizer.benchmark_split("blessing-review-00042")
|
||||
second = benchmark_finalizer.benchmark_split("blessing-review-00042")
|
||||
|
||||
self.assertEqual(first, second)
|
||||
self.assertIn(first, {"calibration", "test"})
|
||||
|
||||
def test_binary_kappa_reports_partial_agreement(self) -> None:
|
||||
annotation_a = {
|
||||
"1": {"label": True},
|
||||
"2": {"label": True},
|
||||
"3": {"label": False},
|
||||
"4": {"label": False},
|
||||
}
|
||||
annotation_b = {
|
||||
"1": {"label": True},
|
||||
"2": {"label": False},
|
||||
"3": {"label": False},
|
||||
"4": {"label": False},
|
||||
}
|
||||
|
||||
agreement, kappa = benchmark_finalizer.binary_cohen_kappa(
|
||||
annotation_a,
|
||||
annotation_b,
|
||||
)
|
||||
|
||||
self.assertEqual(agreement, 0.75)
|
||||
self.assertEqual(kappa, 0.5)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,216 @@
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
import build_corpus_registry as registry
|
||||
|
||||
|
||||
class CorpusRegistryTests(unittest.TestCase):
|
||||
def test_evaluation_overlap_blocks_training(self):
|
||||
report, records, train, _, _ = self._build(
|
||||
train_records=[
|
||||
self._record("train-1", "Send report 123", "train", task=True)
|
||||
],
|
||||
evaluation_records=[
|
||||
self._record("eval-1", "Send report 123", "test", task=True)
|
||||
],
|
||||
)
|
||||
|
||||
self.assertEqual(1, report["canonicalRecordCount"])
|
||||
self.assertEqual("evaluation-only", records[0]["allowedUse"])
|
||||
self.assertEqual([], train)
|
||||
self.assertEqual(1, report["exactDuplicateCount"])
|
||||
|
||||
def test_unknown_labels_remain_unknown(self):
|
||||
report, records, train, pilot, _ = self._build(
|
||||
train_records=[
|
||||
{
|
||||
**self._record(
|
||||
"open-1",
|
||||
"Could you send the report?",
|
||||
"train",
|
||||
task=True,
|
||||
),
|
||||
"knownLabels": ["task"],
|
||||
}
|
||||
],
|
||||
all_intents_known=False,
|
||||
)
|
||||
|
||||
self.assertEqual(1, report["trainCandidateCount"])
|
||||
self.assertEqual("true", records[0]["labels"]["task"])
|
||||
self.assertEqual("unknown", records[0]["labels"]["question"])
|
||||
self.assertEqual("unknown", records[0]["labels"]["assistantCommand"])
|
||||
self.assertEqual(1, len(train))
|
||||
self.assertEqual(1, len(pilot))
|
||||
|
||||
def test_domain_only_record_and_weight_reach_training_output(self):
|
||||
report, records, train, _, _ = self._build(
|
||||
train_records=[
|
||||
{
|
||||
**self._record("domain-1", "Table for five", "train", task=False),
|
||||
"domain": "dining",
|
||||
"knownLabels": ["domain"],
|
||||
"sampleWeight": 0.5,
|
||||
}
|
||||
],
|
||||
all_intents_known=False,
|
||||
source_weight=0.7,
|
||||
)
|
||||
|
||||
self.assertEqual(1, report["trainCandidateCount"])
|
||||
self.assertEqual("dining", records[0]["domain"])
|
||||
self.assertEqual(["domain"], train[0]["knownLabels"])
|
||||
self.assertEqual(0.35, train[0]["sampleWeight"])
|
||||
|
||||
def test_conflicting_source_labels_enter_human_review(self):
|
||||
_, records, train, _, review = self._build(
|
||||
train_records=[
|
||||
self._record("one", "Please send it.", "train", task=True),
|
||||
self._record("two", "Please send it.", "train", task=False),
|
||||
],
|
||||
)
|
||||
|
||||
self.assertEqual(["task"], records[0]["labelConflicts"])
|
||||
self.assertEqual([], train)
|
||||
self.assertEqual(["task"], review[0]["conflicts"])
|
||||
|
||||
def test_rejects_unsafe_training_license(self):
|
||||
report, records, train, pilot, _ = self._build(
|
||||
train_records=[
|
||||
self._record("unsafe", "Research dialogue", "train", task=True)
|
||||
],
|
||||
train_license="research-only",
|
||||
)
|
||||
|
||||
self.assertEqual(0, report["canonicalRecordCount"])
|
||||
self.assertEqual({"unsafe-license": 1}, report["excludedRecordCounts"])
|
||||
self.assertEqual([], records)
|
||||
self.assertEqual([], train)
|
||||
self.assertEqual([], pilot)
|
||||
|
||||
def test_pilot_uses_unique_clusters(self):
|
||||
records = [
|
||||
self._record(
|
||||
f"record-{index}",
|
||||
f"Could you send report {index} before Friday?",
|
||||
"train",
|
||||
task=True,
|
||||
)
|
||||
for index in range(4)
|
||||
]
|
||||
_, _, _, pilot, _ = self._build(
|
||||
train_records=records,
|
||||
pilot_count=10,
|
||||
)
|
||||
|
||||
self.assertEqual(1, len(pilot))
|
||||
|
||||
def _build(
|
||||
self,
|
||||
train_records,
|
||||
evaluation_records=None,
|
||||
all_intents_known=True,
|
||||
train_license="MIT",
|
||||
pilot_count=10,
|
||||
source_weight=1.0,
|
||||
):
|
||||
evaluation_records = evaluation_records or []
|
||||
with tempfile.TemporaryDirectory() as raw_directory:
|
||||
directory = Path(raw_directory)
|
||||
train_path = directory / "train.jsonl"
|
||||
evaluation_path = directory / "evaluation.jsonl"
|
||||
self._write_json_lines(train_path, train_records)
|
||||
self._write_json_lines(evaluation_path, evaluation_records)
|
||||
manifest_path = directory / "sources.json"
|
||||
manifest_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"sources": [
|
||||
{
|
||||
"id": "train",
|
||||
"path": str(train_path),
|
||||
"license": train_license,
|
||||
"defaultUse": "train",
|
||||
"sourceType": "fixture",
|
||||
"allIntentLabelsKnown": all_intents_known,
|
||||
"sentimentKnown": False,
|
||||
"weight": source_weight,
|
||||
"required": True,
|
||||
},
|
||||
{
|
||||
"id": "evaluation",
|
||||
"path": str(evaluation_path),
|
||||
"license": "evaluation-only",
|
||||
"defaultUse": "evaluation-only",
|
||||
"sourceType": "fixture",
|
||||
"allIntentLabelsKnown": True,
|
||||
"sentimentKnown": False,
|
||||
"required": True,
|
||||
},
|
||||
],
|
||||
"excludedSources": [],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
paths = {
|
||||
name: directory / f"{name}.jsonl"
|
||||
for name in (
|
||||
"registry",
|
||||
"train_candidates",
|
||||
"pilot",
|
||||
"human_review",
|
||||
)
|
||||
}
|
||||
report_path = directory / "report.json"
|
||||
report = registry.build(
|
||||
SimpleNamespace(
|
||||
repository_root=directory,
|
||||
source_manifest=manifest_path,
|
||||
report=report_path,
|
||||
pilot_count=pilot_count,
|
||||
**paths,
|
||||
)
|
||||
)
|
||||
return (
|
||||
report,
|
||||
self._read_json_lines(paths["registry"]),
|
||||
self._read_json_lines(paths["train_candidates"]),
|
||||
self._read_json_lines(paths["pilot"]),
|
||||
self._read_json_lines(paths["human_review"]),
|
||||
)
|
||||
|
||||
def _record(self, identifier, text, split, task):
|
||||
return {
|
||||
"id": identifier,
|
||||
"text": text,
|
||||
"language": "en",
|
||||
"family": "fixture",
|
||||
"split": split,
|
||||
"task": task,
|
||||
}
|
||||
|
||||
def _write_json_lines(self, path, records):
|
||||
path.write_text(
|
||||
"\n".join(json.dumps(record) for record in records)
|
||||
+ ("\n" if records else ""),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
def _read_json_lines(self, path):
|
||||
return [
|
||||
json.loads(line)
|
||||
for line in path.read_text(encoding="utf-8").splitlines()
|
||||
if line
|
||||
]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,144 @@
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
import evaluate_product_policy_anchors as policy
|
||||
|
||||
|
||||
class ProductPolicyAnchorTests(unittest.TestCase):
|
||||
def test_prepare_and_evaluate_anchor_accuracy(self):
|
||||
with tempfile.TemporaryDirectory() as raw_directory:
|
||||
directory = Path(raw_directory)
|
||||
anchors = directory / "anchors.json"
|
||||
anchors.write_text(
|
||||
json.dumps(
|
||||
[
|
||||
{
|
||||
"id": "one",
|
||||
"text": "open my inbox",
|
||||
"language": "en",
|
||||
"expected": {
|
||||
"recordDisposition": "exclude-device-command"
|
||||
},
|
||||
},
|
||||
{
|
||||
"id": "two",
|
||||
"text": "Could you send the report?",
|
||||
"language": "en",
|
||||
"expected": {
|
||||
"recordDisposition": "keep",
|
||||
"replyableMessage": "true",
|
||||
"task": "true",
|
||||
"question": "true",
|
||||
"ambiguous": "false",
|
||||
},
|
||||
},
|
||||
]
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
queue = directory / "queue.jsonl"
|
||||
policy.prepare(SimpleNamespace(anchors=anchors, queue=queue))
|
||||
labeler = directory / "labeler.jsonl"
|
||||
records = [
|
||||
self._record(
|
||||
"one",
|
||||
"open my inbox",
|
||||
disposition="exclude-device-command",
|
||||
replyableMessage="false",
|
||||
task="true",
|
||||
question="false",
|
||||
ambiguous="false",
|
||||
),
|
||||
self._record(
|
||||
"two",
|
||||
"Could you send the report?",
|
||||
disposition="keep",
|
||||
replyableMessage="true",
|
||||
task="true",
|
||||
question="true",
|
||||
ambiguous="false",
|
||||
),
|
||||
]
|
||||
labeler.write_text(
|
||||
"\n".join(json.dumps(record) for record in records) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
report = policy.evaluate(
|
||||
SimpleNamespace(
|
||||
anchors=anchors,
|
||||
queue=queue,
|
||||
labeler=[("test", labeler)],
|
||||
minimum_accuracy=0.95,
|
||||
report=directory / "report.json",
|
||||
)
|
||||
)
|
||||
|
||||
self.assertTrue(report["eligibleForCorpusReadjudication"])
|
||||
self.assertEqual(
|
||||
1.0,
|
||||
report["labelers"]["test"]["decisionAccuracy"],
|
||||
)
|
||||
|
||||
def test_target_gate_ignores_non_target_mismatch(self):
|
||||
anchors = [
|
||||
{
|
||||
"id": "one",
|
||||
"text": "take your time",
|
||||
"language": "en",
|
||||
"expected": {
|
||||
"recordDisposition": "keep",
|
||||
"replyableMessage": "true",
|
||||
"task": "false",
|
||||
"question": "false",
|
||||
"ambiguous": "false",
|
||||
},
|
||||
}
|
||||
]
|
||||
adjudication = self._record(
|
||||
"one",
|
||||
"take your time",
|
||||
disposition="keep",
|
||||
replyableMessage="true",
|
||||
task="false",
|
||||
question="false",
|
||||
ambiguous="true",
|
||||
)
|
||||
|
||||
result = policy.evaluate_labeler(
|
||||
anchors,
|
||||
{"one": adjudication},
|
||||
{"replyableMessage", "task", "question"},
|
||||
)
|
||||
|
||||
self.assertEqual(1.0, result["gateDecisionAccuracy"])
|
||||
self.assertLess(result["decisionAccuracy"], 1.0)
|
||||
|
||||
def _record(self, identifier, text, disposition, **values):
|
||||
resolutions = {
|
||||
field: (
|
||||
"unknown"
|
||||
if field == "domain"
|
||||
else "false"
|
||||
)
|
||||
for field in policy.ANCHOR_FIELDS
|
||||
}
|
||||
resolutions.update(values)
|
||||
return {
|
||||
"id": identifier,
|
||||
"recordDisposition": disposition,
|
||||
"dispositionConfidence": 0.99,
|
||||
"dispositionEvidence": text,
|
||||
"resolutions": resolutions,
|
||||
"confidence": {field: 0.99 for field in resolutions},
|
||||
"evidence": {field: text for field in resolutions},
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,181 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
import finalize_v6_blind_holdout as finalizer
|
||||
|
||||
|
||||
class FinalizeV6BlindHoldoutTests(unittest.TestCase):
|
||||
def test_review_accepts_each_high_confidence_field_independently(self):
|
||||
primary = self._labelers(
|
||||
"primary",
|
||||
[
|
||||
self._states(task="true", question="true", domain="finance"),
|
||||
self._states(task="true", question="true", domain="travel"),
|
||||
self._states(task="true", question="false", domain="calendar"),
|
||||
],
|
||||
)
|
||||
reviewers = self._labelers(
|
||||
"reviewer",
|
||||
[
|
||||
self._states(task="true", question="true", domain="finance"),
|
||||
self._states(task="false", question="false", domain="travel"),
|
||||
],
|
||||
)
|
||||
|
||||
states = finalizer.resolve_model_states(
|
||||
"record-1",
|
||||
primary,
|
||||
reviewers,
|
||||
in_review_queue=True,
|
||||
)
|
||||
output = finalizer.output_record(
|
||||
{"id": "record-1", "text": "Text", "language": "en"},
|
||||
states,
|
||||
"test",
|
||||
)
|
||||
|
||||
self.assertEqual("true", states["task"])
|
||||
self.assertEqual("unknown", states["question"])
|
||||
self.assertEqual("unknown", states["domain"])
|
||||
self.assertTrue(output["task"])
|
||||
self.assertIn("task", output["knownLabels"])
|
||||
self.assertNotIn("question", output["knownLabels"])
|
||||
self.assertNotIn("domain", output["knownLabels"])
|
||||
self.assertIsNone(output["domain"])
|
||||
|
||||
def test_human_fields_override_models_but_exclusion_infers_nothing(self):
|
||||
model_states = self._states(
|
||||
task="false",
|
||||
question="false",
|
||||
replyableMessage="false",
|
||||
)
|
||||
overrides = finalizer.apply_human_overrides(
|
||||
model_states,
|
||||
{
|
||||
"recordDisposition": "keep",
|
||||
"task": "true",
|
||||
"question": "unknown",
|
||||
"replyableMessage": "true",
|
||||
"ambiguous": "false",
|
||||
},
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
["task", "question", "replyableMessage", "ambiguous"],
|
||||
overrides,
|
||||
)
|
||||
self.assertEqual("true", model_states["task"])
|
||||
self.assertEqual("unknown", model_states["question"])
|
||||
self.assertEqual("true", model_states["replyableMessage"])
|
||||
|
||||
excluded_states = self._states(task="true")
|
||||
excluded_overrides = finalizer.apply_human_overrides(
|
||||
excluded_states,
|
||||
{"recordDisposition": "exclude-device-command"},
|
||||
)
|
||||
self.assertEqual([], excluded_overrides)
|
||||
self.assertEqual("true", excluded_states["task"])
|
||||
|
||||
def test_split_assigns_twenty_of_each_kind_per_language(self):
|
||||
records = [
|
||||
{
|
||||
"id": f"{language}-{index:03d}",
|
||||
"text": f"{language} {index}",
|
||||
"language": language,
|
||||
}
|
||||
for language in ("en", "zh-Hans")
|
||||
for index in range(60)
|
||||
]
|
||||
|
||||
assignments = finalizer.assign_splits(records, records_per_split=20)
|
||||
|
||||
for language in ("en", "zh-Hans"):
|
||||
counts = {
|
||||
split: sum(
|
||||
assignments[f"{language}-{index:03d}"] == split
|
||||
for index in range(60)
|
||||
)
|
||||
for split in finalizer.SPLITS
|
||||
}
|
||||
self.assertEqual(
|
||||
{"validation": 20, "test": 20, "golden": 20},
|
||||
counts,
|
||||
)
|
||||
self.assertEqual("validation", assignments["en-000"])
|
||||
self.assertEqual("test", assignments["en-020"])
|
||||
self.assertEqual("golden", assignments["en-040"])
|
||||
|
||||
def test_unknown_fields_are_not_known_or_positive(self):
|
||||
states = self._states(
|
||||
assistantCommand="unknown",
|
||||
sentiment="unknown",
|
||||
domain="unknown",
|
||||
)
|
||||
states["ambiguous"] = "unknown"
|
||||
|
||||
output = finalizer.output_record(
|
||||
{"id": "record-1", "text": "unchanged", "language": "en"},
|
||||
states,
|
||||
"golden",
|
||||
)
|
||||
|
||||
self.assertFalse(output["assistantCommand"])
|
||||
self.assertEqual("neutral", output["sentiment"])
|
||||
self.assertIsNone(output["domain"])
|
||||
self.assertIsNone(output["ambiguous"])
|
||||
self.assertNotIn("assistantCommand", output["knownLabels"])
|
||||
self.assertNotIn("sentiment", output["knownLabels"])
|
||||
self.assertNotIn("domain", output["knownLabels"])
|
||||
self.assertNotEqual("train", output["split"])
|
||||
self.assertEqual("unchanged", output["text"])
|
||||
|
||||
def test_unreviewed_field_requires_unanimous_non_unknown_vote(self):
|
||||
primary = self._labelers(
|
||||
"primary",
|
||||
[
|
||||
self._states(blessing="unknown"),
|
||||
self._states(blessing="unknown"),
|
||||
self._states(blessing="unknown"),
|
||||
],
|
||||
)
|
||||
|
||||
states = finalizer.resolve_model_states(
|
||||
"record-1",
|
||||
primary,
|
||||
[],
|
||||
in_review_queue=False,
|
||||
)
|
||||
|
||||
self.assertEqual("unknown", states["blessing"])
|
||||
self.assertEqual("false", states["task"])
|
||||
|
||||
def test_duplicate_ids_are_rejected(self):
|
||||
with self.assertRaisesRegex(ValueError, "duplicate id"):
|
||||
finalizer.unique_records(
|
||||
[{"id": "same"}, {"id": "same"}],
|
||||
"test input",
|
||||
)
|
||||
|
||||
def _states(self, **overrides):
|
||||
states = {field: "false" for field in finalizer.INTENT_LABELS}
|
||||
states["sentiment"] = "neutral"
|
||||
states["domain"] = "unknown"
|
||||
states["ambiguous"] = "false"
|
||||
states.update(overrides)
|
||||
return states
|
||||
|
||||
def _labelers(self, prefix, states):
|
||||
return [
|
||||
(f"{prefix}-{index}", {"record-1": state})
|
||||
for index, state in enumerate(states)
|
||||
]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,175 @@
|
||||
import json
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
import generate_open_training_corpus as corpus
|
||||
|
||||
|
||||
class OpenTrainingCorpusTests(unittest.TestCase):
|
||||
def test_schema_contains_new_intents_and_single_domain(self):
|
||||
record = corpus.make_record(
|
||||
record_id="one",
|
||||
text="Play some jazz",
|
||||
language="en",
|
||||
family="fixture",
|
||||
source_dataset="fixture",
|
||||
source_license="CC0-1.0",
|
||||
source_url="https://example.test/train",
|
||||
source_revision="deadbeef",
|
||||
source_split="train",
|
||||
known_labels=corpus.known_labels_for_mapping(
|
||||
"assistantCommand",
|
||||
"media",
|
||||
),
|
||||
labeling_method="fixture",
|
||||
assistant_command=True,
|
||||
domain="media",
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
{"assistantCommand", "informationQuery", "systemNotification"}
|
||||
<= set(corpus.INTENT_LABELS),
|
||||
True,
|
||||
)
|
||||
self.assertEqual("media", record["domain"])
|
||||
self.assertEqual(
|
||||
[
|
||||
"assistantCommand",
|
||||
"domain",
|
||||
"informationQuery",
|
||||
"systemNotification",
|
||||
],
|
||||
record["knownLabels"],
|
||||
)
|
||||
self.assertFalse(record["task"])
|
||||
self.assertFalse(record["informationQuery"])
|
||||
|
||||
def test_massive_mapping_is_conservative(self):
|
||||
self.assertEqual(
|
||||
("assistantCommand", "calendar"),
|
||||
corpus.massive_mapping("alarm_set"),
|
||||
)
|
||||
self.assertEqual(
|
||||
("informationQuery", "generalKnowledge"),
|
||||
corpus.massive_mapping("qa_factoid"),
|
||||
)
|
||||
self.assertEqual(
|
||||
("task", "travel"),
|
||||
corpus.massive_mapping("transport_taxi"),
|
||||
)
|
||||
self.assertEqual((None, None), corpus.massive_mapping("general_greet"))
|
||||
|
||||
def test_bitod_mapping_supports_official_chinese_intents(self):
|
||||
self.assertEqual(
|
||||
("informationQuery", "dining"),
|
||||
corpus.bitod_mapping("餐馆查询"),
|
||||
)
|
||||
self.assertEqual(("task", "travel"), corpus.bitod_mapping("宾馆预订"))
|
||||
self.assertEqual(
|
||||
("informationQuery", "travel"),
|
||||
corpus.bitod_mapping("香港地铁"),
|
||||
)
|
||||
self.assertEqual(
|
||||
("informationQuery", "weather"),
|
||||
corpus.bitod_mapping("天气查询"),
|
||||
)
|
||||
|
||||
def test_new_builders_are_pinned_optional_and_isolated(self):
|
||||
builders = corpus.configured_source_builders()
|
||||
names = {builder.name for builder in builders}
|
||||
|
||||
self.assertTrue(
|
||||
{
|
||||
"SNIPS",
|
||||
"MInDS-14 zh-CN",
|
||||
"BiToD",
|
||||
"RESTAURANTS-8K",
|
||||
"FormosaNLU Synth v1",
|
||||
}
|
||||
<= names
|
||||
)
|
||||
self.assertNotIn("CFPB", names)
|
||||
self.assertNotIn("CLINC150", names)
|
||||
self.assertNotIn("openclaw-zh-greetings", names)
|
||||
self.assertNotIn("WeChat-AutoSendBless", names)
|
||||
self.assertTrue(all(builder.optional for builder in builders))
|
||||
for revision in (
|
||||
corpus.SNIPS_REVISION,
|
||||
corpus.MINDS14_REVISION,
|
||||
corpus.BITOD_REVISION,
|
||||
corpus.RESTAURANT8K_REVISION,
|
||||
corpus.FORMOSA_NLU_REVISION,
|
||||
):
|
||||
self.assertRegex(revision, r"^[0-9a-f]{40}$")
|
||||
|
||||
def test_optional_builder_failure_is_reported_unavailable(self):
|
||||
def unavailable(_seed):
|
||||
raise corpus.SourceUnavailable("offline")
|
||||
|
||||
sources, failures = corpus.build_available_sources(
|
||||
(corpus.SourceBuilder("fixture", unavailable),),
|
||||
seed=7,
|
||||
allow_unavailable_sources=False,
|
||||
)
|
||||
|
||||
self.assertEqual([], sources)
|
||||
self.assertEqual("fixture", failures[0]["dataset"])
|
||||
self.assertIn("offline", failures[0]["reason"])
|
||||
|
||||
def test_snips_uses_only_official_train_intents(self):
|
||||
def payload(url):
|
||||
intent = next(
|
||||
value for value in corpus.SNIPS_MAPPING if f"/{value}/" in url
|
||||
)
|
||||
return json.dumps(
|
||||
{
|
||||
intent: [
|
||||
{
|
||||
"data": [
|
||||
{"text": "example "},
|
||||
{"text": intent},
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
).encode()
|
||||
|
||||
with mock.patch.object(corpus, "fetch_bytes", side_effect=payload):
|
||||
records = corpus.snips_records(seed=3)
|
||||
|
||||
self.assertEqual(6, len(records))
|
||||
self.assertTrue(all(record["sourceSplit"] == "train" for record in records))
|
||||
self.assertTrue(
|
||||
all(record["sourceRevision"] == corpus.SNIPS_REVISION for record in records)
|
||||
)
|
||||
play = next(record for record in records if "PlayMusic" in record["text"])
|
||||
self.assertTrue(play["assistantCommand"])
|
||||
self.assertEqual("media", play["domain"])
|
||||
|
||||
def test_formosa_synthetic_records_are_simplified_and_keep_low_weight(self):
|
||||
payload = (
|
||||
json.dumps(
|
||||
{
|
||||
"id": "syn-1",
|
||||
"utt": "播放爵士樂",
|
||||
"intent": "play_music",
|
||||
}
|
||||
)
|
||||
+ "\n"
|
||||
).encode()
|
||||
with mock.patch.object(corpus, "fetch_bytes", return_value=payload):
|
||||
records = corpus.formosa_nlu_records(seed=3)
|
||||
|
||||
self.assertEqual(1, len(records))
|
||||
self.assertEqual("播放爵士乐", records[0]["text"])
|
||||
self.assertEqual("zh-Hans", records[0]["language"])
|
||||
self.assertEqual(0.35, records[0]["sampleWeight"])
|
||||
self.assertEqual("train", records[0]["sourceSplit"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,199 @@
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import unicodedata
|
||||
import unittest
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
import generate_v6_boundary_corpus as corpus
|
||||
|
||||
|
||||
REQUIRED_FIELDS = {
|
||||
"id",
|
||||
"text",
|
||||
"language",
|
||||
"split",
|
||||
"family",
|
||||
"task",
|
||||
"question",
|
||||
"invitation",
|
||||
"complaint",
|
||||
"scheduleNegotiation",
|
||||
"confirmationDecision",
|
||||
"followUpReminder",
|
||||
"blessing",
|
||||
"sentiment",
|
||||
"replyable",
|
||||
"assistantCommand",
|
||||
"informationQuery",
|
||||
"systemNotification",
|
||||
"domain",
|
||||
"sampleWeight",
|
||||
"knownLabels",
|
||||
"labelingMethod",
|
||||
"sourceDataset",
|
||||
"sourceLicense",
|
||||
"sourceURL",
|
||||
"sourceRevision",
|
||||
"sourceSplit",
|
||||
"synthetic",
|
||||
"templateFamily",
|
||||
}
|
||||
|
||||
|
||||
class V6BoundaryCorpusTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.records, cls.target_counts, cls.excluded = corpus.generate_records()
|
||||
cls.payload = corpus.serialize_records(cls.records)
|
||||
|
||||
def test_default_generation_is_reproducible(self):
|
||||
records, target_counts, excluded = corpus.generate_records()
|
||||
payload = corpus.serialize_records(records)
|
||||
|
||||
self.assertEqual(self.payload, payload)
|
||||
self.assertEqual(self.target_counts, target_counts)
|
||||
self.assertEqual(self.excluded, excluded)
|
||||
self.assertEqual(
|
||||
hashlib.sha256(self.payload).hexdigest(),
|
||||
hashlib.sha256(payload).hexdigest(),
|
||||
)
|
||||
|
||||
def test_default_scale_language_intent_and_domain_balance(self):
|
||||
self.assertGreaterEqual(len(self.records), 6_000)
|
||||
self.assertEqual(
|
||||
{"en": 3_600, "zh-Hans": 3_600},
|
||||
Counter(record["language"] for record in self.records),
|
||||
)
|
||||
for language in corpus.LANGUAGES:
|
||||
for intent in corpus.NEW_INTENTS:
|
||||
self.assertEqual(
|
||||
1_000,
|
||||
self.target_counts[f"{language}|{intent}"],
|
||||
)
|
||||
for intent in corpus.HARD_NEGATIVE_INTENTS:
|
||||
self.assertEqual(
|
||||
200,
|
||||
self.target_counts[f"{language}|{intent}"],
|
||||
)
|
||||
self.assertEqual(
|
||||
{domain: 600 for domain in corpus.DOMAINS},
|
||||
Counter(record["domain"] for record in self.records),
|
||||
)
|
||||
|
||||
def test_records_have_full_schema_low_weight_and_known_boundaries(self):
|
||||
for record in self.records:
|
||||
self.assertEqual(REQUIRED_FIELDS, set(record))
|
||||
self.assertEqual(0.35, record["sampleWeight"])
|
||||
self.assertEqual("train", record["split"])
|
||||
self.assertEqual("train", record["sourceSplit"])
|
||||
self.assertEqual(corpus.SOURCE_DATASET, record["sourceDataset"])
|
||||
self.assertEqual(corpus.SOURCE_LICENSE, record["sourceLicense"])
|
||||
self.assertEqual(corpus.SOURCE_REVISION, record["sourceRevision"])
|
||||
self.assertTrue(record["synthetic"])
|
||||
self.assertEqual(record["family"], record["templateFamily"])
|
||||
self.assertEqual(corpus.KNOWN_LABELS, record["knownLabels"])
|
||||
self.assertEqual(record["text"], unicodedata.normalize("NFKC", record["text"]))
|
||||
|
||||
routing_count = sum(record[intent] for intent in corpus.NEW_INTENTS)
|
||||
if any(record[intent] for intent in corpus.NEW_INTENTS):
|
||||
self.assertEqual(1, routing_count)
|
||||
self.assertFalse(record["task"])
|
||||
self.assertFalse(record["question"])
|
||||
self.assertFalse(record["replyable"])
|
||||
else:
|
||||
self.assertEqual(0, routing_count)
|
||||
self.assertTrue(
|
||||
record["task"] or record["question"] or record["replyable"]
|
||||
)
|
||||
|
||||
def test_generated_ids_and_normalized_texts_are_unique(self):
|
||||
ids = [record["id"] for record in self.records]
|
||||
texts = [corpus.fingerprint(record["text"]) for record in self.records]
|
||||
|
||||
self.assertEqual(len(ids), len(set(ids)))
|
||||
self.assertEqual(len(texts), len(set(texts)))
|
||||
|
||||
def test_holdout_discovery_and_nfkc_overlap_exclusion(self):
|
||||
system_record = next(
|
||||
record
|
||||
for record in self.records
|
||||
if record["language"] == "zh-Hans"
|
||||
and record["systemNotification"]
|
||||
and ":" in record["text"]
|
||||
)
|
||||
blind_record = next(
|
||||
record
|
||||
for record in self.records
|
||||
if record["language"] == "en" and record["informationQuery"]
|
||||
)
|
||||
with tempfile.TemporaryDirectory() as temporary_directory:
|
||||
directory = Path(temporary_directory)
|
||||
wildcard_holdout = directory / "frozen-holdout-corpus.jsonl"
|
||||
blind_holdout = directory / "product-policy-blind-holdout-v1.jsonl"
|
||||
wildcard_holdout.write_text(
|
||||
json.dumps(
|
||||
{"text": system_record["text"].replace(":", "：")},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
blind_holdout.write_text(
|
||||
json.dumps({"text": blind_record["text"]}) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
paths = corpus.discover_holdout_paths(directory)
|
||||
holdouts = corpus.load_holdout_fingerprints(paths)
|
||||
records, _, excluded = corpus.generate_records(
|
||||
holdout_fingerprints=holdouts
|
||||
)
|
||||
|
||||
generated = {corpus.fingerprint(record["text"]) for record in records}
|
||||
self.assertEqual({wildcard_holdout, blind_holdout}, set(paths))
|
||||
self.assertNotIn(corpus.fingerprint(system_record["text"]), generated)
|
||||
self.assertNotIn(corpus.fingerprint(blind_record["text"]), generated)
|
||||
self.assertGreaterEqual(excluded, 2)
|
||||
self.assertEqual(len(self.records), len(records))
|
||||
|
||||
def test_summary_counts_and_hash_match_written_jsonl(self):
|
||||
with tempfile.TemporaryDirectory() as temporary_directory:
|
||||
directory = Path(temporary_directory)
|
||||
output = directory / "corpus.jsonl"
|
||||
summary_path = directory / "summary.json"
|
||||
|
||||
summary = corpus.write_corpus(
|
||||
output,
|
||||
summary_path,
|
||||
positive_per_intent_language=12,
|
||||
negative_per_intent_language=12,
|
||||
)
|
||||
persisted = json.loads(summary_path.read_text(encoding="utf-8"))
|
||||
output_sha256 = hashlib.sha256(output.read_bytes()).hexdigest()
|
||||
|
||||
self.assertEqual(summary, persisted)
|
||||
self.assertEqual(output_sha256, summary["corpusSHA256"])
|
||||
self.assertEqual(144, summary["recordCount"])
|
||||
self.assertEqual(
|
||||
{"en": 72, "zh-Hans": 72},
|
||||
summary["counts"]["byLanguage"],
|
||||
)
|
||||
self.assertEqual(48, summary["counts"]["byIntent"]["replyableMessage"])
|
||||
self.assertEqual(
|
||||
12,
|
||||
summary["counts"]["byBoundaryTargetAndLanguage"]["en"][
|
||||
"replyableMessage"
|
||||
],
|
||||
)
|
||||
self.assertEqual(set(corpus.DOMAINS), set(summary["counts"]["byDomain"]))
|
||||
self.assertTrue(summary["counts"]["byTemplateFamily"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -91,6 +91,30 @@ class IterativeRetrainingTests(unittest.TestCase):
|
||||
|
||||
self.assertEqual([False, True], predicted["task"].tolist())
|
||||
|
||||
def test_registry_sample_weight_is_applied(self):
|
||||
configuration = research.configurations()[0]
|
||||
record = {
|
||||
"id": "weighted",
|
||||
"text": "Play music",
|
||||
"language": "en",
|
||||
"knownLabels": ["assistantCommand"],
|
||||
"assistantCommand": True,
|
||||
"sampleWeight": 0.35,
|
||||
}
|
||||
|
||||
self.assertIn("assistantCommand", research.INTENTS)
|
||||
self.assertAlmostEqual(
|
||||
configuration.external_weight * 0.35,
|
||||
research.sample_weight(record, configuration),
|
||||
)
|
||||
|
||||
def test_legacy_records_do_not_define_new_labels_as_false(self):
|
||||
record = {"id": "legacy", "text": "Play music", "language": "en"}
|
||||
|
||||
self.assertTrue(research.is_known(record, "task"))
|
||||
self.assertFalse(research.is_known(record, "assistantCommand"))
|
||||
self.assertFalse(research.is_known(record, "systemNotification"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -0,0 +1,286 @@
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
import merge_consensus_labels_v2 as consensus
|
||||
|
||||
|
||||
class ConsensusV2Tests(unittest.TestCase):
|
||||
def test_primary_unanimous_becomes_tier_a(self):
|
||||
report, tier_a, tier_b, human = self._merge(
|
||||
primary_states=[self._states(task="true")] * 3,
|
||||
review_states=[],
|
||||
)
|
||||
|
||||
self.assertEqual(1, report["tierACount"])
|
||||
self.assertEqual(1, len(tier_a))
|
||||
self.assertTrue(tier_a[0]["task"])
|
||||
self.assertEqual(1.0, tier_a[0]["sampleWeight"])
|
||||
self.assertEqual([], tier_b)
|
||||
self.assertEqual([], human)
|
||||
|
||||
def test_four_of_five_becomes_tier_b(self):
|
||||
report, tier_a, tier_b, human = self._merge(
|
||||
primary_states=[
|
||||
self._states(task="true"),
|
||||
self._states(task="true"),
|
||||
self._states(task="false"),
|
||||
],
|
||||
review_states=[
|
||||
self._states(task="true"),
|
||||
self._states(task="true"),
|
||||
],
|
||||
)
|
||||
|
||||
self.assertEqual(0, report["tierACount"])
|
||||
self.assertEqual(1, report["tierBCount"])
|
||||
self.assertEqual([], tier_a)
|
||||
self.assertTrue(tier_b[0]["task"])
|
||||
self.assertEqual(0.65, tier_b[0]["sampleWeight"])
|
||||
self.assertEqual([], human)
|
||||
|
||||
def test_three_two_vote_requires_human_review(self):
|
||||
report, _, tier_b, human = self._merge(
|
||||
primary_states=[
|
||||
self._states(task="true"),
|
||||
self._states(task="true"),
|
||||
self._states(task="false"),
|
||||
],
|
||||
review_states=[
|
||||
self._states(task="true"),
|
||||
self._states(task="false"),
|
||||
],
|
||||
)
|
||||
|
||||
self.assertEqual(1, report["humanReviewCount"])
|
||||
self.assertEqual([], tier_b)
|
||||
self.assertIn("task", human[0]["unresolvedFields"])
|
||||
|
||||
def test_unknown_primary_state_triggers_review(self):
|
||||
with tempfile.TemporaryDirectory() as raw_directory:
|
||||
directory = Path(raw_directory)
|
||||
queue = directory / "queue.jsonl"
|
||||
self._write(
|
||||
queue,
|
||||
[{"id": "record-1", "text": "Hello", "language": "en"}],
|
||||
)
|
||||
primary = []
|
||||
for index, state in enumerate(
|
||||
[
|
||||
self._states(question="unknown"),
|
||||
self._states(question="false"),
|
||||
self._states(question="false"),
|
||||
]
|
||||
):
|
||||
path = directory / f"primary-{index}.jsonl"
|
||||
self._write(path, [self._label("record-1", state)])
|
||||
primary.append((f"primary-{index}", path))
|
||||
output = directory / "review.jsonl"
|
||||
report_path = directory / "report.json"
|
||||
report = consensus.prepare_review(
|
||||
SimpleNamespace(
|
||||
queue=queue,
|
||||
primary=primary,
|
||||
output=output,
|
||||
report=report_path,
|
||||
)
|
||||
)
|
||||
|
||||
self.assertEqual(1, report["reviewCount"])
|
||||
self.assertEqual("record-1", self._read(output)[0]["id"])
|
||||
|
||||
def test_quoted_positive_is_sent_to_human(self):
|
||||
report, _, _, human = self._merge(
|
||||
primary_states=[
|
||||
self._states(blessing="true"),
|
||||
self._states(blessing="true"),
|
||||
self._states(blessing="false"),
|
||||
],
|
||||
review_states=[
|
||||
self._states(blessing="true"),
|
||||
self._states(blessing="true"),
|
||||
],
|
||||
review_quoted=[True, True],
|
||||
)
|
||||
|
||||
self.assertEqual(1, report["humanReviewCount"])
|
||||
self.assertIn("quotedOrMeta", human[0]["unresolvedFields"])
|
||||
|
||||
def test_unanimous_unknown_stays_out_of_known_labels(self):
|
||||
_, tier_a, _, _ = self._merge(
|
||||
primary_states=[self._states(assistantCommand="unknown")] * 3,
|
||||
review_states=[],
|
||||
)
|
||||
|
||||
self.assertEqual(1, len(tier_a))
|
||||
self.assertNotIn("assistantCommand", tier_a[0]["knownLabels"])
|
||||
self.assertFalse(tier_a[0]["assistantCommand"])
|
||||
self.assertIsNone(tier_a[0]["domain"])
|
||||
|
||||
def test_split_and_combine_preserve_queue_order(self):
|
||||
with tempfile.TemporaryDirectory() as raw_directory:
|
||||
directory = Path(raw_directory)
|
||||
queue_path = directory / "queue.jsonl"
|
||||
queue = [
|
||||
{"id": f"record-{index}", "text": str(index), "language": "en"}
|
||||
for index in range(5)
|
||||
]
|
||||
self._write(queue_path, queue)
|
||||
chunks = directory / "chunks"
|
||||
split_report = consensus.split_queue(
|
||||
SimpleNamespace(
|
||||
queue=queue_path,
|
||||
output_directory=chunks,
|
||||
chunk_size=2,
|
||||
report=directory / "split-report.json",
|
||||
)
|
||||
)
|
||||
self.assertEqual(3, split_report["chunkCount"])
|
||||
outputs = []
|
||||
for path in sorted(chunks.glob("*.jsonl")):
|
||||
output_path = directory / f"labeled-{path.name}"
|
||||
self._write(
|
||||
output_path,
|
||||
[
|
||||
self._label(record["id"], self._states())
|
||||
for record in self._read(path)
|
||||
],
|
||||
)
|
||||
outputs.append(output_path)
|
||||
combined_path = directory / "combined.jsonl"
|
||||
report = consensus.combine_labeler(
|
||||
SimpleNamespace(
|
||||
queue=queue_path,
|
||||
input=outputs,
|
||||
output=combined_path,
|
||||
report=directory / "combine-report.json",
|
||||
)
|
||||
)
|
||||
|
||||
self.assertEqual(5, report["outputCount"])
|
||||
self.assertEqual(
|
||||
[record["id"] for record in queue],
|
||||
[record["id"] for record in self._read(combined_path)],
|
||||
)
|
||||
|
||||
def _merge(
|
||||
self,
|
||||
primary_states,
|
||||
review_states,
|
||||
review_quoted=None,
|
||||
):
|
||||
review_quoted = review_quoted or [False] * len(review_states)
|
||||
with tempfile.TemporaryDirectory() as raw_directory:
|
||||
directory = Path(raw_directory)
|
||||
queue_path = directory / "queue.jsonl"
|
||||
self._write(
|
||||
queue_path,
|
||||
[
|
||||
{
|
||||
"id": "record-1",
|
||||
"text": "Could you send the report?",
|
||||
"language": "en",
|
||||
}
|
||||
],
|
||||
)
|
||||
primary = []
|
||||
for index, state in enumerate(primary_states):
|
||||
path = directory / f"primary-{index}.jsonl"
|
||||
self._write(path, [self._label("record-1", state)])
|
||||
primary.append((f"primary-{index}", path))
|
||||
review = []
|
||||
review_required = (
|
||||
len(
|
||||
consensus.primary_review_ids(
|
||||
consensus.load_labelers(
|
||||
primary,
|
||||
{"record-1"},
|
||||
True,
|
||||
),
|
||||
{"record-1"},
|
||||
)
|
||||
)
|
||||
== 1
|
||||
)
|
||||
if review_required:
|
||||
for index, state in enumerate(review_states):
|
||||
path = directory / f"review-{index}.jsonl"
|
||||
self._write(
|
||||
path,
|
||||
[
|
||||
self._label(
|
||||
"record-1",
|
||||
state,
|
||||
quoted=review_quoted[index],
|
||||
)
|
||||
],
|
||||
)
|
||||
review.append((f"review-{index}", path))
|
||||
else:
|
||||
for index in range(2):
|
||||
path = directory / f"review-{index}.jsonl"
|
||||
self._write(path, [])
|
||||
review.append((f"review-{index}", path))
|
||||
outputs = {
|
||||
name: directory / f"{name}.jsonl"
|
||||
for name in ("tier_a", "tier_b", "accepted", "human_review")
|
||||
}
|
||||
report_path = directory / "report.json"
|
||||
report = consensus.merge(
|
||||
SimpleNamespace(
|
||||
queue=queue_path,
|
||||
primary=primary,
|
||||
reviewer=review,
|
||||
report=report_path,
|
||||
**outputs,
|
||||
)
|
||||
)
|
||||
return (
|
||||
report,
|
||||
self._read(outputs["tier_a"]),
|
||||
self._read(outputs["tier_b"]),
|
||||
self._read(outputs["human_review"]),
|
||||
)
|
||||
|
||||
def _states(self, **overrides):
|
||||
values = {label: "false" for label in consensus.INTENT_LABELS}
|
||||
values.update(overrides)
|
||||
values["sentiment"] = overrides.get("sentiment", "neutral")
|
||||
values["domain"] = overrides.get("domain", "unknown")
|
||||
return values
|
||||
|
||||
def _label(self, identifier, states, quoted=False):
|
||||
return {
|
||||
"id": identifier,
|
||||
"labels": {
|
||||
label: states[label] for label in consensus.INTENT_LABELS
|
||||
},
|
||||
"sentiment": states["sentiment"],
|
||||
"domain": states["domain"],
|
||||
"ambiguous": False,
|
||||
"quotedOrMeta": quoted,
|
||||
"confidence": 0.95,
|
||||
}
|
||||
|
||||
def _write(self, path, records):
|
||||
path.write_text(
|
||||
"\n".join(json.dumps(record) for record in records)
|
||||
+ ("\n" if records else ""),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
def _read(self, path):
|
||||
return [
|
||||
json.loads(line)
|
||||
for line in path.read_text(encoding="utf-8").splitlines()
|
||||
if line
|
||||
]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,63 @@
|
||||
import importlib.util
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
SCRIPT_PATH = (
|
||||
Path(__file__).resolve().parents[1] / "prepare_apple_nl_hardening_corpus.py"
|
||||
)
|
||||
SPEC = importlib.util.spec_from_file_location("prepare_apple_nl_hardening_corpus", SCRIPT_PATH)
|
||||
MODULE = importlib.util.module_from_spec(SPEC)
|
||||
assert SPEC.loader is not None
|
||||
SPEC.loader.exec_module(MODULE)
|
||||
|
||||
|
||||
class PrepareAppleNLHardeningCorpusTests(unittest.TestCase):
|
||||
def test_prepare_removes_id_and_normalized_text_overlap(self) -> None:
|
||||
training = [
|
||||
{"id": "train-1", "text": "Keep me", "language": "en", "split": "train"},
|
||||
{"id": "shared-id", "text": "Different", "language": "en", "split": "train"},
|
||||
{"id": "train-3", "text": " SAME TEXT ", "language": "en", "split": "train"},
|
||||
]
|
||||
product = [
|
||||
{
|
||||
"id": "shared-id",
|
||||
"text": "Evaluation by id",
|
||||
"language": "en",
|
||||
"split": "validation",
|
||||
},
|
||||
{
|
||||
"id": "evaluation-2",
|
||||
"text": "same text",
|
||||
"language": "en",
|
||||
"split": "test",
|
||||
},
|
||||
{
|
||||
"id": "ignored-train",
|
||||
"text": "Not evaluation",
|
||||
"language": "en",
|
||||
"split": "train",
|
||||
},
|
||||
]
|
||||
|
||||
records, report = MODULE.prepare(training, product)
|
||||
|
||||
self.assertEqual([record["id"] for record in records], [
|
||||
"train-1",
|
||||
"shared-id",
|
||||
"evaluation-2",
|
||||
])
|
||||
self.assertEqual(report["filteredTrainingCount"], 1)
|
||||
self.assertEqual(report["excludedTrainingOverlapCount"], 2)
|
||||
self.assertEqual(report["evaluationOverlapCount"], 0)
|
||||
|
||||
def test_prepare_requires_evaluation_records(self) -> None:
|
||||
with self.assertRaisesRegex(ValueError, "no evaluation records"):
|
||||
MODULE.prepare(
|
||||
[{"id": "train", "text": "x", "language": "en", "split": "train"}],
|
||||
[{"id": "product", "text": "y", "language": "en", "split": "train"}],
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user