Rocky
aa37067f79
- Add reproducible v6 boundary, blessing, and consensus-adjudication corpora, plus the tiny-transformer trainer and v6 release-gate evaluator that gate every candidate on the deployed baselines. - Wire consensus-label merging, product-policy anchor evaluation, and sealed blessing benchmark review with their pytest coverage. - Refresh open-training corpus generation, iterative retraining runner, and random-holdout evaluation so v6 candidates can be benchmarked end-to-end.
269 lines
9.9 KiB
Python
269 lines
9.9 KiB
Python
import json
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
|
|
import adjudicate_consensus_conflicts as adjudication
|
|
|
|
|
|
class AdjudicationTests(unittest.TestCase):
|
|
def test_prepare_adds_all_product_policy_fields(self):
|
|
with tempfile.TemporaryDirectory() as raw_directory:
|
|
directory = Path(raw_directory)
|
|
conflicts = directory / "conflicts.jsonl"
|
|
self._write(
|
|
conflicts,
|
|
[
|
|
{
|
|
"id": "one",
|
|
"text": "Could you send the report?",
|
|
"language": "en",
|
|
"unresolvedFields": ["sentiment"],
|
|
}
|
|
],
|
|
)
|
|
queue = directory / "queue.jsonl"
|
|
report = adjudication.prepare(
|
|
SimpleNamespace(
|
|
conflicts=conflicts,
|
|
queue=queue,
|
|
chunk_directory=directory / "chunks",
|
|
chunk_size=10,
|
|
report=directory / "report.json",
|
|
include_product_policy_fields=True,
|
|
)
|
|
)
|
|
|
|
fields = self._read(queue)[0]["unresolvedFields"]
|
|
self.assertTrue(set(adjudication.PRODUCT_POLICY_FIELDS) <= set(fields))
|
|
self.assertIn("sentiment", fields)
|
|
self.assertTrue(report["includesProductPolicyFields"])
|
|
|
|
def test_requires_evidence_from_input_text(self):
|
|
queue = {
|
|
"id": "one",
|
|
"text": "Could you send the report?",
|
|
"unresolvedFields": ["task"],
|
|
}
|
|
record = self._adjudication("one", task="true")
|
|
record["evidence"]["task"] = "not present"
|
|
|
|
with self.assertRaisesRegex(ValueError, "exact text quote"):
|
|
adjudication.validate_adjudication(record, queue)
|
|
|
|
def test_accepts_matching_high_confidence_adjudication(self):
|
|
report, accepted, excluded, remaining = self._merge(
|
|
first=self._adjudication("one", task="true"),
|
|
second=self._adjudication("one", task="true"),
|
|
)
|
|
|
|
self.assertEqual(1, report["acceptedTierCCount"])
|
|
self.assertEqual(0, report["remainingHumanReviewCount"])
|
|
self.assertTrue(accepted[0]["task"])
|
|
self.assertEqual("C", accepted[0]["labelQualityTier"])
|
|
self.assertEqual(0.35, accepted[0]["sampleWeight"])
|
|
self.assertEqual([], excluded)
|
|
self.assertEqual([], remaining)
|
|
|
|
def test_keeps_disagreement_for_human_review(self):
|
|
report, accepted, excluded, remaining = self._merge(
|
|
first=self._adjudication("one", task="true"),
|
|
second=self._adjudication("one", task="false"),
|
|
)
|
|
|
|
self.assertEqual(0, report["acceptedTierCCount"])
|
|
self.assertEqual([], accepted)
|
|
self.assertEqual([], excluded)
|
|
self.assertEqual(
|
|
["adjudicator-disagreement"],
|
|
remaining[0]["aiAdjudication"]["rejectedFields"]["task"],
|
|
)
|
|
|
|
def test_keeps_low_confidence_for_human_review(self):
|
|
first = self._adjudication("one", task="true")
|
|
first["confidence"]["task"] = 0.89
|
|
report, _, _, remaining = self._merge(
|
|
first=first,
|
|
second=self._adjudication("one", task="true"),
|
|
)
|
|
|
|
self.assertEqual(1, report["remainingHumanReviewCount"])
|
|
self.assertEqual(
|
|
["low-confidence"],
|
|
remaining[0]["aiAdjudication"]["rejectedFields"]["task"],
|
|
)
|
|
|
|
def test_excludes_matching_high_confidence_device_command(self):
|
|
first = self._adjudication("one", task="true")
|
|
second = self._adjudication("one", task="true")
|
|
for record in (first, second):
|
|
record["recordDisposition"] = "exclude-device-command"
|
|
|
|
report, accepted, excluded, remaining = self._merge(first, second)
|
|
|
|
self.assertEqual(1, report["excludedDeviceCommandCount"])
|
|
self.assertEqual([], accepted)
|
|
self.assertEqual("exclude-device-command", excluded[0]["disposition"])
|
|
self.assertEqual([], remaining)
|
|
|
|
def test_keeps_disposition_disagreement_for_human_review(self):
|
|
first = self._adjudication("one", task="true")
|
|
second = self._adjudication("one", task="true")
|
|
second["recordDisposition"] = "exclude-device-command"
|
|
|
|
report, accepted, excluded, remaining = self._merge(first, second)
|
|
|
|
self.assertEqual(1, report["remainingHumanReviewCount"])
|
|
self.assertEqual([], accepted)
|
|
self.assertEqual([], excluded)
|
|
self.assertEqual(
|
|
["adjudicator-disagreement"],
|
|
remaining[0]["aiAdjudication"]["rejectedFields"][
|
|
"recordDisposition"
|
|
],
|
|
)
|
|
|
|
def test_review_sample_is_unique_and_includes_decision_template(self):
|
|
with tempfile.TemporaryDirectory() as raw_directory:
|
|
directory = Path(raw_directory)
|
|
remaining = directory / "remaining.jsonl"
|
|
records = []
|
|
for index, field in enumerate(("task", "recordDisposition", "task")):
|
|
first = self._adjudication(str(index), task="true")
|
|
second = self._adjudication(str(index), task="false")
|
|
records.append(
|
|
{
|
|
"id": str(index),
|
|
"text": "Could you send the report?",
|
|
"language": "en" if index % 2 else "zh-Hans",
|
|
"aiAdjudication": {
|
|
"rejectedFields": {
|
|
field: ["adjudicator-disagreement"]
|
|
},
|
|
"adjudicatorA": first,
|
|
"adjudicatorB": second,
|
|
},
|
|
}
|
|
)
|
|
self._write(remaining, records)
|
|
sample = directory / "sample.jsonl"
|
|
report = adjudication.review_sample(
|
|
SimpleNamespace(
|
|
remaining=remaining,
|
|
sample=sample,
|
|
sample_size=3,
|
|
report=directory / "report.json",
|
|
)
|
|
)
|
|
output = self._read(sample)
|
|
|
|
self.assertEqual(3, report["sampleCount"])
|
|
self.assertEqual(3, len({record["id"] for record in output}))
|
|
self.assertIsNone(output[0]["humanDecision"].popitem()[1])
|
|
disposition = next(
|
|
record
|
|
for record in output
|
|
if "recordDisposition" in record["fieldReviews"]
|
|
)
|
|
self.assertEqual(
|
|
"keep",
|
|
disposition["fieldReviews"]["recordDisposition"][
|
|
"adjudicatorA"
|
|
]["value"],
|
|
)
|
|
|
|
def _merge(self, first, second):
|
|
with tempfile.TemporaryDirectory() as raw_directory:
|
|
directory = Path(raw_directory)
|
|
conflicts = directory / "conflicts.jsonl"
|
|
queue = directory / "queue.jsonl"
|
|
first_path = directory / "first.jsonl"
|
|
second_path = directory / "second.jsonl"
|
|
self._write(
|
|
conflicts,
|
|
[
|
|
{
|
|
"id": "one",
|
|
"text": "Could you send the report?",
|
|
"language": "en",
|
|
"unresolvedFields": ["task"],
|
|
"modelVotes": {
|
|
**{
|
|
label: {"false": 4}
|
|
for label in adjudication.INTENT_LABELS
|
|
},
|
|
"sentiment": {"neutral": 4},
|
|
},
|
|
}
|
|
],
|
|
)
|
|
self._write(
|
|
queue,
|
|
[
|
|
{
|
|
"id": "one",
|
|
"text": "Could you send the report?",
|
|
"language": "en",
|
|
"unresolvedFields": ["task"],
|
|
}
|
|
],
|
|
)
|
|
self._write(first_path, [first])
|
|
self._write(second_path, [second])
|
|
accepted_path = directory / "accepted.jsonl"
|
|
excluded_path = directory / "excluded.jsonl"
|
|
remaining_path = directory / "remaining.jsonl"
|
|
report = adjudication.merge(
|
|
SimpleNamespace(
|
|
conflicts=conflicts,
|
|
queue=queue,
|
|
adjudicator_a=[first_path],
|
|
adjudicator_b=[second_path],
|
|
adjudicator_a_name="first",
|
|
adjudicator_b_name="second",
|
|
minimum_confidence=0.9,
|
|
accepted=accepted_path,
|
|
excluded=excluded_path,
|
|
remaining=remaining_path,
|
|
report=directory / "report.json",
|
|
)
|
|
)
|
|
return (
|
|
report,
|
|
self._read(accepted_path),
|
|
self._read(excluded_path),
|
|
self._read(remaining_path),
|
|
)
|
|
|
|
def _adjudication(self, identifier, task):
|
|
return {
|
|
"id": identifier,
|
|
"recordDisposition": "keep",
|
|
"dispositionConfidence": 0.95,
|
|
"dispositionEvidence": "send the report",
|
|
"resolutions": {"task": task},
|
|
"confidence": {"task": 0.95},
|
|
"evidence": {"task": "send the report"},
|
|
}
|
|
|
|
def _write(self, path, records):
|
|
path.write_text(
|
|
"\n".join(json.dumps(record) for record in records) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
def _read(self, path):
|
|
return [
|
|
json.loads(line)
|
|
for line in path.read_text(encoding="utf-8").splitlines()
|
|
if line
|
|
]
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|