Rocky
139 lines
4.4 KiB
Python
139 lines
4.4 KiB
Python
import json
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
import sys
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
|
|
import generate_consensus_labels as consensus
|
|
|
|
|
|
class ConsensusLabelTests(unittest.TestCase):
|
|
def test_requires_source_support_for_two_of_three_vote(self):
|
|
report, accepted, conflicts = self._merge(
|
|
source_labels={},
|
|
label_sets=[["task"], ["task"], []],
|
|
)
|
|
|
|
self.assertEqual([], accepted)
|
|
self.assertEqual("no-supported-consensus", conflicts[0]["rejectedReason"])
|
|
self.assertEqual(1, report["conflictCount"])
|
|
|
|
def test_accepts_two_of_three_when_official_label_supports_vote(self):
|
|
report, accepted, conflicts = self._merge(
|
|
source_labels={"task": True},
|
|
label_sets=[["task"], ["task"], []],
|
|
)
|
|
|
|
self.assertEqual([], conflicts)
|
|
self.assertEqual("taskOnly", accepted[0]["actionVerifierLabel"])
|
|
self.assertTrue(accepted[0]["task"])
|
|
self.assertEqual(1, report["acceptedCount"])
|
|
|
|
def test_rejects_multi_coordination_consensus(self):
|
|
_, accepted, conflicts = self._merge(
|
|
source_labels={"invitation": True, "scheduleNegotiation": True},
|
|
label_sets=[
|
|
["invitation", "scheduleNegotiation"],
|
|
["invitation", "scheduleNegotiation"],
|
|
["invitation", "scheduleNegotiation"],
|
|
],
|
|
)
|
|
|
|
self.assertEqual([], accepted)
|
|
self.assertEqual(
|
|
"multiple-coordination-labels",
|
|
conflicts[0]["rejectedReason"],
|
|
)
|
|
|
|
def test_near_duplicate_slot_variants_share_split(self):
|
|
first = {
|
|
"text": "Could you send report 123 before Friday?",
|
|
}
|
|
second = {
|
|
"text": "Could you send report 456 before Friday?",
|
|
}
|
|
|
|
self.assertEqual(
|
|
consensus.split_for(first),
|
|
consensus.split_for(second),
|
|
)
|
|
|
|
def _merge(self, source_labels, label_sets):
|
|
with tempfile.TemporaryDirectory() as raw_directory:
|
|
directory = Path(raw_directory)
|
|
queue_path = directory / "queue.jsonl"
|
|
self._write_json_lines(
|
|
queue_path,
|
|
[
|
|
{
|
|
"id": "record-1",
|
|
"text": "Could you send the report?",
|
|
"language": "en",
|
|
"sourceDataset": "fixture",
|
|
"sourceLicense": "MIT",
|
|
"sourceLabels": source_labels,
|
|
}
|
|
],
|
|
)
|
|
labelers = []
|
|
for index, labels in enumerate(label_sets):
|
|
path = directory / f"labeler-{index}.jsonl"
|
|
self._write_json_lines(
|
|
path,
|
|
[
|
|
{
|
|
"id": "record-1",
|
|
"labels": labels,
|
|
"ambiguous": False,
|
|
"quotedOrMeta": False,
|
|
"confidence": 0.95,
|
|
}
|
|
],
|
|
)
|
|
labelers.append((f"labeler-{index}", path))
|
|
paths = {
|
|
name: directory / f"{name}.jsonl"
|
|
for name in (
|
|
"consensus",
|
|
"conflicts",
|
|
"train",
|
|
"calibration",
|
|
"acceptance",
|
|
)
|
|
}
|
|
report_path = directory / "report.json"
|
|
consensus.merge(
|
|
SimpleNamespace(
|
|
queue=queue_path,
|
|
labeler=labelers,
|
|
report=report_path,
|
|
**paths,
|
|
)
|
|
)
|
|
return (
|
|
json.loads(report_path.read_text(encoding="utf-8")),
|
|
self._read_json_lines(paths["consensus"]),
|
|
self._read_json_lines(paths["conflicts"]),
|
|
)
|
|
|
|
def _write_json_lines(self, path, records):
|
|
path.write_text(
|
|
"\n".join(json.dumps(record) for record in records) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
def _read_json_lines(self, path):
|
|
return [
|
|
json.loads(line)
|
|
for line in path.read_text(encoding="utf-8").splitlines()
|
|
if line
|
|
]
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|