Cursor: Apply local changes for cloud agent
This commit is contained in:
@@ -0,0 +1,138 @@
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
import generate_consensus_labels as consensus
|
||||
|
||||
|
||||
class ConsensusLabelTests(unittest.TestCase):
|
||||
def test_requires_source_support_for_two_of_three_vote(self):
|
||||
report, accepted, conflicts = self._merge(
|
||||
source_labels={},
|
||||
label_sets=[["task"], ["task"], []],
|
||||
)
|
||||
|
||||
self.assertEqual([], accepted)
|
||||
self.assertEqual("no-supported-consensus", conflicts[0]["rejectedReason"])
|
||||
self.assertEqual(1, report["conflictCount"])
|
||||
|
||||
def test_accepts_two_of_three_when_official_label_supports_vote(self):
|
||||
report, accepted, conflicts = self._merge(
|
||||
source_labels={"task": True},
|
||||
label_sets=[["task"], ["task"], []],
|
||||
)
|
||||
|
||||
self.assertEqual([], conflicts)
|
||||
self.assertEqual("taskOnly", accepted[0]["actionVerifierLabel"])
|
||||
self.assertTrue(accepted[0]["task"])
|
||||
self.assertEqual(1, report["acceptedCount"])
|
||||
|
||||
def test_rejects_multi_coordination_consensus(self):
|
||||
_, accepted, conflicts = self._merge(
|
||||
source_labels={"invitation": True, "scheduleNegotiation": True},
|
||||
label_sets=[
|
||||
["invitation", "scheduleNegotiation"],
|
||||
["invitation", "scheduleNegotiation"],
|
||||
["invitation", "scheduleNegotiation"],
|
||||
],
|
||||
)
|
||||
|
||||
self.assertEqual([], accepted)
|
||||
self.assertEqual(
|
||||
"multiple-coordination-labels",
|
||||
conflicts[0]["rejectedReason"],
|
||||
)
|
||||
|
||||
def test_near_duplicate_slot_variants_share_split(self):
|
||||
first = {
|
||||
"text": "Could you send report 123 before Friday?",
|
||||
}
|
||||
second = {
|
||||
"text": "Could you send report 456 before Friday?",
|
||||
}
|
||||
|
||||
self.assertEqual(
|
||||
consensus.split_for(first),
|
||||
consensus.split_for(second),
|
||||
)
|
||||
|
||||
def _merge(self, source_labels, label_sets):
|
||||
with tempfile.TemporaryDirectory() as raw_directory:
|
||||
directory = Path(raw_directory)
|
||||
queue_path = directory / "queue.jsonl"
|
||||
self._write_json_lines(
|
||||
queue_path,
|
||||
[
|
||||
{
|
||||
"id": "record-1",
|
||||
"text": "Could you send the report?",
|
||||
"language": "en",
|
||||
"sourceDataset": "fixture",
|
||||
"sourceLicense": "MIT",
|
||||
"sourceLabels": source_labels,
|
||||
}
|
||||
],
|
||||
)
|
||||
labelers = []
|
||||
for index, labels in enumerate(label_sets):
|
||||
path = directory / f"labeler-{index}.jsonl"
|
||||
self._write_json_lines(
|
||||
path,
|
||||
[
|
||||
{
|
||||
"id": "record-1",
|
||||
"labels": labels,
|
||||
"ambiguous": False,
|
||||
"quotedOrMeta": False,
|
||||
"confidence": 0.95,
|
||||
}
|
||||
],
|
||||
)
|
||||
labelers.append((f"labeler-{index}", path))
|
||||
paths = {
|
||||
name: directory / f"{name}.jsonl"
|
||||
for name in (
|
||||
"consensus",
|
||||
"conflicts",
|
||||
"train",
|
||||
"calibration",
|
||||
"acceptance",
|
||||
)
|
||||
}
|
||||
report_path = directory / "report.json"
|
||||
consensus.merge(
|
||||
SimpleNamespace(
|
||||
queue=queue_path,
|
||||
labeler=labelers,
|
||||
report=report_path,
|
||||
**paths,
|
||||
)
|
||||
)
|
||||
return (
|
||||
json.loads(report_path.read_text(encoding="utf-8")),
|
||||
self._read_json_lines(paths["consensus"]),
|
||||
self._read_json_lines(paths["conflicts"]),
|
||||
)
|
||||
|
||||
def _write_json_lines(self, path, records):
|
||||
path.write_text(
|
||||
"\n".join(json.dumps(record) for record in records) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
def _read_json_lines(self, path):
|
||||
return [
|
||||
json.loads(line)
|
||||
for line in path.read_text(encoding="utf-8").splitlines()
|
||||
if line
|
||||
]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user