test(semantics): strengthen consensus split auditing

Co-authored-by: Rocky <hkgood@users.noreply.github.com>

Scripts/clipboard_semantics/generate_consensus_labels.py

@@ -13,7 +13,6 @@ import unicodedata
 from collections import Counter, defaultdict
 from pathlib import Path
 
-
 SEED = 20260827
 INTENT_LABELS = (
     "task",
@@ -48,9 +47,7 @@ DEFAULT_INPUT = Path("ModelTraining/ClipboardSemantics/open-training-corpus.json
 
 def normalized_text(value: str) -> str:
     return " ".join(
-        unicodedata.normalize("NFKC", value)
+        unicodedata.normalize("NFKC", value).replace("\u0000", " ").split()
-        .replace("\u0000", " ")
-        .split()
     ).strip()
 
 
@@ -69,8 +66,7 @@ def read_json_lines(path: Path) -> list[dict]:
 def write_json_lines(path: Path, records: list[dict]) -> None:
     path.parent.mkdir(parents=True, exist_ok=True)
     serialized = "\n".join(
-        json.dumps(record, ensure_ascii=False, sort_keys=True)
+        json.dumps(record, ensure_ascii=False, sort_keys=True) for record in records
-        for record in records
     )
     path.write_text(serialized + ("\n" if serialized else ""), encoding="utf-8")
 
@@ -118,8 +114,8 @@ def stratified_queue(
                 record.get("language", "unknown"),
             )
         ].append(record)
-    for key in grouped:
+    for values in grouped.values():
-        grouped[key].sort(
+        values.sort(
             key=lambda record: queue_priority(record, hard_negative_texts),
             reverse=True,
         )
@@ -232,8 +228,7 @@ def prepare(arguments: argparse.Namespace) -> None:
         "sourceCounts": dict(
             sorted(
                 Counter(
-                    record.get("sourceDataset") or "unknown"
+                    record.get("sourceDataset") or "unknown" for record in queue_records
-                    for record in queue_records
                 ).items()
             )
         ),
@@ -263,14 +258,16 @@ def validate_labeler_record(record: dict, expected_ids: set[str]) -> dict:
     if record_id not in expected_ids:
         raise ValueError(f"Unexpected labeler record id: {record_id}")
     labels = record.get("labels")
-    if not isinstance(labels, list) or any(label not in INTENT_LABELS for label in labels):
+    if not isinstance(labels, list) or any(
+        label not in INTENT_LABELS for label in labels
+    ):
         raise ValueError(f"Unsupported labels for {record_id}: {labels}")
     confidence = record.get("confidence")
     if not isinstance(confidence, (int, float)) or not 0 <= confidence <= 1:
         raise ValueError(f"Invalid confidence for {record_id}: {confidence}")
     for flag in SPECIAL_FLAGS:
         if not isinstance(record.get(flag), bool):
-            raise ValueError(f"Missing boolean {flag} for {record_id}")
+            raise TypeError(f"Missing boolean {flag} for {record_id}")
     return {
         "id": record_id,
         "labels": sorted(set(labels)),
@@ -294,9 +291,7 @@ def action_label(labels: set[str]) -> str:
 
 def coordination_label(labels: set[str]) -> str | None:
     matches = [
-        label
+        label for label in COORDINATION_LABELS if label != "neither" and label in labels
-        for label in COORDINATION_LABELS
-        if label != "neither" and label in labels
     ]
     if len(matches) > 1:
         return None
@@ -381,9 +376,7 @@ def merge(arguments: argparse.Namespace) -> None:
     for record_id in sorted(expected_ids):
         queue_record = queue_by_id[record_id]
         votes = Counter(
-            label
+            label for _, records in labelers for label in records[record_id]["labels"]
-            for _, records in labelers
-            for label in records[record_id]["labels"]
         )
         source_labels = {
             label
@@ -403,9 +396,8 @@ def merge(arguments: argparse.Namespace) -> None:
         )
         coordination = coordination_label(consensus_labels)
         full_agreement = all(
-            set(records[record_id]["labels"]) == set(
+            set(records[record_id]["labels"])
-                labelers[0][1][record_id]["labels"]
+            == set(labelers[0][1][record_id]["labels"])
-            )
             and records[record_id]["ambiguous"]
             == labelers[0][1][record_id]["ambiguous"]
             and records[record_id]["quotedOrMeta"]
@@ -426,8 +418,7 @@ def merge(arguments: argparse.Namespace) -> None:
             "id": record_id,
             "labelerVotes": dict(sorted(votes.items())),
             "labelerConfidences": {
-                name: records[record_id]["confidence"]
+                name: records[record_id]["confidence"] for name, records in labelers
-                for name, records in labelers
             },
             "labelerResponseHashes": {
                 name: hashlib.sha256(
@@ -472,9 +463,8 @@ def merge(arguments: argparse.Namespace) -> None:
             "quotedOrMeta": quoted_votes >= 2,
             "consensusAgreement": round(
                 max(
-                    [votes.get(label, 0) for label in INTENT_LABELS] + [
+                    [votes.get(label, 0) for label in INTENT_LABELS]
-                        len(labelers) if not labels and full_agreement else 0
+                    + [len(labelers) if not labels and full_agreement else 0]
-                    ]
                 )
                 / len(labelers),
                 4,
@@ -503,14 +493,23 @@ def merge(arguments: argparse.Namespace) -> None:
     labeler_maps = [records for _, records in labelers]
     text_split_map: dict[str, set[str]] = defaultdict(set)
     cluster_split_map: dict[str, set[str]] = defaultdict(set)
+    source_cluster_split_map: dict[str, set[str]] = defaultdict(set)
     for record in accepted:
         text_split_map[normalized_text(record["text"]).casefold()].add(record["split"])
         cluster_split_map[cluster_signature(record["text"])].add(record["split"])
+        source_cluster_key = (
+            f"{record.get('sourceDataset') or 'unknown'}|"
+            f"{cluster_signature(record['text'])}"
+        )
+        source_cluster_split_map[source_cluster_key].add(record["split"])
     exact_overlap_count = sum(len(splits) > 1 for splits in text_split_map.values())
     cluster_overlap_count = sum(
         len(splits) > 1 for splits in cluster_split_map.values()
     )
-    if exact_overlap_count or cluster_overlap_count:
+    source_cluster_overlap_count = sum(
+        len(splits) > 1 for splits in source_cluster_split_map.values()
+    )
+    if exact_overlap_count or cluster_overlap_count or source_cluster_overlap_count:
         raise ValueError("Silver split overlap validation failed")
     report = {
         "schemaVersion": 1,
@@ -527,6 +526,7 @@ def merge(arguments: argparse.Namespace) -> None:
         "overlapChecks": {
             "exactTextAcrossSplits": exact_overlap_count,
             "nearDuplicateClusterAcrossSplits": cluster_overlap_count,
+            "sourceNearDuplicateClusterAcrossSplits": (source_cluster_overlap_count),
         },
         "fullAgreementCount": sum(record["fullAgreement"] for record in accepted)
         + sum(record["fullAgreement"] for record in conflicts),
@@ -535,9 +535,13 @@ def merge(arguments: argparse.Namespace) -> None:
             sorted(expected_ids),
         ),
         "labelers": [name for name, _ in labelers],
-        "splitCounts": dict(sorted(Counter(record["split"] for record in accepted).items())),
+        "splitCounts": dict(
+            sorted(Counter(record["split"] for record in accepted).items())
+        ),
         "actionLabelCounts": dict(
-            sorted(Counter(record["actionVerifierLabel"] for record in accepted).items())
+            sorted(
+                Counter(record["actionVerifierLabel"] for record in accepted).items()
+            )
         ),
         "coordinationLabelCounts": dict(
             sorted(
@@ -556,11 +560,24 @@ def merge(arguments: argparse.Namespace) -> None:
         "sourceCounts": dict(
             sorted(
                 Counter(
-                    record.get("sourceDataset") or "unknown"
+                    record.get("sourceDataset") or "unknown" for record in accepted
-                    for record in accepted
                 ).items()
             )
         ),
+        "sourceSplitCounts": {
+            source: dict(
+                sorted(
+                    Counter(
+                        record["split"]
+                        for record in accepted
+                        if (record.get("sourceDataset") or "unknown") == source
+                    ).items()
+                )
+            )
+            for source in sorted(
+                {record.get("sourceDataset") or "unknown" for record in accepted}
+            )
+        },
     }
     arguments.report.write_text(
         json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n",

Scripts/clipboard_semantics/tests/test_generate_consensus_labels.py

@@ -1,11 +1,10 @@
 import json
+import sys
 import tempfile
 import unittest
 from pathlib import Path
 from types import SimpleNamespace
 
-import sys
-
 sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
 
 import generate_consensus_labels as consensus
@@ -32,6 +31,26 @@ class ConsensusLabelTests(unittest.TestCase):
         self.assertEqual("taskOnly", accepted[0]["actionVerifierLabel"])
         self.assertTrue(accepted[0]["task"])
         self.assertEqual(1, report["acceptedCount"])
+        self.assertEqual(
+            {"labeler-0", "labeler-1", "labeler-2"},
+            set(accepted[0]["labelerResponseHashes"]),
+        )
+        self.assertTrue(
+            all(
+                len(value) == 64
+                for value in accepted[0]["labelerResponseHashes"].values()
+            )
+        )
+        self.assertEqual(
+            0,
+            report["overlapChecks"][
+                "sourceNearDuplicateClusterAcrossSplits"
+            ],
+        )
+        self.assertEqual(
+            1,
+            sum(report["sourceSplitCounts"]["fixture"].values()),
+        )
 
     def test_rejects_multi_coordination_consensus(self):
         _, accepted, conflicts = self._merge(
@@ -52,9 +71,11 @@ class ConsensusLabelTests(unittest.TestCase):
     def test_near_duplicate_slot_variants_share_split(self):
         first = {
             "text": "Could you send report 123 before Friday?",
+            "sourceDataset": "fixture",
         }
         second = {
             "text": "Could you send report 456 before Friday?",
+            "sourceDataset": "fixture",
         }
 
         self.assertEqual(
@@ -62,7 +83,41 @@ class ConsensusLabelTests(unittest.TestCase):
             consensus.split_for(second),
         )
 
-    def _merge(self, source_labels, label_sets):
+    def test_rejects_ambiguous_majority(self):
+        _, accepted, conflicts = self._merge(
+            source_labels={"task": True},
+            label_sets=[["task"], ["task"], ["task"]],
+            ambiguous_flags=[True, True, False],
+        )
+
+        self.assertEqual([], accepted)
+        self.assertEqual(
+            "ambiguous-majority",
+            conflicts[0]["rejectedReason"],
+        )
+
+    def test_rejects_quoted_intent_majority(self):
+        _, accepted, conflicts = self._merge(
+            source_labels={"question": True},
+            label_sets=[["question"], ["question"], ["question"]],
+            quoted_flags=[True, True, False],
+        )
+
+        self.assertEqual([], accepted)
+        self.assertEqual(
+            "quoted-or-meta-intent",
+            conflicts[0]["rejectedReason"],
+        )
+
+    def _merge(
+        self,
+        source_labels,
+        label_sets,
+        ambiguous_flags=None,
+        quoted_flags=None,
+    ):
+        ambiguous_flags = ambiguous_flags or [False] * len(label_sets)
+        quoted_flags = quoted_flags or [False] * len(label_sets)
         with tempfile.TemporaryDirectory() as raw_directory:
             directory = Path(raw_directory)
             queue_path = directory / "queue.jsonl"
@@ -88,8 +143,8 @@ class ConsensusLabelTests(unittest.TestCase):
                         {
                             "id": "record-1",
                             "labels": labels,
-                            "ambiguous": False,
+                            "ambiguous": ambiguous_flags[index],
-                            "quotedOrMeta": False,
+                            "quotedOrMeta": quoted_flags[index],
                             "confidence": 0.95,
                         }
                     ],