chore(semantics): add iterative retraining research pipeline

Co-authored-by: Rocky <hkgood@users.noreply.github.com>

Scripts/clipboard_semantics/generate_open_training_corpus.py

@@ -366,6 +366,11 @@ def parse_arguments() -> argparse.Namespace:
     parser.add_argument("--open-output", type=Path, default=OPEN_CORPUS_PATH)
     parser.add_argument("--combined-output", type=Path, default=COMBINED_CORPUS_PATH)
     parser.add_argument("--sources-output", type=Path, default=SOURCES_PATH)
+    parser.add_argument(
+        "--allow-unavailable-sources",
+        action="store_true",
+        help="Continue with license-safe sources that are reachable.",
+    )
     return parser.parse_args()
 
 
@@ -1172,16 +1177,30 @@ def validate_records(records: list[dict], holdouts: set[str]) -> None:
 def main() -> None:
     arguments = parse_arguments()
     rng = random.Random(arguments.seed)
-    sources = (
+    source_builders = (
-        massive_records(arguments.seed),
+        ("MASSIVE", massive_records),
-        crosswoz_records(arguments.seed),
+        ("CrossWOZ", crosswoz_records),
-        go_emotions_records(arguments.seed),
+        ("GoEmotions", go_emotions_records),
-        multidogo_records(arguments.seed),
+        ("MultiDoGO", multidogo_records),
-        taskmaster_records(arguments.seed),
+        ("Taskmaster-1", taskmaster_records),
-        clinc_records(arguments.seed),
+        ("CLINC150", clinc_records),
-        cfpb_records(arguments.seed),
+        ("CFPB", cfpb_records),
-        asap_records(arguments.seed),
+        ("ASAP", asap_records),
     )
+    sources: list[list[dict]] = []
+    unavailable_sources: list[dict[str, str]] = []
+    for source_name, builder in source_builders:
+        try:
+            sources.append(builder(arguments.seed))
+        except (HTTPError, URLError, TimeoutError, ConnectionError, OSError) as error:
+            if not arguments.allow_unavailable_sources:
+                raise
+            unavailable_sources.append(
+                {
+                    "dataset": source_name,
+                    "reason": f"{type(error).__name__}: {error}",
+                }
+            )
     candidates = [record_value for source in sources for record_value in source]
     rng.shuffle(candidates)
 
@@ -1282,6 +1301,7 @@ def main() -> None:
                 "reason": "No clean official train split independent from the frozen holdout.",
             },
         ],
+        "unavailableSources": unavailable_sources,
         "baseCorpusRecords": len(base_records),
         "openTrainingRecords": len(selected),
         "combinedRecords": len(combined),

Scripts/clipboard_semantics/requirements-research.txt

@@ -0,0 +1,3 @@
+numpy==2.4.4
+scikit-learn==1.9.0
+scipy==1.18.1

Scripts/clipboard_semantics/tests/test_iterative_retraining.py

@@ -0,0 +1,96 @@
+import sys
+import unittest
+from pathlib import Path
+
+import numpy as np
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+import run_iterative_retraining as research
+
+
+class IterativeRetrainingTests(unittest.TestCase):
+    def test_defines_exactly_twenty_distinct_rounds(self):
+        configurations = research.configurations()
+
+        self.assertEqual(20, len(configurations))
+        self.assertEqual(list(range(1, 21)), [value.round for value in configurations])
+        self.assertEqual(
+            20,
+            len(
+                {
+                    (
+                        value.char_min,
+                        value.char_max,
+                        value.word_max,
+                        value.alpha,
+                        value.augmentation,
+                        value.hard_example_weight,
+                    )
+                    for value in configurations
+                }
+            ),
+        )
+
+    def test_threshold_selection_prioritizes_precision(self):
+        expected = np.array([True, True, False, False], dtype=bool)
+        probabilities = np.array([0.99, 0.70, 0.80, 0.10])
+
+        selection = research.select_threshold(
+            expected,
+            probabilities,
+            minimum_predictions=1,
+        )
+
+        self.assertGreater(selection["threshold"], 0.80)
+        self.assertEqual(1, selection["metrics"]["truePositive"])
+        self.assertEqual(0, selection["metrics"]["falsePositive"])
+
+    def test_runtime_requires_explicit_blessing_marker(self):
+        records = [
+            {"text": "The article quotes best wishes.", "language": "en"},
+            {"text": "Best wishes for your new role!", "language": "en"},
+        ]
+        probabilities = {
+            intent: np.array([0.0, 0.0]) for intent in research.INTENTS
+        }
+        probabilities["blessing"] = np.array([0.99, 0.99])
+        thresholds = {
+            intent: {"threshold": 0.5, "byLanguage": {}}
+            for intent in research.INTENTS
+        }
+
+        predicted = research.runtime_predictions(
+            records,
+            probabilities,
+            thresholds,
+        )
+
+        self.assertEqual([False, True], predicted["blessing"].tolist())
+
+    def test_runtime_suppresses_implicit_task_when_complaint_is_high(self):
+        records = [
+            {"text": "This is broken again.", "language": "en"},
+            {"text": "This is broken again, please fix it.", "language": "en"},
+        ]
+        probabilities = {
+            intent: np.array([0.0, 0.0]) for intent in research.INTENTS
+        }
+        probabilities["task"] = np.array([0.99, 0.99])
+        probabilities["complaint"] = np.array([0.90, 0.90])
+        thresholds = {
+            intent: {"threshold": 0.5, "byLanguage": {}}
+            for intent in research.INTENTS
+        }
+
+        predicted = research.runtime_predictions(
+            records,
+            probabilities,
+            thresholds,
+        )
+
+        self.assertEqual([False, True], predicted["task"].tolist())
+
+
+if __name__ == "__main__":
+    unittest.main()