Rocky
aa37067f79
- Add reproducible v6 boundary, blessing, and consensus-adjudication corpora, plus the tiny-transformer trainer and v6 release-gate evaluator that gate every candidate on the deployed baselines. - Wire consensus-label merging, product-policy anchor evaluation, and sealed blessing benchmark review with their pytest coverage. - Refresh open-training corpus generation, iterative retraining runner, and random-holdout evaluation so v6 candidates can be benchmarked end-to-end.
121 lines
3.9 KiB
Python
121 lines
3.9 KiB
Python
import sys
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
import numpy as np
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
|
|
import run_iterative_retraining as research
|
|
|
|
|
|
class IterativeRetrainingTests(unittest.TestCase):
|
|
def test_defines_exactly_twenty_distinct_rounds(self):
|
|
configurations = research.configurations()
|
|
|
|
self.assertEqual(20, len(configurations))
|
|
self.assertEqual(list(range(1, 21)), [value.round for value in configurations])
|
|
self.assertEqual(
|
|
20,
|
|
len(
|
|
{
|
|
(
|
|
value.char_min,
|
|
value.char_max,
|
|
value.word_max,
|
|
value.alpha,
|
|
value.augmentation,
|
|
value.hard_example_weight,
|
|
)
|
|
for value in configurations
|
|
}
|
|
),
|
|
)
|
|
|
|
def test_threshold_selection_prioritizes_precision(self):
|
|
expected = np.array([True, True, False, False], dtype=bool)
|
|
probabilities = np.array([0.99, 0.70, 0.80, 0.10])
|
|
|
|
selection = research.select_threshold(
|
|
expected,
|
|
probabilities,
|
|
minimum_predictions=1,
|
|
)
|
|
|
|
self.assertGreater(selection["threshold"], 0.80)
|
|
self.assertEqual(1, selection["metrics"]["truePositive"])
|
|
self.assertEqual(0, selection["metrics"]["falsePositive"])
|
|
|
|
def test_runtime_requires_explicit_blessing_marker(self):
|
|
records = [
|
|
{"text": "The article quotes best wishes.", "language": "en"},
|
|
{"text": "Best wishes for your new role!", "language": "en"},
|
|
]
|
|
probabilities = {
|
|
intent: np.array([0.0, 0.0]) for intent in research.INTENTS
|
|
}
|
|
probabilities["blessing"] = np.array([0.99, 0.99])
|
|
thresholds = {
|
|
intent: {"threshold": 0.5, "byLanguage": {}}
|
|
for intent in research.INTENTS
|
|
}
|
|
|
|
predicted = research.runtime_predictions(
|
|
records,
|
|
probabilities,
|
|
thresholds,
|
|
)
|
|
|
|
self.assertEqual([False, True], predicted["blessing"].tolist())
|
|
|
|
def test_runtime_suppresses_implicit_task_when_complaint_is_high(self):
|
|
records = [
|
|
{"text": "This is broken again.", "language": "en"},
|
|
{"text": "This is broken again, please fix it.", "language": "en"},
|
|
]
|
|
probabilities = {
|
|
intent: np.array([0.0, 0.0]) for intent in research.INTENTS
|
|
}
|
|
probabilities["task"] = np.array([0.99, 0.99])
|
|
probabilities["complaint"] = np.array([0.90, 0.90])
|
|
thresholds = {
|
|
intent: {"threshold": 0.5, "byLanguage": {}}
|
|
for intent in research.INTENTS
|
|
}
|
|
|
|
predicted = research.runtime_predictions(
|
|
records,
|
|
probabilities,
|
|
thresholds,
|
|
)
|
|
|
|
self.assertEqual([False, True], predicted["task"].tolist())
|
|
|
|
def test_registry_sample_weight_is_applied(self):
|
|
configuration = research.configurations()[0]
|
|
record = {
|
|
"id": "weighted",
|
|
"text": "Play music",
|
|
"language": "en",
|
|
"knownLabels": ["assistantCommand"],
|
|
"assistantCommand": True,
|
|
"sampleWeight": 0.35,
|
|
}
|
|
|
|
self.assertIn("assistantCommand", research.INTENTS)
|
|
self.assertAlmostEqual(
|
|
configuration.external_weight * 0.35,
|
|
research.sample_weight(record, configuration),
|
|
)
|
|
|
|
def test_legacy_records_do_not_define_new_labels_as_false(self):
|
|
record = {"id": "legacy", "text": "Play music", "language": "en"}
|
|
|
|
self.assertTrue(research.is_known(record, "task"))
|
|
self.assertFalse(research.is_known(record, "assistantCommand"))
|
|
self.assertFalse(research.is_known(record, "systemNotification"))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|