Cursor: Apply local changes for cloud agent
This commit is contained in:
Rocky
@@ -0,0 +1,109 @@
|
||||
import json
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
SCRIPT = Path(__file__).resolve().parents[1] / "apply_verifier_deployment_policy.py"
|
||||
|
||||
|
||||
class VerifierDeploymentPolicyTests(unittest.TestCase):
|
||||
def test_rejected_candidate_cannot_change_deployed_manifest(self):
|
||||
with self._fixture(accepted=False) as fixture:
|
||||
self._run(fixture)
|
||||
|
||||
deployed = self._read(fixture["deployed_manifest"])
|
||||
report = self._read(fixture["report"])
|
||||
self.assertEqual(2, deployed["schemaVersion"])
|
||||
self.assertFalse(report["applied"])
|
||||
self.assertEqual(0, report["eligibleVerifierCount"])
|
||||
|
||||
def test_passing_candidate_enters_shadow_and_preserves_classifiers(self):
|
||||
with self._fixture(accepted=True) as fixture:
|
||||
self._run(fixture)
|
||||
|
||||
deployed = self._read(fixture["deployed_manifest"])
|
||||
report = self._read(fixture["report"])
|
||||
self.assertEqual(3, deployed["schemaVersion"])
|
||||
self.assertEqual("task", deployed["classifiers"][0]["id"])
|
||||
self.assertEqual("shadow", deployed["verifiers"][0]["deploymentMode"])
|
||||
self.assertFalse(
|
||||
deployed["verifiers"][0]["acceptedForAutomaticRouting"]
|
||||
)
|
||||
self.assertTrue(report["applied"])
|
||||
|
||||
def _fixture(self, accepted):
|
||||
temporary = tempfile.TemporaryDirectory()
|
||||
root = Path(temporary.name)
|
||||
candidate = root / "candidate"
|
||||
resource = root / "resource"
|
||||
candidate.mkdir()
|
||||
resource.mkdir()
|
||||
verifier = {
|
||||
"id": "action",
|
||||
"modelFile": "ActionIntentVerifier.mlmodel",
|
||||
"acceptedForAutomaticRouting": accepted,
|
||||
"deploymentMode": "automatic" if accepted else "shadow",
|
||||
}
|
||||
(candidate / verifier["modelFile"]).write_bytes(b"model")
|
||||
self._write(
|
||||
candidate / "clipboard-semantic-models.json",
|
||||
{
|
||||
"schemaVersion": 3,
|
||||
"classifiers": [{"id": "untrusted-candidate"}],
|
||||
"verifiers": [verifier],
|
||||
},
|
||||
)
|
||||
deployed_manifest = resource / "clipboard-semantic-models.json"
|
||||
self._write(
|
||||
deployed_manifest,
|
||||
{
|
||||
"schemaVersion": 2,
|
||||
"classifiers": [{"id": "task"}],
|
||||
},
|
||||
)
|
||||
fixture = {
|
||||
"temporary": temporary,
|
||||
"candidate": candidate,
|
||||
"resource": resource,
|
||||
"deployed_manifest": deployed_manifest,
|
||||
"report": root / "report.json",
|
||||
}
|
||||
|
||||
class Context:
|
||||
def __enter__(self):
|
||||
return fixture
|
||||
|
||||
def __exit__(self, *unused):
|
||||
temporary.cleanup()
|
||||
|
||||
return Context()
|
||||
|
||||
def _run(self, fixture):
|
||||
subprocess.run(
|
||||
[
|
||||
"python3",
|
||||
str(SCRIPT),
|
||||
"--candidate-directory",
|
||||
str(fixture["candidate"]),
|
||||
"--resource-directory",
|
||||
str(fixture["resource"]),
|
||||
"--report",
|
||||
str(fixture["report"]),
|
||||
"--apply",
|
||||
],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
def _write(self, path, value):
|
||||
path.write_text(json.dumps(value), encoding="utf-8")
|
||||
|
||||
def _read(self, path):
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,138 @@
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
import generate_consensus_labels as consensus
|
||||
|
||||
|
||||
class ConsensusLabelTests(unittest.TestCase):
|
||||
def test_requires_source_support_for_two_of_three_vote(self):
|
||||
report, accepted, conflicts = self._merge(
|
||||
source_labels={},
|
||||
label_sets=[["task"], ["task"], []],
|
||||
)
|
||||
|
||||
self.assertEqual([], accepted)
|
||||
self.assertEqual("no-supported-consensus", conflicts[0]["rejectedReason"])
|
||||
self.assertEqual(1, report["conflictCount"])
|
||||
|
||||
def test_accepts_two_of_three_when_official_label_supports_vote(self):
|
||||
report, accepted, conflicts = self._merge(
|
||||
source_labels={"task": True},
|
||||
label_sets=[["task"], ["task"], []],
|
||||
)
|
||||
|
||||
self.assertEqual([], conflicts)
|
||||
self.assertEqual("taskOnly", accepted[0]["actionVerifierLabel"])
|
||||
self.assertTrue(accepted[0]["task"])
|
||||
self.assertEqual(1, report["acceptedCount"])
|
||||
|
||||
def test_rejects_multi_coordination_consensus(self):
|
||||
_, accepted, conflicts = self._merge(
|
||||
source_labels={"invitation": True, "scheduleNegotiation": True},
|
||||
label_sets=[
|
||||
["invitation", "scheduleNegotiation"],
|
||||
["invitation", "scheduleNegotiation"],
|
||||
["invitation", "scheduleNegotiation"],
|
||||
],
|
||||
)
|
||||
|
||||
self.assertEqual([], accepted)
|
||||
self.assertEqual(
|
||||
"multiple-coordination-labels",
|
||||
conflicts[0]["rejectedReason"],
|
||||
)
|
||||
|
||||
def test_near_duplicate_slot_variants_share_split(self):
|
||||
first = {
|
||||
"text": "Could you send report 123 before Friday?",
|
||||
}
|
||||
second = {
|
||||
"text": "Could you send report 456 before Friday?",
|
||||
}
|
||||
|
||||
self.assertEqual(
|
||||
consensus.split_for(first),
|
||||
consensus.split_for(second),
|
||||
)
|
||||
|
||||
def _merge(self, source_labels, label_sets):
|
||||
with tempfile.TemporaryDirectory() as raw_directory:
|
||||
directory = Path(raw_directory)
|
||||
queue_path = directory / "queue.jsonl"
|
||||
self._write_json_lines(
|
||||
queue_path,
|
||||
[
|
||||
{
|
||||
"id": "record-1",
|
||||
"text": "Could you send the report?",
|
||||
"language": "en",
|
||||
"sourceDataset": "fixture",
|
||||
"sourceLicense": "MIT",
|
||||
"sourceLabels": source_labels,
|
||||
}
|
||||
],
|
||||
)
|
||||
labelers = []
|
||||
for index, labels in enumerate(label_sets):
|
||||
path = directory / f"labeler-{index}.jsonl"
|
||||
self._write_json_lines(
|
||||
path,
|
||||
[
|
||||
{
|
||||
"id": "record-1",
|
||||
"labels": labels,
|
||||
"ambiguous": False,
|
||||
"quotedOrMeta": False,
|
||||
"confidence": 0.95,
|
||||
}
|
||||
],
|
||||
)
|
||||
labelers.append((f"labeler-{index}", path))
|
||||
paths = {
|
||||
name: directory / f"{name}.jsonl"
|
||||
for name in (
|
||||
"consensus",
|
||||
"conflicts",
|
||||
"train",
|
||||
"calibration",
|
||||
"acceptance",
|
||||
)
|
||||
}
|
||||
report_path = directory / "report.json"
|
||||
consensus.merge(
|
||||
SimpleNamespace(
|
||||
queue=queue_path,
|
||||
labeler=labelers,
|
||||
report=report_path,
|
||||
**paths,
|
||||
)
|
||||
)
|
||||
return (
|
||||
json.loads(report_path.read_text(encoding="utf-8")),
|
||||
self._read_json_lines(paths["consensus"]),
|
||||
self._read_json_lines(paths["conflicts"]),
|
||||
)
|
||||
|
||||
def _write_json_lines(self, path, records):
|
||||
path.write_text(
|
||||
"\n".join(json.dumps(record) for record in records) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
def _read_json_lines(self, path):
|
||||
return [
|
||||
json.loads(line)
|
||||
for line in path.read_text(encoding="utf-8").splitlines()
|
||||
if line
|
||||
]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user