From aa37067f796d8c47f813db79bc7f7528cb282f28 Mon Sep 17 00:00:00 2001 From: Rocky <72559939+hkgood@users.noreply.github.com> Date: Sat, 29 Aug 2026 11:51:42 +0800 Subject: [PATCH] chore(semantics): add v6 release gate pipeline - Add reproducible v6 boundary, blessing, and consensus-adjudication corpora, plus the tiny-transformer trainer and v6 release-gate evaluator that gate every candidate on the deployed baselines. - Wire consensus-label merging, product-policy anchor evaluation, and sealed blessing benchmark review with their pytest coverage. - Refresh open-training corpus generation, iterative retraining runner, and random-holdout evaluation so v6 candidates can be benchmarked end-to-end. --- .../BlessingBenchmark/README.md | 67 + .../Consensus/adjudication-instructions-v1.md | 40 + .../Consensus/adjudication-instructions-v2.md | 56 + .../Consensus/adjudication-instructions-v3.md | 23 + .../Consensus/adjudication-instructions-v4.md | 17 + .../Consensus/adjudication-instructions-v5.md | 79 + .../Consensus/labeling-instructions-v2.md | 70 + .../Consensus/labeling-instructions-v3.md | 46 + .../Consensus/labeling-instructions-v4.md | 64 + .../Consensus/labeling-instructions-v5.md | 39 + .../Consensus/labeling-instructions-v6.md | 201 ++ .../Consensus/product-policy-anchors-v1.json | 182 ++ .../blessing-labeling-guidelines.md | 70 + .../chinese-corpus-candidate-audit-v1.json | 235 +++ .../corpus-registry-sources.json | 302 +++ .../v6-blind-evaluation-report.json | 99 + ...-boundary-training-supplement-summary.json | 121 ++ .../v6-release-gate-report.json | 111 ++ .../adjudicate_consensus_conflicts.py | 563 ++++++ .../assemble_v6_model_corpus.py | 130 ++ .../benchmark_v6_models.swift | 569 ++++++ .../build_corpus_registry.py | 653 +++++++ .../evaluate_product_policy_anchors.py | 304 +++ .../evaluate_random_holdout.swift | 41 +- .../evaluate_v6_release_gates.py | 237 +++ .../extract_lccc_blessing_candidates.py | 291 +++ .../finalize_blessing_benchmark.py | 217 +++ .../finalize_v6_blind_holdout.py | 650 +++++++ .../generate_blessing_training_corpus.py | 1648 +++++++++++++++++ .../clipboard_semantics/generate_corpus.py | 35 +- .../generate_open_training_corpus.py | 873 +++++++-- .../generate_v6_boundary_corpus.py | 722 ++++++++ .../merge_consensus_labels_v2.py | 607 ++++++ .../prepare_apple_nl_hardening_corpus.py | 137 ++ .../prepare_blessing_benchmark.py | 306 +++ .../requirements-research.txt | 3 + .../run_iterative_retraining.py | 37 +- .../test_adjudicate_consensus_conflicts.py | 268 +++ .../tests/test_assemble_v6_model_corpus.py | 66 + .../tests/test_blessing_corpus_tools.py | 170 ++ .../tests/test_build_corpus_registry.py | 216 +++ .../test_evaluate_product_policy_anchors.py | 144 ++ .../tests/test_finalize_v6_blind_holdout.py | 181 ++ .../test_generate_open_training_corpus.py | 175 ++ .../tests/test_generate_v6_boundary_corpus.py | 199 ++ .../tests/test_iterative_retraining.py | 24 + .../tests/test_merge_consensus_labels_v2.py | 286 +++ .../test_prepare_apple_nl_hardening_corpus.py | 63 + .../clipboard_semantics/train_models.swift | 176 +- .../train_v6_tiny_transformers.py | 491 +++++ 50 files changed, 12107 insertions(+), 197 deletions(-) create mode 100644 ModelTraining/ClipboardSemantics/BlessingBenchmark/README.md create mode 100644 ModelTraining/ClipboardSemantics/Consensus/adjudication-instructions-v1.md create mode 100644 ModelTraining/ClipboardSemantics/Consensus/adjudication-instructions-v2.md create mode 100644 ModelTraining/ClipboardSemantics/Consensus/adjudication-instructions-v3.md create mode 100644 ModelTraining/ClipboardSemantics/Consensus/adjudication-instructions-v4.md create mode 100644 ModelTraining/ClipboardSemantics/Consensus/adjudication-instructions-v5.md create mode 100644 ModelTraining/ClipboardSemantics/Consensus/labeling-instructions-v2.md create mode 100644 ModelTraining/ClipboardSemantics/Consensus/labeling-instructions-v3.md create mode 100644 ModelTraining/ClipboardSemantics/Consensus/labeling-instructions-v4.md create mode 100644 ModelTraining/ClipboardSemantics/Consensus/labeling-instructions-v5.md create mode 100644 ModelTraining/ClipboardSemantics/Consensus/labeling-instructions-v6.md create mode 100644 ModelTraining/ClipboardSemantics/Consensus/product-policy-anchors-v1.json create mode 100644 ModelTraining/ClipboardSemantics/blessing-labeling-guidelines.md create mode 100644 ModelTraining/ClipboardSemantics/chinese-corpus-candidate-audit-v1.json create mode 100644 ModelTraining/ClipboardSemantics/corpus-registry-sources.json create mode 100644 ModelTraining/ClipboardSemantics/v6-blind-evaluation-report.json create mode 100644 ModelTraining/ClipboardSemantics/v6-boundary-training-supplement-summary.json create mode 100644 ModelTraining/ClipboardSemantics/v6-release-gate-report.json create mode 100644 Scripts/clipboard_semantics/adjudicate_consensus_conflicts.py create mode 100644 Scripts/clipboard_semantics/assemble_v6_model_corpus.py create mode 100644 Scripts/clipboard_semantics/benchmark_v6_models.swift create mode 100644 Scripts/clipboard_semantics/build_corpus_registry.py create mode 100644 Scripts/clipboard_semantics/evaluate_product_policy_anchors.py create mode 100644 Scripts/clipboard_semantics/evaluate_v6_release_gates.py create mode 100644 Scripts/clipboard_semantics/extract_lccc_blessing_candidates.py create mode 100644 Scripts/clipboard_semantics/finalize_blessing_benchmark.py create mode 100644 Scripts/clipboard_semantics/finalize_v6_blind_holdout.py create mode 100644 Scripts/clipboard_semantics/generate_blessing_training_corpus.py create mode 100644 Scripts/clipboard_semantics/generate_v6_boundary_corpus.py create mode 100644 Scripts/clipboard_semantics/merge_consensus_labels_v2.py create mode 100644 Scripts/clipboard_semantics/prepare_apple_nl_hardening_corpus.py create mode 100644 Scripts/clipboard_semantics/prepare_blessing_benchmark.py create mode 100644 Scripts/clipboard_semantics/tests/test_adjudicate_consensus_conflicts.py create mode 100644 Scripts/clipboard_semantics/tests/test_assemble_v6_model_corpus.py create mode 100644 Scripts/clipboard_semantics/tests/test_blessing_corpus_tools.py create mode 100644 Scripts/clipboard_semantics/tests/test_build_corpus_registry.py create mode 100644 Scripts/clipboard_semantics/tests/test_evaluate_product_policy_anchors.py create mode 100644 Scripts/clipboard_semantics/tests/test_finalize_v6_blind_holdout.py create mode 100644 Scripts/clipboard_semantics/tests/test_generate_open_training_corpus.py create mode 100644 Scripts/clipboard_semantics/tests/test_generate_v6_boundary_corpus.py create mode 100644 Scripts/clipboard_semantics/tests/test_merge_consensus_labels_v2.py create mode 100644 Scripts/clipboard_semantics/tests/test_prepare_apple_nl_hardening_corpus.py create mode 100644 Scripts/clipboard_semantics/train_v6_tiny_transformers.py diff --git a/ModelTraining/ClipboardSemantics/BlessingBenchmark/README.md b/ModelTraining/ClipboardSemantics/BlessingBenchmark/README.md new file mode 100644 index 0000000..10bc4b3 --- /dev/null +++ b/ModelTraining/ClipboardSemantics/BlessingBenchmark/README.md @@ -0,0 +1,67 @@ +# Blessing benchmark review + +This directory contains a blind, evaluation-only review queue derived from the +frozen comprehensive holdout. It must never be merged into training. + +## Review process + +1. Give `review-queue.jsonl` and one annotation template to each of two + independent human annotators. +2. Do not give annotators `sealed-provenance.jsonl`, the other annotator's + answers, model predictions, or previous weak labels. +3. Each annotator fills every `label`, `boundaryCategory`, and `confidence` + field in their own JSONL file. +4. Run `finalize_blessing_benchmark.py` with distinct annotator IDs. +5. If annotations disagree, give only `adjudication-needed.jsonl` to a third + reviewer and rerun the finalizer with the adjudication file. + +## Label + +Set `label` to `true` only when the author directly expresses a good wish, +congratulation, prayer, or hope for a recipient. Third-person and self-directed +wishes count. Requests for a blessing, quoted examples, received thanks, +celebration descriptions, ordinary greetings, reports of someone else's wish, +and sarcasm do not count. + +## Boundary categories + +Use one of these stable values: + +### Positive + +- `festival_or_birthday` +- `congratulation` +- `health_or_recovery` +- `travel_or_safety` +- `study_or_career` +- `general_good_wish` +- `third_person_or_group` +- `spiritual_or_prayer` + +### Negative + +- `meta_request_or_template` +- `received_thanks` +- `quoted_or_documented` +- `celebration_mention` +- `ordinary_greeting` +- `positive_language_only` +- `reported_wish` +- `sarcasm_or_anti_blessing` +- `unrelated` + +Use `confidence` values `high`, `medium`, or `low`. Explain genuinely +ambiguous context in `notes`. + +## Finalization + +```bash +python3 Scripts/clipboard_semantics/finalize_blessing_benchmark.py \ + --annotator-a-id reviewer-a \ + --annotator-b-id reviewer-b +``` + +The command refuses incomplete annotation, duplicate IDs, non-boolean labels, +invalid confidence, missing adjudication, or use of the same person as both +annotators. The finalized benchmark is split deterministically into calibration +and test records. diff --git a/ModelTraining/ClipboardSemantics/Consensus/adjudication-instructions-v1.md b/ModelTraining/ClipboardSemantics/Consensus/adjudication-instructions-v1.md new file mode 100644 index 0000000..6864373 --- /dev/null +++ b/ModelTraining/ClipboardSemantics/Consensus/adjudication-instructions-v1.md @@ -0,0 +1,40 @@ +# Clipboard semantic evidence adjudication v1 + +Review only the fields listed in `unresolvedFields`. Judge from the text itself; +do not inspect previous model votes, source labels, or another adjudicator. + +Return one JSON object per input record: + +```json +{ + "id": "same id", + "resolutions": { + "task": "true", + "ambiguous": "false" + }, + "confidence": { + "task": 0.97, + "ambiguous": 0.94 + }, + "evidence": { + "task": "send the report", + "ambiguous": "by Friday" + } +} +``` + +Requirements: + +- `resolutions`, `confidence`, and `evidence` must contain exactly the fields in + `unresolvedFields`. +- Intent and flag values are `true`, `false`, or `unknown`. +- Sentiment values are `positive`, `neutral`, `negative`, or `unknown`. +- Evidence must be a short exact quote copied from the input text. +- Use `unknown` when the text alone does not justify a decision. +- Confidence is per field and must be between 0 and 1. +- Do not output explanations, markdown, or additional fields. + +Use the boundaries from `labeling-instructions-v2.md`. In particular, distinguish +requests from personal plans, genuine information questions from request-shaped +commands, direct messages from terminal notices, and expressed wishes from +quoted, future, sarcastic, or received blessings. diff --git a/ModelTraining/ClipboardSemantics/Consensus/adjudication-instructions-v2.md b/ModelTraining/ClipboardSemantics/Consensus/adjudication-instructions-v2.md new file mode 100644 index 0000000..9d49cf0 --- /dev/null +++ b/ModelTraining/ClipboardSemantics/Consensus/adjudication-instructions-v2.md @@ -0,0 +1,56 @@ +# Clipboard semantic evidence adjudication v2 + +Review only the fields listed in `unresolvedFields`. Judge from the text itself; +do not inspect previous model votes, source labels, or another adjudicator. +Apply the product-approved boundaries from `labeling-instructions-v3.md`. + +Return one JSON object per input record: + +```json +{ + "id": "same id", + "recordDisposition": "keep", + "dispositionConfidence": 0.98, + "dispositionEvidence": "send the report", + "resolutions": { + "task": "true", + "ambiguous": "false" + }, + "confidence": { + "task": 0.97, + "ambiguous": 0.94 + }, + "evidence": { + "task": "send the report", + "ambiguous": "by Friday" + } +} +``` + +Requirements: + +- `recordDisposition` must be `keep` or `exclude-device-command`. +- Use `exclude-device-command` only when the text is clearly addressed to a + device, app, search engine, or virtual assistant. Do not use it for an + ordinary request to another person. +- `dispositionConfidence` must be between 0 and 1. +- `dispositionEvidence` must be a short exact quote copied from the text. +- `resolutions`, `confidence`, and `evidence` must contain exactly the fields in + `unresolvedFields`, even when the record is marked for exclusion. +- Intent and flag values are `true`, `false`, or `unknown`. +- Sentiment values are `positive`, `neutral`, `negative`, or `unknown`. +- Field evidence must be a short exact quote copied from the input text. +- Use `unknown` when the text alone does not justify a decision. +- Confidence is per field and must be between 0 and 1. +- Do not output explanations, markdown, or additional fields. + +Critical product decisions: + +- An invitation question is `invitation=true`, `question=false`, and normally + `replyableMessage=true`. +- An explicit self-reminder is `followUpReminder=true`, `task=false`. +- A first-person need implying action is `task=true`. +- A problem is not a complaint unless dissatisfaction, criticism, or objection + is explicitly expressed. +- Generic encouragement or happiness is not a blessing; require an explicit + wish, prayer, congratulation, or conventional blessing. diff --git a/ModelTraining/ClipboardSemantics/Consensus/adjudication-instructions-v3.md b/ModelTraining/ClipboardSemantics/Consensus/adjudication-instructions-v3.md new file mode 100644 index 0000000..c978347 --- /dev/null +++ b/ModelTraining/ClipboardSemantics/Consensus/adjudication-instructions-v3.md @@ -0,0 +1,23 @@ +# Clipboard semantic evidence adjudication v3 + +Review only the fields listed in `unresolvedFields`. Judge from the text itself; +do not inspect previous model votes, source labels, or another adjudicator. +Apply `labeling-instructions-v4.md`. + +Return one JSON object per input record using the exact schema defined in +`adjudication-instructions-v2.md`. + +For compatibility, `recordDisposition` remains `keep` or +`exclude-device-command`. In v3, `exclude-device-command` also covers generic +search, system/account queries, and other clearly virtual-assistant-only text. +It does not cover real-world service requests, named-recipient communication +with content, or private/shared-context interpersonal questions. + +Requirements: + +- Evidence must be a short exact quote copied from the input text. +- `resolutions`, `confidence`, and `evidence` must contain exactly the fields in + `unresolvedFields`. +- Use `unknown` rather than inventing context for low-information fragments. +- Confidence is per field and must be between 0 and 1. +- Do not output explanations, markdown, or additional fields. diff --git a/ModelTraining/ClipboardSemantics/Consensus/adjudication-instructions-v4.md b/ModelTraining/ClipboardSemantics/Consensus/adjudication-instructions-v4.md new file mode 100644 index 0000000..f9a399a --- /dev/null +++ b/ModelTraining/ClipboardSemantics/Consensus/adjudication-instructions-v4.md @@ -0,0 +1,17 @@ +# Clipboard semantic evidence adjudication v4 + +Review only the fields listed in `unresolvedFields`. Judge from the text itself; +do not inspect human anchor labels, previous model votes, source labels, or +another adjudicator. + +Apply `labeling-instructions-v5.md` and every inherited rule from +`labeling-instructions-v4.md`. Return one JSON object per input record using the +exact schema from `adjudication-instructions-v2.md`. + +For compatibility, `recordDisposition` remains `keep` or +`exclude-device-command`; the excluded state also covers generic search, +system/account queries, and clearly virtual-assistant-only commands. + +Evidence must be an exact text quote. Use `unknown` for every affected intent +when a short fragment lacks enough context. Do not output explanations, +markdown, or additional fields. diff --git a/ModelTraining/ClipboardSemantics/Consensus/adjudication-instructions-v5.md b/ModelTraining/ClipboardSemantics/Consensus/adjudication-instructions-v5.md new file mode 100644 index 0000000..29a5cb7 --- /dev/null +++ b/ModelTraining/ClipboardSemantics/Consensus/adjudication-instructions-v5.md @@ -0,0 +1,79 @@ +# Clipboard semantic evidence adjudication v5 + +Review only the fields listed in `unresolvedFields`. Judge from the text itself; +do not inspect source labels, provenance, previous model votes, human anchors, +or another adjudicator. + +Apply `labeling-instructions-v6.md`. Return one JSON object per input record +using the evidence schema from `adjudication-instructions-v2.md`: + +```json +{ + "id": "same id", + "recordDisposition": "keep", + "dispositionConfidence": 0.98, + "dispositionEvidence": "play my workout playlist", + "resolutions": { + "assistantCommand": "true", + "domain": "media" + }, + "confidence": { + "assistantCommand": 0.97, + "domain": 0.95 + }, + "evidence": { + "assistantCommand": "play", + "domain": "workout playlist" + } +} +``` + +Requirements: + +- `recordDisposition` remains `keep` or `exclude-device-command` for file-format + compatibility. Under v6, assistant commands, information queries, and system + notifications are taxonomy records and must be `keep`. +- Use `exclude-device-command` only when replaying a queue explicitly frozen + under v4/v5 exclusion policy. Do not use it in a new v6 queue. +- `dispositionConfidence` and every per-field confidence must be between 0 and + 1. +- `dispositionEvidence` and field evidence must be short exact quotes copied + from the input text. +- `resolutions`, `confidence`, and `evidence` must contain exactly the fields in + `unresolvedFields`, even if a legacy replay record is excluded. +- Intent and flag values are `true`, `false`, or `unknown`. +- `domain` is `finance`, `travel`, `calendar`, `communication`, `media`, + `smartHome`, `shopping`, `dining`, `health`, `weather`, `accountService`, + `generalKnowledge`, or `unknown`. +- `sentiment` is `positive`, `neutral`, `negative`, or `unknown`. +- Use `unknown` when the text alone does not justify a stable decision. A + low-information fragment must not be forced to `false` or into + `generalKnowledge`. +- Do not output explanations, markdown, comments, or additional fields. + +Critical boundary checks: + +- Resolve clearly device-, app-, or assistant-directed direct digital + operations as `assistantCommand=true`; resolve generic lookups as + `informationQuery=true`; resolve machine-authored alerts and status messages + as `systemNotification=true`. +- A request for a real-world service that naturally needs confirmation, such as + booking a taxi, restaurant, hotel, or ticket, remains `task=true` rather than + `assistantCommand=true`. +- Do not infer `task`, `question`, or `replyableMessage` from those records + unless the text independently contains an interpersonal act. +- A named-recipient communication request with actual content is an + interpersonal replyable task. A bare “call Mark” remains unknown for affected + fields when the addressee is unclear. +- Invitation questions are `invitation=true`, `question=false`; explicit + self-reminders are `followUpReminder=true`, `task=false`; first-person needs + implying personal action remain tasks. +- Complaints require explicit dissatisfaction, and blessings require an + explicit wish, prayer, congratulation, or conventional blessing. +- Choose one domain from the operation target. If no primary target can be + established, resolve `domain` as `unknown`. + +`knownLabels` is corpus metadata and must not appear in adjudicator output. +Downstream merging may add an adjudicated field to `knownLabels` only when both +the field value and its evidence pass the configured acceptance gate and the +value is not `unknown`. diff --git a/ModelTraining/ClipboardSemantics/Consensus/labeling-instructions-v2.md b/ModelTraining/ClipboardSemantics/Consensus/labeling-instructions-v2.md new file mode 100644 index 0000000..fcb8f6f --- /dev/null +++ b/ModelTraining/ClipboardSemantics/Consensus/labeling-instructions-v2.md @@ -0,0 +1,70 @@ +# Clipboard semantic consensus labeling v2 + +Prompt version: `clipboard-consensus-v2` + +Label each record independently using only its text. Do not infer missing +conversation history, and do not inspect source labels or other model outputs. + +Return exactly one JSON object per input record: + +```json +{ + "id": "same id", + "labels": { + "task": "false", + "question": "false", + "invitation": "false", + "complaint": "false", + "scheduleNegotiation": "false", + "confirmationDecision": "false", + "followUpReminder": "false", + "blessing": "false", + "replyableMessage": "true" + }, + "sentiment": "neutral", + "ambiguous": false, + "quotedOrMeta": false, + "confidence": 0.96 +} +``` + +Every intent value must be `true`, `false`, or `unknown`. Use `unknown` when +the text alone does not contain enough evidence. Absence of evidence is not +automatically evidence of a negative label. + +## Intent boundaries + +- `task`: another person is explicitly requested or assigned to perform an + action. A personal plan is not a task. +- `question`: a genuine request for information. Rhetorical, quoted, search, + and documentation examples are not questions. +- `invitation`: an invitation to join an event, meeting, visit, meal, or social + activity. +- `complaint`: present dissatisfaction, malfunction, bad service, or an + unresolved problem. Negative sentiment alone is insufficient. +- `scheduleNegotiation`: proposing, changing, comparing, or choosing between + times. A fixed appointment or deadline alone is insufficient. +- `confirmationDecision`: explicit approval, rejection, commitment, or + selection of an option. Acknowledgment alone is insufficient. +- `followUpReminder`: a request to remind, check back, or follow up later or + after a trigger. An ordinary task with a deadline is insufficient. +- `blessing`: the author directly expresses a good wish, congratulation, + prayer, or hope for any recipient, including self or third parties. +- `replyableMessage`: a direct conversational message that naturally invites + a response. Terminal acknowledgments, personal notes, quoted examples, and + factual notices are negative. + +Multi-label combinations are valid. For example, “Could you send the report?” +is `task + question + replyableMessage`. + +## Special cases + +- Set `quotedOrMeta = true` when intent-bearing language is quoted, reported, + searched, documented, requested as a writing example, or discussed rather + than performed. +- Set `ambiguous = true` when material context is missing or multiple + interpretations remain equally plausible. +- Sarcasm, negation, hypothetical future intent, and received thanks must be + interpreted semantically rather than by keyword matching. +- `sentiment` must be `positive`, `neutral`, `negative`, or `unknown`. +- Do not output reasoning, markdown, comments, or additional fields. diff --git a/ModelTraining/ClipboardSemantics/Consensus/labeling-instructions-v3.md b/ModelTraining/ClipboardSemantics/Consensus/labeling-instructions-v3.md new file mode 100644 index 0000000..22152a3 --- /dev/null +++ b/ModelTraining/ClipboardSemantics/Consensus/labeling-instructions-v3.md @@ -0,0 +1,46 @@ +# Clipboard semantic consensus labeling v3 + +Prompt version: `clipboard-consensus-v3` + +Label each record independently using only its text. Do not infer missing +conversation history, and do not inspect source labels or other model outputs. + +Return exactly one JSON object per input record using the schema from +`labeling-instructions-v2.md`. + +## Product-approved boundaries + +These rules override the corresponding v2 boundaries: + +- Exclude commands that are clearly addressed to a device, app, search engine, + or virtual assistant rather than another person. Examples include opening an + inbox, playing music, changing device volume, or showing an account value. + Ordinary requests sent to another person remain in scope. +- `task`: a first-person need that implies an action is a task even when the + recipient is not explicit. A self-reminder is not a task. A device or virtual + assistant command is excluded before intent labeling. +- `question`: an invitation phrased as a question is not a `question`. + Request-shaped commands are also not information questions. +- `invitation`: an invitation phrased as a question is + `invitation=true`, `question=false`, and normally + `replyableMessage=true`. +- `complaint`: require an explicit expression of dissatisfaction, criticism, or + objection. A loss, theft, malfunction, or unresolved problem without + expressed dissatisfaction is not a complaint. +- `followUpReminder`: an explicit self-reminder is + `followUpReminder=true` and `task=false`. +- `blessing`: require an explicit wish, prayer, congratulation, or conventional + blessing. Generic encouragement, happiness for someone, optimism, or “good + luck”-free motivational language is not sufficient. Conventional expressions + such as “生日快乐”, “一路顺风”, “恭喜晋升”, “happy birthday”, and + “congratulations on the promotion” are explicit. + +## Unchanged requirements + +- Every intent value is `true`, `false`, or `unknown`. +- `sentiment` is `positive`, `neutral`, `negative`, or `unknown`. +- Set `quotedOrMeta=true` for quoted, reported, searched, documented, or + example-only intent language. +- Set `ambiguous=true` only when missing context materially prevents a stable + product label. +- Do not output reasoning, markdown, comments, or additional fields. diff --git a/ModelTraining/ClipboardSemantics/Consensus/labeling-instructions-v4.md b/ModelTraining/ClipboardSemantics/Consensus/labeling-instructions-v4.md new file mode 100644 index 0000000..ba0d2d1 --- /dev/null +++ b/ModelTraining/ClipboardSemantics/Consensus/labeling-instructions-v4.md @@ -0,0 +1,64 @@ +# Clipboard semantic consensus labeling v4 + +Prompt version: `clipboard-consensus-v4` + +Label from the text alone. Do not inspect source labels, model votes, or hidden +conversation history. The product owner decisions below override all earlier +versions. + +## Record scope + +Exclude text that is clearly a generic search, device control, app/account +query, alarm/calendar operation, or other virtual-assistant-only command. + +Keep these in scope: + +- a request to perform a real-world service that naturally needs confirmation, + such as booking a taxi; +- a request to communicate with a named recipient when the content to convey is + present; +- a private or shared-context information question that could naturally be sent + to another person, such as asking for a relative's email address. + +If a short fragment does not contain enough evidence to distinguish a human +message from a query or command, keep it unresolved with `ambiguous=true` and +the affected intents set to `unknown`. Do not force it into the excluded or +negative class. + +## Product intent boundaries + +- `replyableMessage=true` when an in-scope interpersonal message naturally + supports a response. Questions, assignments, ongoing decisions, emotional + updates, and outcome sharing can be replyable. +- Terminal acknowledgments and thanks such as “知道了，谢谢” are not + replyable. A passive factual notice that creates no conversational next step + is also not replyable. +- `task=true` for an assigned action, an explicit first-person commitment, or a + first-person need that implies a personal action. A pure status question is + not a task. +- A request to email, text, or otherwise contact a named recipient is a + replyable task when the message content or purpose is included. A bare + command such as “call Mark” is ambiguous without more context. +- `question=true` for any genuine request for information, including an + imperative such as “tell me her email address”. +- A polite interrogative action request such as “Can you send the report?” is + both `task=true` and `question=true`. A question about when an existing task + will happen is `question=true`, `task=false`. +- A request for a recommendation is excluded when it is clearly a generic + assistant/search query rather than an interpersonal request. +- An invitation phrased as a question remains `invitation=true`, + `question=false`, and normally `replyableMessage=true`. +- An explicit self-reminder remains `followUpReminder=true`, `task=false`. +- A complaint still requires explicit dissatisfaction, criticism, or objection. +- A blessing still requires an explicit wish, prayer, congratulation, or + conventional blessing. + +## Output states + +- Intent values are `true`, `false`, or `unknown`. +- `sentiment` is `positive`, `neutral`, `negative`, or `unknown`. +- Use `ambiguous=true` only when missing context materially prevents a stable + product label. +- Use `quotedOrMeta=true` for quoted, reported, searched, documented, or + example-only intent language. +- Do not output explanations, markdown, comments, or additional fields. diff --git a/ModelTraining/ClipboardSemantics/Consensus/labeling-instructions-v5.md b/ModelTraining/ClipboardSemantics/Consensus/labeling-instructions-v5.md new file mode 100644 index 0000000..784cf08 --- /dev/null +++ b/ModelTraining/ClipboardSemantics/Consensus/labeling-instructions-v5.md @@ -0,0 +1,39 @@ +# Clipboard semantic consensus labeling v5 + +Prompt version: `clipboard-consensus-v5` + +Apply every rule in `labeling-instructions-v4.md`, with the following +product-owner clarifications taking precedence. + +## Replyable message clarifications + +- “take your time” is an interpersonal supportive message: + `replyableMessage=true`, `task=false`. +- “知道了，谢谢” is terminal and not replyable. +- “活动规则按当前方案通过” is a passive decision notice and not replyable. +- Sharing a personal outcome such as “事情总算处理完了，结果居然成了” is + replyable even without a direct question. +- A private first-person need such as “I need to set up a new PIN” is a task + but not replyable unless it is addressed to another person. + +## Task clarifications + +- A first-person decision followed by an impersonal consequence is not + automatically an assignment. “我拍板先发布基础版，其他候选停止评估” is + replyable but not a task because it does not directly assign the recipient. +- A decision that explicitly hands off a next action is a task. “我批准退款流程 + 的最终版本，可以签字” is replyable and a task. +- Named-recipient communication with actual content is a replyable task: + “text Sarah that I'll be late” and “send an email to Julie that I can meet + Saturday” are both `replyableMessage=true`, `task=true`. + +## Scope and ambiguity clarifications + +- “tell me what's new” and “my claim status” are generic assistant/system + queries and must be excluded. +- A bare fragment such as “call Mark” does not reveal whether it is an + interpersonal assignment or an assistant command. Keep it unresolved: + `ambiguous=true`, with `replyableMessage`, `task`, and `question` all + `unknown`. +- Apply the same unknown treatment to other low-information fragments rather + than converting unspecified fields to `false`. diff --git a/ModelTraining/ClipboardSemantics/Consensus/labeling-instructions-v6.md b/ModelTraining/ClipboardSemantics/Consensus/labeling-instructions-v6.md new file mode 100644 index 0000000..d91f8f1 --- /dev/null +++ b/ModelTraining/ClipboardSemantics/Consensus/labeling-instructions-v6.md @@ -0,0 +1,201 @@ +# Clipboard semantic consensus labeling v6 + +Prompt version: `clipboard-consensus-v6` + +Label each record independently from its text. Do not inspect source labels, +model votes, provenance, or hidden conversation history. This version keeps the +nine product intents from v5 and adds three routing intents plus one domain +field. Its definitions override earlier instructions when they conflict. + +## Output schema + +Return exactly one JSON object per input record: + +```json +{ + "id": "same id", + "labels": { + "task": "false", + "question": "false", + "invitation": "false", + "complaint": "false", + "scheduleNegotiation": "false", + "confirmationDecision": "false", + "followUpReminder": "false", + "blessing": "false", + "replyableMessage": "true", + "assistantCommand": "false", + "informationQuery": "false", + "systemNotification": "false" + }, + "domain": "communication", + "sentiment": "neutral", + "ambiguous": false, + "quotedOrMeta": false, + "confidence": 0.96 +} +``` + +Every intent value is `true`, `false`, or `unknown`. `domain` is one of the +twelve values below or `unknown`. `sentiment` is `positive`, `neutral`, +`negative`, or `unknown`. Do not output reasoning, comments, markdown, or +additional fields. + +## Nine product intents + +- `task`: a person is assigned or asked to perform an action, or the author + states an explicit personal commitment or need that implies action. A + request for a real-world service that naturally needs confirmation, such as + booking a taxi, restaurant, hotel, or ticket, is also a task. A status + question, self-reminder, passive decision notice, and pure device/app + operation are not tasks. +- `question`: a genuine interpersonal request for information. It includes + imperative requests such as “tell me her email address” when they could + naturally be sent to a person. Invitation questions and clearly generic + assistant/search/account queries are not questions. +- `invitation`: an invitation to join an event, meeting, visit, meal, or social + activity. A question-shaped invitation is normally also + `replyableMessage=true`, but `question=false`. +- `complaint`: explicit present dissatisfaction, criticism, objection, bad + service, or an unresolved problem framed as a complaint. A loss, + malfunction, negative fact, or negative sentiment without expressed + dissatisfaction is insufficient. +- `scheduleNegotiation`: proposing, changing, comparing, or choosing between + times. A fixed appointment, reminder time, or deadline alone is insufficient. +- `confirmationDecision`: explicit approval, rejection, commitment, or + selection of an option. Acknowledgment, receipt confirmation, and passive + status notice alone are insufficient. +- `followUpReminder`: an explicit request to remind, check back, or follow up + later or after a trigger, including a self-reminder. An ordinary task with a + deadline is insufficient; a self-reminder is not a `task`. +- `blessing`: the author directly expresses an explicit wish, prayer, + congratulation, or conventional blessing. Generic encouragement, optimism, + happiness for someone, quoted wishes, and requests to write a blessing are + insufficient. +- `replyableMessage`: an interpersonal message that naturally supports a + response. Questions, assignments, invitations, ongoing decisions, emotional + updates, and outcome sharing may qualify. Terminal acknowledgments or thanks, + private notes, passive factual notices, generic assistant interactions, and + machine notifications do not. + +The v5 product-owner examples remain authoritative: “take your time” is +replyable but not a task; “知道了，谢谢” is terminal; personal outcome sharing +may be replyable; a private first-person need may be a task without being +replyable; and named-recipient communication with actual content is a replyable +task. + +## Three routing intents + +- `assistantCommand`: an instruction to a device, app, service, search engine, + or virtual assistant to perform a direct digital or device operation. This + includes opening or changing app state, alarms and calendar operations, media + playback, smart-home control, and immediate account/app settings. A + real-world service request that naturally needs confirmation remains a + `task`, even when submitted through an assistant. +- `informationQuery`: a generic assistant, search, reference, weather, account, + or service-status lookup that asks for information rather than asking a + person. “tell me what's new”, “my claim status”, and generic recommendation + searches qualify. +- `systemNotification`: machine- or service-generated status, alert, receipt, + security warning, delivery update, or other notification presented to the + user rather than authored as an interpersonal message. + +These three labels replace the old blanket exclusion of assistant-only text. +Keep such records and label them explicitly. They are normally mutually +exclusive, and their clearly assistant/system-scoped records must not become +`task`, `question`, or `replyableMessage` merely because similar words could +occur in human conversation. Real-world bookings remain tasks. A request to +contact a named person with message content is interpersonal, not an +`assistantCommand`; a bare fragment such as “call Mark” remains ambiguous when +addressee and interaction mode cannot be determined. + +## Domains + +Choose the single primary subject or operation target: + +- `finance`: banking, payments, cards, transfers, investments, insurance, or + claims. +- `travel`: transport, routes, tickets, hotels, trips, or reservations other + than restaurant bookings. +- `calendar`: dates, events, meetings, availability, alarms, reminders, or + scheduling. +- `communication`: calls, contacts, messages, email, social communication, or + interpersonal conversation. +- `media`: music, podcasts, radio, video, photos, news playback, or media + discovery. +- `smartHome`: lights, appliances, climate, locks, cameras, or other connected + home devices. +- `shopping`: products, orders, retail delivery, returns, refunds, or + marketplace activity. +- `dining`: restaurants, food, menus, takeaway, restaurant reservations, or + dining service. +- `health`: symptoms, care, medicine, fitness, wellbeing, or medical + appointments. +- `weather`: current conditions, forecasts, temperature, or weather alerts. +- `accountService`: login, identity, profile, PIN/password, subscription, + membership, entitlement, or general service support not better covered above. +- `generalKnowledge`: general facts, definitions, recommendations, and + non-specialized content that does not fit another domain. + +Use the action target to resolve a cross-domain record: “text Sam about the +flight” is `communication`, while “is my flight delayed?” is `travel`. Use +`unknown`, not `generalKnowledge`, when missing context prevents a stable +choice. + +## Unknown, ambiguity, and metadata + +- Use `unknown` only when the text lacks enough evidence for that field. Do not + turn missing annotation or missing context into `false`. +- Use `false` when the field is in scope and the text provides enough evidence + that the intent is absent. +- Set `ambiguous=true` when missing context materially prevents a stable product + label. Set each affected intent and `domain` to `unknown`; unaffected fields + may still be resolved. +- Set `quotedOrMeta=true` when intent-bearing language is quoted, reported, + searched, documented, requested as a writing example, or discussed rather + than performed. +- Multi-label product combinations remain valid, such as + `task + question + replyableMessage` for an interpersonal “Could you send the + report?” + +## `knownLabels` contract for corpus records + +`knownLabels` is ingestion metadata, not part of labeler output. It lists only +the fields a source genuinely annotates after an audited deterministic mapping. +Allowed names are the twelve intent names, `domain`, and `sentiment`. + +- A field in `knownLabels` may train from its resolved value, including an + explicit `false`. +- A field absent from `knownLabels` is `unknown` for training and contributes no + positive or negative loss. +- Source intent names, topic names, or missing columns must never be expanded + into negative labels for the rest of the taxonomy. +- A mapped source label may make only its audited target fields known. + Synthetic data must not claim all labels known merely because the generator + omitted them. +- Consensus or human review may add a field to `knownLabels` only after that + field receives a non-`unknown` decision under this taxonomy. + +## Training-data boundary + +- External data may enter candidate generation only when its commercial-use + rights and required notices are recorded, its immutable revision is pinned, + and it comes from the upstream official `train` split. Upstream validation, + development, test, challenge, and hidden-evaluation records never train. +- When an upstream source publishes only one split explicitly named `train`, it + may supply training candidates but may not supply OSGKeyboard calibration or + evaluation truth. If no official train designation exists, the source waits + in audit and is not locally re-split into eligibility. +- Exact and normalized near-duplicate overlap with any frozen local holdout is a + fatal exclusion. Privacy, credentials, direct contact data, unsafe content, + and unsupported language variants are filtered before labeling. +- Synthetic records are training-only, carry explicit synthetic provenance, + use sample weight at most `0.35`, and may know only the fields guaranteed by + their generation contract. They cannot enter calibration, evaluation, human + gold, or policy-anchor sets; cannot override a conflicting human or licensed + non-synthetic example; and cannot by themselves authorize a new boundary or + deployment threshold. +- Dataset admission means eligibility for the audited candidate queue, not + automatic inclusion in commercial training. Every generated artifact still + requires pinned license evidence, attribution, mapping review, deduplication, + and acceptance gates. diff --git a/ModelTraining/ClipboardSemantics/Consensus/product-policy-anchors-v1.json b/ModelTraining/ClipboardSemantics/Consensus/product-policy-anchors-v1.json new file mode 100644 index 0000000..f17315e --- /dev/null +++ b/ModelTraining/ClipboardSemantics/Consensus/product-policy-anchors-v1.json @@ -0,0 +1,182 @@ +[ + { + "id": "human-anchor-001", + "text": "下一班离开纽约市的火车几点出发", + "language": "zh-Hans", + "expected": {"recordDisposition": "exclude-device-command"} + }, + { + "id": "human-anchor-002", + "text": "告诉我露丝姑妈的电子邮件地址", + "language": "zh-Hans", + "expected": {"recordDisposition": "keep", "replyableMessage": "true", "task": "false", "question": "true", "ambiguous": "false"} + }, + { + "id": "human-anchor-003", + "text": "顺便说一下，这件事我会亲自处理：预约会议室，计划周五之前开始。", + "language": "zh-Hans", + "expected": {"recordDisposition": "keep", "replyableMessage": "true", "task": "true", "question": "false", "ambiguous": "false"} + }, + { + "id": "human-anchor-004", + "text": "在我上班的路上有什么事故吗", + "language": "zh-Hans", + "expected": {"recordDisposition": "exclude-device-command"} + }, + { + "id": "human-anchor-005", + "text": "take your time", + "language": "en", + "expected": {"recordDisposition": "keep", "replyableMessage": "true", "task": "false", "question": "false", "ambiguous": "false"} + }, + { + "id": "human-anchor-006", + "text": "tell me what's new", + "language": "en", + "expected": {"recordDisposition": "exclude-device-command"} + }, + { + "id": "human-anchor-007", + "text": "知道了，这边谢谢。", + "language": "zh-Hans", + "expected": {"recordDisposition": "keep", "replyableMessage": "false", "task": "false", "question": "false", "ambiguous": "false"} + }, + { + "id": "human-anchor-008", + "text": "另外，活动规则按当前方案通过。", + "language": "zh-Hans", + "expected": {"recordDisposition": "keep", "replyableMessage": "false", "task": "false", "question": "false", "ambiguous": "false"} + }, + { + "id": "human-anchor-009", + "text": "我拍板用先发布基础版，其他候选停止评估。", + "language": "zh-Hans", + "expected": {"recordDisposition": "keep", "replyableMessage": "true", "task": "false", "question": "false", "ambiguous": "false"} + }, + { + "id": "human-anchor-010", + "text": "还有一件事，我批准退款流程的最终版本，可以签字。", + "language": "zh-Hans", + "expected": {"recordDisposition": "keep", "replyableMessage": "true", "task": "true", "question": "false", "ambiguous": "false"} + }, + { + "id": "human-anchor-011", + "text": "事情总算处理完了，结果居然成了。", + "language": "zh-Hans", + "expected": {"recordDisposition": "keep", "replyableMessage": "true", "task": "false", "question": "false", "ambiguous": "false"} + }, + { + "id": "human-anchor-012", + "text": "好的，谢谢您的回答。", + "language": "zh-Hans", + "expected": {"recordDisposition": "keep", "replyableMessage": "false", "task": "false", "question": "false", "ambiguous": "false"} + }, + { + "id": "human-anchor-013", + "text": "你能把报告发给我吗？", + "language": "zh-Hans", + "expected": {"recordDisposition": "keep", "replyableMessage": "true", "task": "true", "question": "true", "ambiguous": "false"} + }, + { + "id": "human-anchor-014", + "text": "报告什么时候发给我？", + "language": "zh-Hans", + "expected": {"recordDisposition": "keep", "replyableMessage": "true", "task": "false", "question": "true", "ambiguous": "false"} + }, + { + "id": "human-anchor-015", + "text": "请推荐一家附近评分 4.5 以上的餐馆。", + "language": "zh-Hans", + "expected": {"recordDisposition": "exclude-device-command"} + }, + { + "id": "human-anchor-016", + "text": "I need to set up a new PIN.", + "language": "en", + "expected": {"recordDisposition": "keep", "replyableMessage": "false", "task": "true", "question": "false", "ambiguous": "false"} + }, + { + "id": "human-anchor-017", + "text": "my claim status", + "language": "en", + "expected": {"recordDisposition": "exclude-device-command"} + }, + { + "id": "human-anchor-018", + "text": "send an email to Julie that I can meet her Saturday at 1 p.m.", + "language": "en", + "expected": {"recordDisposition": "keep", "replyableMessage": "true", "task": "true", "question": "false", "ambiguous": "false"} + }, + { + "id": "human-anchor-019", + "text": "call Mark", + "language": "en", + "expected": {"replyableMessage": "unknown", "task": "unknown", "question": "unknown", "ambiguous": "true"} + }, + { + "id": "human-anchor-020", + "text": "text Sarah that I'll be late", + "language": "en", + "expected": {"recordDisposition": "keep", "replyableMessage": "true", "task": "true", "question": "false", "ambiguous": "false"} + }, + { + "id": "human-anchor-021", + "text": "open my inbox", + "language": "en", + "expected": {"recordDisposition": "exclude-device-command"} + }, + { + "id": "human-anchor-022", + "text": "show me my account balance", + "language": "en", + "expected": {"recordDisposition": "exclude-device-command"} + }, + { + "id": "human-anchor-023", + "text": "book me a taxi to the airport", + "language": "en", + "expected": {"recordDisposition": "keep", "replyableMessage": "true", "task": "true", "question": "false", "ambiguous": "false"} + }, + { + "id": "human-anchor-024", + "text": "set an alarm for 4 a.m.", + "language": "en", + "expected": {"recordDisposition": "exclude-device-command"} + }, + { + "id": "human-anchor-025", + "text": "internet", + "language": "en", + "expected": {"replyableMessage": "unknown", "task": "unknown", "question": "unknown", "ambiguous": "true"} + }, + { + "id": "human-anchor-026", + "text": "chennai 45879", + "language": "en", + "expected": {"replyableMessage": "unknown", "task": "unknown", "question": "unknown", "ambiguous": "true"} + }, + { + "id": "human-anchor-027", + "text": "seat assignment", + "language": "en", + "expected": {"replyableMessage": "unknown", "task": "unknown", "question": "unknown", "ambiguous": "true"} + }, + { + "id": "human-anchor-028", + "text": "maximum kilometer to delivery", + "language": "en", + "expected": {"replyableMessage": "unknown", "task": "unknown", "question": "unknown", "ambiguous": "true"} + }, + { + "id": "human-anchor-029", + "text": "事件重复的整个月", + "language": "zh-Hans", + "expected": {"replyableMessage": "unknown", "task": "unknown", "question": "unknown", "ambiguous": "true"} + }, + { + "id": "human-anchor-030", + "text": "意大利面的食谱", + "language": "zh-Hans", + "expected": {"recordDisposition": "exclude-device-command"} + } +] diff --git a/ModelTraining/ClipboardSemantics/blessing-labeling-guidelines.md b/ModelTraining/ClipboardSemantics/blessing-labeling-guidelines.md new file mode 100644 index 0000000..53749d9 --- /dev/null +++ b/ModelTraining/ClipboardSemantics/blessing-labeling-guidelines.md @@ -0,0 +1,70 @@ +# Blessing intent labeling guidelines + +## Definition + +Label `blessing = true` when the author directly expresses a good wish, +congratulation, prayer, or hope for a recipient. The recipient may be the +reader, a named third party, a group, or the author. + +The label is intentionally broad. It includes: + +- Festival, birthday, wedding, anniversary, graduation, promotion, new-job, + housewarming, newborn, and retirement wishes. +- Short congratulations such as `恭喜`, `恭喜发财`, `Congratulations`, and + `Congrats`. +- Health, recovery, travel, safety, exam, competition, career, and general + good-luck wishes. +- Good-night and good-day wishes when they express a desired outcome, such as + `祝你好梦` or `Hope you have a wonderful day`. +- Religious or spiritual prayers directed toward a recipient. +- Wishes expressed for a third person, such as `我衷心祝愿她早日康复`. + +## Negative boundaries + +Label `blessing = false` when the text only: + +- Requests, searches for, or discusses how to write a blessing. +- Thanks someone for a blessing already received. +- Mentions that other people sent blessings. +- Describes a celebration without wishing anyone well. +- Quotes or documents a blessing as an example. +- Greets someone without expressing a desired outcome. +- Gives positive feedback or praise without a wish or congratulation. +- Contains a lexical collision such as a title, name, product, or historical + reference that happens to include a blessing keyword. + +## Context-dependent cases + +Annotators must use surrounding context when available: + +- Sarcastic congratulations are negative unless the product intentionally + treats the surface utterance as reply-worthy congratulations. +- `祝我好运` and equivalent self-directed wishes are positive. +- Conditional or unrealized intent such as `等会儿再祝他生日快乐` is negative + until the text actually expresses the wish. +- A message may be both `blessing` and another intent. For example, a wedding + invitation containing `祝你们幸福` is both an invitation and a blessing. + +## Annotation process + +1. Normalize only invisible whitespace; preserve wording, punctuation, and + emoji for labeling. +2. Two annotators label every golden-set record independently. +3. Disagreements are adjudicated by a third reviewer using this document. +4. Record the boundary category and adjudication reason, not only the binary + label. +5. Keep all evaluation examples isolated from generation prompts, training + sources, and active-learning exports. + +## Release evaluation + +The dedicated blessing benchmark must contain: + +- At least 3,000 Chinese and 1,500 English human-reviewed records. +- Equal positive and hard-negative strata for diagnostic metrics. +- A separate natural-prevalence calibration set for threshold selection. +- At least 100 records for each major positive and negative boundary category. + +The release gate is precision at least 0.95, recall at least 0.85, and F1 at +least 0.90 on the adjudicated benchmark, with no major boundary category below +0.80 F1. diff --git a/ModelTraining/ClipboardSemantics/chinese-corpus-candidate-audit-v1.json b/ModelTraining/ClipboardSemantics/chinese-corpus-candidate-audit-v1.json new file mode 100644 index 0000000..023381a --- /dev/null +++ b/ModelTraining/ClipboardSemantics/chinese-corpus-candidate-audit-v1.json @@ -0,0 +1,235 @@ +{ + "schemaVersion": 1, + "auditVersion": "chinese-corpus-candidate-audit-v1", + "reviewedAt": "2026-08-28", + "taxonomyVersion": "clipboard-consensus-v6", + "scope": "面向简体中文、繁体中文及配套英文边界样本的 taxonomy 语料候选准入审计。", + "policy": { + "admittedMeaning": "准入仅允许来源进入候选抽取、映射和清洗队列，不代表已经进入商业训练集。", + "splitBoundary": "只允许上游官方明确标识的 train 数据产生训练候选；dev、validation、test、challenge、hidden test 和本地冻结 holdout 一律不得训练。", + "commercialBoundary": "导入前必须固定上游 revision、保存许可证及 attribution、确认数据本体而非仅代码受该许可证覆盖，并完成隐私、内容安全和重复检查。", + "knownLabelsBoundary": "每条记录只声明来源实际标注且经过确定性映射审计的 knownLabels；未声明字段保持 unknown，不得作为负例。", + "syntheticBoundary": "合成语料仅用于 train，必须记录生成与上游 provenance，sampleWeight 不得高于 0.35，不得进入校准、评估、人类 gold 或 policy anchors，也不得覆盖冲突的真人或许可非合成样本。", + "holdoutBoundary": "与任一冻结 holdout 精确或归一化近重复的文本全局禁止训练。" + }, + "admittedSources": [ + { + "id": "MASSIVE", + "status": "admitted", + "languages": [ + "zh-CN", + "en-US" + ], + "license": "CC-BY-4.0", + "commercialUse": "允许，须署名并保留许可证与修改说明。", + "url": "https://huggingface.co/datasets/AmazonScience/massive", + "licenseUrl": "https://huggingface.co/datasets/AmazonScience/massive/blob/main/LICENSE", + "trainBoundary": "仅固定 revision 的官方 train split；dev/test 禁止训练。", + "synthetic": false, + "reason": "覆盖中英文助手意图与多领域表达，标签结构适合映射 assistantCommand、informationQuery 和 domain；需避免把源 intent 扩展为其他 taxonomy 字段的负例。" + }, + { + "id": "CrossWOZ", + "status": "admitted", + "languages": [ + "zh-CN" + ], + "license": "Apache-2.0", + "commercialUse": "允许，须保留许可证和 NOTICE 要求。", + "url": "https://github.com/thu-coai/CrossWOZ", + "licenseUrl": "https://github.com/thu-coai/CrossWOZ/blob/master/LICENSE", + "trainBoundary": "仅 data/crosswoz/train.json.zip 的固定官方 revision；val/test 禁止训练。", + "synthetic": false, + "reason": "中文跨领域任务对话可补充 travel、dining、calendar 等边界；只抽取当前轮可独立判断的文本，依赖隐藏对话状态的记录保持 unknown 或丢弃。" + }, + { + "id": "BiToD", + "status": "admitted", + "languages": [ + "zh-CN", + "en" + ], + "license": "Apache-2.0", + "commercialUse": "允许，须保留许可证和 NOTICE 要求。", + "url": "https://github.com/HLTCHKUST/BiToD", + "licenseUrl": "https://github.com/HLTCHKUST/BiToD/blob/main/LICENSE", + "trainBoundary": "仅官方 zh_train/en_train 数据；valid/test 和 cross-lingual evaluation split 禁止训练。", + "synthetic": false, + "reason": "官方提供中英双语和明确切分，可用于 travel、dining、calendar 与 assistant 路由映射；只保留脱离上下文仍有稳定语义的轮次。" + }, + { + "id": "MultiDoGO", + "status": "admitted", + "languages": [ + "en" + ], + "license": "CDLA-Permissive-1.0", + "commercialUse": "允许，按数据许可证保留来源和许可记录。", + "url": "https://github.com/awslabs/multi-domain-goal-oriented-dialogues-dataset", + "licenseUrl": "https://github.com/awslabs/multi-domain-goal-oriented-dialogues-dataset/blob/master/LICENSE.txt", + "trainBoundary": "仅 data/paper_splits 下官方 train.tsv；dev/test 禁止训练。", + "synthetic": false, + "reason": "金融、媒体、软件等域的 turn-level intent 可补充英文边界与跨语言对照；只映射明确源标签，不推断其余 intent。" + }, + { + "id": "Taskmaster-1", + "status": "admitted", + "languages": [ + "en" + ], + "license": "CC-BY-4.0（数据；代码 Apache-2.0）", + "commercialUse": "允许，数据须署名并记录修改。", + "url": "https://github.com/google-research-datasets/Taskmaster/tree/master/TM-1-2019", + "licenseUrl": "https://creativecommons.org/licenses/by/4.0/legalcode", + "trainBoundary": "只使用官方训练用途文件并固定清单；任何官方 evaluation/test 文件禁止训练。", + "synthetic": false, + "reason": "可补充服务预订、计划和确认边界；Wizard/self-dialogue 风格需单独标记 provenance，并过滤依赖多轮上下文的 utterance。" + }, + { + "id": "SNIPS", + "status": "admitted", + "languages": [ + "en" + ], + "license": "CC0-1.0", + "commercialUse": "允许；保留数据集来源、固定 revision 和修改记录。", + "url": "https://github.com/sonos/nlu-benchmark", + "licenseUrl": "https://github.com/sonos/nlu-benchmark/blob/master/LICENSE", + "trainBoundary": "仅 sonos/nlu-benchmark 固定 revision 的 train_*_full.json；validate 文件和第三方镜像不得导入。", + "synthetic": false, + "reason": "可提供 assistantCommand 与媒体、天气、smartHome 等清晰边界；准入不延伸到来源和许可不一致的第三方 SNIPS 镜像。" + }, + { + "id": "MInDS-14", + "status": "admitted", + "languages": [ + "zh-CN", + "en-US", + "en-GB", + "en-AU" + ], + "license": "CC-BY-4.0", + "commercialUse": "允许，须署名并记录转录文本的处理。", + "url": "https://huggingface.co/datasets/PolyAI/minds14", + "licenseUrl": "https://creativecommons.org/licenses/by/4.0/legalcode", + "trainBoundary": "上游每个 config 仅发布 train；这些记录只能产生训练候选，不能充当 OSGKeyboard 校准或评估真值。", + "synthetic": false, + "reason": "中英文银行意图可补充 finance、accountService 和 informationQuery；只使用转录文本，音频不进入本项目。" + }, + { + "id": "GoEmotions", + "status": "admitted", + "languages": [ + "en" + ], + "license": "Apache-2.0", + "commercialUse": "允许，须保留许可证和 NOTICE；导入时保存固定 revision 的许可证证据。", + "url": "https://github.com/google-research/google-research/tree/master/goemotions", + "licenseUrl": "https://github.com/google-research/google-research/blob/master/LICENSE", + "trainBoundary": "仅官方 train.tsv；dev/test 禁止训练。", + "synthetic": false, + "reason": "只用于 sentiment 的已审计映射，不从情绪标签推断 complaint、blessing 或 replyableMessage；Reddit 文本须经过隐私与内容安全过滤。" + }, + { + "id": "ASAP", + "status": "admitted", + "languages": [ + "zh-CN" + ], + "license": "Apache-2.0", + "commercialUse": "允许，须保留许可证和 notices。", + "url": "https://github.com/Meituan-Dianping/ASAP", + "licenseUrl": "https://github.com/Meituan-Dianping/ASAP/blob/master/LICENSE", + "trainBoundary": "仅 data/train.csv；dev/test 禁止训练。", + "synthetic": false, + "reason": "中文餐饮评价可补充 dining、sentiment 与显式 complaint 边界；负面 aspect 或低评分不能自动映射为 complaint。" + }, + { + "id": "Restaurant8k", + "status": "admitted", + "languages": [ + "en" + ], + "license": "CC-BY-4.0", + "commercialUse": "允许，须署名并记录修改。", + "url": "https://github.com/PolyAI-LDN/task-specific-datasets/tree/master/span_extraction/restaurant8k", + "licenseUrl": "https://github.com/PolyAI-LDN/task-specific-datasets/blob/master/LICENSE", + "trainBoundary": "仅 train_0.json 作为完整官方训练集；其下采样副本不得重复导入，test.json 禁止训练。", + "synthetic": false, + "reason": "可补充 dining domain 和餐饮查询边界；span 标注只使经审计映射的字段 known，不使十二个 intent 全部已知。" + }, + { + "id": "FormosaNLU-Synth-v1", + "status": "admitted", + "languages": [ + "zh-TW" + ], + "license": "CC-BY-4.0（合成数据及 MASSIVE zh-TW seed）", + "commercialUse": "允许，须同时署名 MASSIVE 与 FormosaNLU Synth，并说明合成和过滤修改。", + "url": "https://huggingface.co/datasets/steven0226/formosa-nlu-synth-v1", + "licenseUrl": "https://huggingface.co/datasets/steven0226/formosa-nlu-synth-v1/blob/main/LICENSE", + "trainBoundary": "仅固定 release manifest 对应的 train；不得作为校准、评估或人工 gold。", + "synthetic": true, + "maximumSampleWeight": 0.35, + "reason": "可补充繁体中文和台湾表达，但必须保留双重 attribution、生成 provenance 与 synthetic 标记；只声明生成契约保证的 knownLabels。" + } + ], + "quarantinedSources": [ + { + "id": "BANKING77", + "status": "quarantined", + "license": "CC-BY-4.0", + "commercialUse": "许可证允许商业使用，但当前不准进入训练。", + "url": "https://huggingface.co/datasets/PolyAI/banking77", + "reason": "与 MInDS-14 的 finance/accountService 覆盖高度重叠，且大量短查询会放大 informationQuery 与低信息 fragment 偏差；待完成去重、源域平衡和独立边界审计。" + }, + { + "id": "ABCD", + "status": "quarantined", + "license": "MIT", + "commercialUse": "许可证允许商业使用，但当前不准进入训练。", + "url": "https://github.com/asappresearch/abcd", + "reason": "虚构零售客服的多轮 action dialogue 强依赖角色与上下文，直接抽取会污染单条 clipboard 的 task、confirmationDecision 和 systemNotification 边界。" + }, + { + "id": "MultiWOZ", + "status": "quarantined", + "license": "MIT（官方仓库）", + "commercialUse": "许可证允许商业使用，但当前不准进入训练。", + "url": "https://github.com/budzianowski/multiwoz", + "reason": "版本多、历史标注错误和修订差异明显，且多轮状态依赖强；在固定唯一版本、验证官方 train 和完成 turn-level 质量审计前隔离。" + }, + { + "id": "CLINC150", + "status": "quarantined", + "license": "CC-BY-4.0（UCI 官方分发）", + "commercialUse": "许可证允许商业使用，但当前不准进入 v6 新训练。", + "url": "https://archive.ics.uci.edu/dataset/570/clinc150", + "reason": "以虚拟助手和 OOS 检测为目标，短命令分布会主导新增路由标签；历史实验及 holdout 已使用相关文本，须先完成全局泄漏审计和旧产物隔离。" + }, + { + "id": "CFPB", + "status": "quarantined", + "license": "CC0-1.0", + "commercialUse": "许可证允许商业使用，但真实投诉叙述当前不准进入训练。", + "url": "https://www.consumerfinance.gov/data-research/consumer-complaints/", + "reason": "真实消费者投诉可能包含敏感财务、身份和叙事隐私信息，且 2026 年官方停止主动发布投诉 narratives；即使是 CC0，也需法律、隐私和历史快照来源审查。" + }, + { + "id": "openclaw-zh-greetings", + "status": "quarantined", + "license": "MIT（数据卡声明）", + "commercialUse": "许可声明表面允许，但当前证据不足以批准商业训练。", + "url": "https://huggingface.co/datasets/trytax/openclaw-zh-greetings", + "reason": "小型非官方示例集缺少稳定上游、版本化生成过程和逐条权利链；仓库/软件的 MIT 许可不能替代对数据文本本体的 provenance 审计。" + }, + { + "id": "LCCC", + "status": "quarantined", + "license": "MIT 仓库标识与官方“仅限科研用途”说明并存", + "commercialUse": "禁止用于当前商业训练。", + "url": "https://github.com/thu-coai/CDial-GPT", + "reason": "官方 README 明确限定科研用途，语料来自抓取的微博对话，缺少完整底层内容权利与隐私链；只能在隔离研究队列中审阅。" + } + ] +} diff --git a/ModelTraining/ClipboardSemantics/corpus-registry-sources.json b/ModelTraining/ClipboardSemantics/corpus-registry-sources.json new file mode 100644 index 0000000..fba81dc --- /dev/null +++ b/ModelTraining/ClipboardSemantics/corpus-registry-sources.json @@ -0,0 +1,302 @@ +{ + "schemaVersion": 2, + "policy": "Only license-safe sources may produce train candidates. Any text seen in calibration or evaluation data is globally barred from training.", + "intentLabels": [ + "task", + "question", + "invitation", + "complaint", + "scheduleNegotiation", + "confirmationDecision", + "followUpReminder", + "blessing", + "replyableMessage", + "assistantCommand", + "informationQuery", + "systemNotification" + ], + "domains": [ + "finance", + "travel", + "calendar", + "communication", + "media", + "smartHome", + "shopping", + "dining", + "health", + "weather", + "accountService", + "generalKnowledge" + ], + "legacyFullyKnownIntentLabels": [ + "task", + "question", + "invitation", + "complaint", + "scheduleNegotiation", + "confirmationDecision", + "followUpReminder", + "blessing", + "replyableMessage" + ], + "sources": [ + { + "id": "historical-nine-model-v1", + "path": "${REPO_ROOT}/ModelTraining/ClipboardSemantics/Generated/historical-nine-model-v1.jsonl", + "license": "OSGKeyboard project license", + "defaultUse": "by-split", + "sourceType": "project-generated", + "allIntentLabelsKnown": true, + "knownIntentLabels": ["task", "question", "invitation", "complaint", "scheduleNegotiation", "confirmationDecision", "followUpReminder", "blessing", "replyableMessage"], + "sentimentKnown": true, + "weight": 1.0, + "required": true + }, + { + "id": "current-product-corpus", + "path": "${REPO_ROOT}/ModelTraining/ClipboardSemantics/clipboard_semantic_corpus.jsonl", + "license": "OSGKeyboard project license", + "defaultUse": "by-split", + "sourceType": "project-generated", + "allIntentLabelsKnown": true, + "knownIntentLabels": ["task", "question", "invitation", "complaint", "scheduleNegotiation", "confirmationDecision", "followUpReminder", "blessing", "replyableMessage"], + "sentimentKnown": true, + "weight": 1.0, + "required": true + }, + { + "id": "product-corpus-16x", + "path": "${REPO_ROOT}/ModelTraining/ClipboardSemantics/Generated/product-corpus-16x.jsonl", + "license": "OSGKeyboard project license", + "defaultUse": "by-split", + "sourceType": "project-generated", + "allIntentLabelsKnown": true, + "knownIntentLabels": ["task", "question", "invitation", "complaint", "scheduleNegotiation", "confirmationDecision", "followUpReminder", "blessing", "replyableMessage"], + "sentimentKnown": true, + "weight": 1.0, + "required": true + }, + { + "id": "licensed-open-training", + "path": "${REPO_ROOT}/ModelTraining/ClipboardSemantics/open-training-corpus.jsonl", + "license": "mixed per-record commercial licenses", + "defaultUse": "train", + "sourceType": "licensed-open-data", + "allIntentLabelsKnown": false, + "sentimentKnown": false, + "weight": 1.0, + "required": true + }, + { + "id": "v6-boundary-synthetic", + "path": "${REPO_ROOT}/ModelTraining/ClipboardSemantics/v6-boundary-training-supplement.jsonl", + "license": "OSGKeyboard project license", + "defaultUse": "train", + "sourceType": "project-generated-synthetic", + "allIntentLabelsKnown": false, + "sentimentKnown": false, + "weight": 1.0, + "required": true + }, + { + "id": "v6-migration-tier-a", + "path": "${REPO_ROOT}/ModelTraining/ClipboardSemantics/CorpusRegistry/Labels/V6Migration/tier-a.jsonl", + "license": "OSGKeyboard project license", + "defaultUse": "train", + "sourceType": "model-consensus", + "allIntentLabelsKnown": false, + "sentimentKnown": false, + "weight": 1.0, + "required": true + }, + { + "id": "v6-migration-tier-b", + "path": "${REPO_ROOT}/ModelTraining/ClipboardSemantics/CorpusRegistry/Labels/V6Migration/tier-b.jsonl", + "license": "OSGKeyboard project license", + "defaultUse": "train", + "sourceType": "model-consensus", + "allIntentLabelsKnown": false, + "sentimentKnown": false, + "weight": 1.0, + "required": true + }, + { + "id": "v6-migration-adjudicated", + "path": "${REPO_ROOT}/ModelTraining/ClipboardSemantics/CorpusRegistry/Labels/V6Migration/adjudicated.jsonl", + "license": "OSGKeyboard project license", + "defaultUse": "train", + "sourceType": "model-adjudicated", + "allIntentLabelsKnown": false, + "sentimentKnown": false, + "weight": 1.0, + "required": true + }, + { + "id": "v6-blind-evaluation", + "path": "${REPO_ROOT}/ModelTraining/ClipboardSemantics/v6-blind-evaluation-corpus.jsonl", + "license": "OSGKeyboard project license", + "defaultUse": "by-split", + "sourceType": "frozen-holdout", + "allIntentLabelsKnown": false, + "sentimentKnown": false, + "weight": 1.0, + "required": true + }, + { + "id": "blessing-synthetic-100k", + "path": "${REPO_ROOT}/ModelTraining/ClipboardSemantics/Generated/blessing-synthetic-100k.jsonl", + "license": "OSGKeyboard project license", + "defaultUse": "train", + "sourceType": "project-generated", + "allIntentLabelsKnown": false, + "sentimentKnown": false, + "weight": 0.35, + "required": true + }, + { + "id": "consensus-silver-v1", + "path": "${REPO_ROOT}/ModelTraining/ClipboardSemantics/Consensus/consensus-silver.jsonl", + "license": "Apache-2.0", + "defaultUse": "by-split", + "sourceType": "model-consensus", + "allIntentLabelsKnown": false, + "sentimentKnown": false, + "weight": 1.0, + "required": false + }, + { + "id": "random-holdout", + "path": "${REPO_ROOT}/ModelTraining/ClipboardSemantics/random-holdout-corpus.jsonl", + "license": "OSGKeyboard project license", + "defaultUse": "evaluation-only", + "sourceType": "frozen-holdout", + "allIntentLabelsKnown": true, + "knownIntentLabels": ["task", "question", "invitation", "complaint", "scheduleNegotiation", "confirmationDecision", "followUpReminder", "blessing", "replyableMessage"], + "sentimentKnown": true, + "weight": 1.0, + "required": true + }, + { + "id": "fresh-metric-holdout", + "path": "${REPO_ROOT}/ModelTraining/ClipboardSemantics/fresh-metric-holdout-corpus.jsonl", + "license": "OSGKeyboard project license", + "defaultUse": "evaluation-only", + "sourceType": "frozen-holdout", + "allIntentLabelsKnown": true, + "knownIntentLabels": ["task", "question", "invitation", "complaint", "scheduleNegotiation", "confirmationDecision", "followUpReminder", "blessing", "replyableMessage"], + "sentimentKnown": true, + "weight": 1.0, + "required": true + }, + { + "id": "targeted-confirmation-holdout", + "path": "${REPO_ROOT}/ModelTraining/ClipboardSemantics/targeted-confirmation-holdout-corpus.jsonl", + "license": "OSGKeyboard project license", + "defaultUse": "evaluation-only", + "sourceType": "frozen-holdout", + "allIntentLabelsKnown": true, + "knownIntentLabels": ["task", "question", "invitation", "complaint", "scheduleNegotiation", "confirmationDecision", "followUpReminder", "blessing", "replyableMessage"], + "sentimentKnown": true, + "weight": 1.0, + "required": true + }, + { + "id": "targeted-final-holdout", + "path": "${REPO_ROOT}/ModelTraining/ClipboardSemantics/targeted-final-holdout-corpus.jsonl", + "license": "OSGKeyboard project license", + "defaultUse": "evaluation-only", + "sourceType": "frozen-holdout", + "allIntentLabelsKnown": true, + "knownIntentLabels": ["task", "question", "invitation", "complaint", "scheduleNegotiation", "confirmationDecision", "followUpReminder", "blessing", "replyableMessage"], + "sentimentKnown": true, + "weight": 1.0, + "required": true + }, + { + "id": "targeted-release-holdout", + "path": "${REPO_ROOT}/ModelTraining/ClipboardSemantics/targeted-release-holdout-corpus.jsonl", + "license": "OSGKeyboard project license", + "defaultUse": "evaluation-only", + "sourceType": "frozen-holdout", + "allIntentLabelsKnown": true, + "knownIntentLabels": ["task", "question", "invitation", "complaint", "scheduleNegotiation", "confirmationDecision", "followUpReminder", "blessing", "replyableMessage"], + "sentimentKnown": true, + "weight": 1.0, + "required": true + }, + { + "id": "online-real-holdout", + "path": "${REPO_ROOT}/ModelTraining/ClipboardSemantics/online-real-holdout-corpus.jsonl", + "license": "evaluation-only mixed provenance", + "defaultUse": "evaluation-only", + "sourceType": "frozen-holdout", + "allIntentLabelsKnown": true, + "knownIntentLabels": ["task", "question", "invitation", "complaint", "scheduleNegotiation", "confirmationDecision", "followUpReminder", "blessing", "replyableMessage"], + "sentimentKnown": true, + "weight": 1.0, + "required": true + }, + { + "id": "comprehensive-online-holdout", + "path": "${REPO_ROOT}/ModelTraining/ClipboardSemantics/comprehensive-online-holdout-corpus.jsonl", + "license": "evaluation-only mixed provenance", + "defaultUse": "evaluation-only", + "sourceType": "frozen-holdout", + "allIntentLabelsKnown": true, + "knownIntentLabels": ["task", "question", "invitation", "complaint", "scheduleNegotiation", "confirmationDecision", "followUpReminder", "blessing", "replyableMessage"], + "sentimentKnown": true, + "weight": 1.0, + "required": true + }, + { + "id": "blessing-benchmark-review", + "path": "${REPO_ROOT}/ModelTraining/ClipboardSemantics/BlessingBenchmark/review-queue.jsonl", + "license": "evaluation-only mixed provenance", + "defaultUse": "evaluation-only", + "sourceType": "blind-review-benchmark", + "allIntentLabelsKnown": false, + "sentimentKnown": false, + "weight": 1.0, + "sensitive": true, + "required": true + }, + { + "id": "blessing-unseen-scenarios", + "path": "${REPO_ROOT}/OSGKeyboardTests/Fixtures/BlessingUnseenScenarios.json", + "license": "OSGKeyboard project license", + "defaultUse": "evaluation-only", + "sourceType": "frozen-holdout", + "allIntentLabelsKnown": false, + "sentimentKnown": false, + "weight": 1.0, + "required": false + } + ], + "excludedSources": [ + { + "id": "LCCC", + "reason": "Research-only terms and incomplete content-rights/privacy chain. It must not influence production corpus generation or training." + }, + { + "id": "DailyDialog-EmpatheticDialogues-Switchboard", + "reason": "Non-commercial license restrictions." + }, + { + "id": "CPED", + "reason": "Repository license does not establish commercial rights to the underlying television dialogue." + }, + { + "id": "Tianji-Wishes-Birthday-Quotes", + "reason": "No sufficiently clear per-record source or generation-rights chain." + }, + { + "id": "CFPB", + "reason": "No official train split; privacy-sensitive narratives overlap the frozen evaluation provenance." + }, + { + "id": "CLINC150", + "reason": "Isolated from product training because its assistant taxonomy is not product-policy compatible." + } + ] +} diff --git a/ModelTraining/ClipboardSemantics/v6-blind-evaluation-report.json b/ModelTraining/ClipboardSemantics/v6-blind-evaluation-report.json new file mode 100644 index 0000000..99baeff --- /dev/null +++ b/ModelTraining/ClipboardSemantics/v6-blind-evaluation-report.json @@ -0,0 +1,99 @@ +{ + "domains": { + "accountService": 14, + "calendar": 8, + "communication": 14, + "dining": 10, + "finance": 5, + "generalKnowledge": 10, + "health": 5, + "media": 1, + "shopping": 6, + "smartHome": 3, + "travel": 6, + "unknown": 38 + }, + "humanCoverage": { + "excludeDeviceCommandRecords": 11, + "overriddenRecords": 49, + "overridesByField": { + "ambiguous": 49, + "question": 49, + "replyableMessage": 49, + "task": 49 + }, + "records": 60 + }, + "inputSHA256": { + "holdout": "95ac25ff01113de05eb3c6b9dd9a7b38efa3fa1d434439097f604bb9f2de1a9b", + "humanLabels": "d6f5897227991a59420d2892e0d946c4fb4125d1cab37f4137deb3e1f3d1f688", + "primary:composer": "6221c8f6fa7a2fed7bb8c8d93f70eee1c1017f6a1dbf75487f90cb9789165644", + "primary:grok": "940ede21bfaaf30db36e931135cbb196240cdf96ff1eff36dbf490156738cb94", + "primary:luna": "63c525c05f146546603610bcb04d229d8ee415259aa5a7bdacb277715aef289b", + "reviewQueue": "7dd224c20dd1ee1da4c8ead6f19f99264fcb5a868e3588432b77ce233c6f500e", + "reviewer:claude": "fe34fea28628f5f84b41c838bfc59591feabd5851a21ba8e08d828811f637b26", + "reviewer:sol": "78f869bcd685cea9c579d488e5f588f21d67a1db1ca4fdf13d3ccb43713b1b9d" + }, + "knownByField": { + "assistantCommand": 111, + "blessing": 120, + "complaint": 118, + "confirmationDecision": 113, + "domain": 82, + "followUpReminder": 120, + "informationQuery": 115, + "invitation": 120, + "question": 107, + "replyableMessage": 100, + "scheduleNegotiation": 112, + "sentiment": 107, + "systemNotification": 118, + "task": 104 + }, + "languages": { + "en": 60, + "zh-Hans": 60 + }, + "outputSHA256": "c800922eb24e9681e34d16d4328ffbdbfc3ba8a0e6b2538ea670d85dc3cf6b03", + "positiveByIntent": { + "assistantCommand": 6, + "blessing": 0, + "complaint": 4, + "confirmationDecision": 7, + "followUpReminder": 2, + "informationQuery": 10, + "invitation": 6, + "question": 23, + "replyableMessage": 63, + "scheduleNegotiation": 2, + "systemNotification": 8, + "task": 25 + }, + "recordCount": 120, + "schemaVersion": 1, + "sourceDataset": "product-policy-blind-holdout-v1", + "sourceLicense": "OSGKeyboard project license", + "sourceRevision": "v1", + "splits": { + "golden": 40, + "test": 40, + "validation": 40 + }, + "unresolvedByField": { + "ambiguous": 2, + "assistantCommand": 9, + "blessing": 0, + "complaint": 2, + "confirmationDecision": 7, + "domain": 38, + "followUpReminder": 0, + "informationQuery": 5, + "invitation": 0, + "question": 13, + "replyableMessage": 20, + "scheduleNegotiation": 8, + "sentiment": 13, + "systemNotification": 2, + "task": 16 + } +} diff --git a/ModelTraining/ClipboardSemantics/v6-boundary-training-supplement-summary.json b/ModelTraining/ClipboardSemantics/v6-boundary-training-supplement-summary.json new file mode 100644 index 0000000..2b42077 --- /dev/null +++ b/ModelTraining/ClipboardSemantics/v6-boundary-training-supplement-summary.json @@ -0,0 +1,121 @@ +{ + "corpusSHA256": "5b435a564e5dae111898f2ac743c9fa6cc862525e9c0d45c7358069b7567297f", + "counts": { + "byBoundaryTargetAndLanguage": { + "en": { + "assistantCommand": 1000, + "informationQuery": 1000, + "question": 200, + "replyableMessage": 200, + "systemNotification": 1000, + "task": 200 + }, + "zh-Hans": { + "assistantCommand": 1000, + "informationQuery": 1000, + "question": 200, + "replyableMessage": 200, + "systemNotification": 1000, + "task": 200 + } + }, + "byDomain": { + "accountService": 600, + "calendar": 600, + "communication": 600, + "dining": 600, + "finance": 600, + "generalKnowledge": 600, + "health": 600, + "media": 600, + "shopping": 600, + "smartHome": 600, + "travel": 600, + "weather": 600 + }, + "byIntent": { + "assistantCommand": 2000, + "informationQuery": 2000, + "question": 400, + "replyableMessage": 800, + "systemNotification": 2000, + "task": 400 + }, + "byIntentAndLanguage": { + "en": { + "assistantCommand": 1000, + "informationQuery": 1000, + "question": 200, + "replyableMessage": 400, + "systemNotification": 1000, + "task": 200 + }, + "zh-Hans": { + "assistantCommand": 1000, + "informationQuery": 1000, + "question": 200, + "replyableMessage": 400, + "systemNotification": 1000, + "task": 200 + } + }, + "byLanguage": { + "en": 3600, + "zh-Hans": 3600 + }, + "byTemplateFamily": { + "v6_assistantCommand_en_template_1": 253, + "v6_assistantCommand_en_template_2": 233, + "v6_assistantCommand_en_template_3": 250, + "v6_assistantCommand_en_template_4": 264, + "v6_assistantCommand_zh-Hans_template_1": 247, + "v6_assistantCommand_zh-Hans_template_2": 250, + "v6_assistantCommand_zh-Hans_template_3": 253, + "v6_assistantCommand_zh-Hans_template_4": 250, + "v6_informationQuery_en_template_1": 243, + "v6_informationQuery_en_template_2": 240, + "v6_informationQuery_en_template_3": 256, + "v6_informationQuery_en_template_4": 261, + "v6_informationQuery_zh-Hans_template_1": 246, + "v6_informationQuery_zh-Hans_template_2": 254, + "v6_informationQuery_zh-Hans_template_3": 245, + "v6_informationQuery_zh-Hans_template_4": 255, + "v6_question_en_template_1": 52, + "v6_question_en_template_2": 54, + "v6_question_en_template_3": 44, + "v6_question_en_template_4": 50, + "v6_question_zh-Hans_template_1": 54, + "v6_question_zh-Hans_template_2": 42, + "v6_question_zh-Hans_template_3": 46, + "v6_question_zh-Hans_template_4": 58, + "v6_replyableMessage_en_template_1": 57, + "v6_replyableMessage_en_template_2": 40, + "v6_replyableMessage_en_template_3": 53, + "v6_replyableMessage_en_template_4": 50, + "v6_replyableMessage_zh-Hans_template_1": 51, + "v6_replyableMessage_zh-Hans_template_2": 46, + "v6_replyableMessage_zh-Hans_template_3": 50, + "v6_replyableMessage_zh-Hans_template_4": 53, + "v6_systemNotification_en_template_1": 243, + "v6_systemNotification_en_template_2": 250, + "v6_systemNotification_en_template_3": 261, + "v6_systemNotification_en_template_4": 246, + "v6_systemNotification_zh-Hans_template_1": 252, + "v6_systemNotification_zh-Hans_template_2": 238, + "v6_systemNotification_zh-Hans_template_3": 245, + "v6_systemNotification_zh-Hans_template_4": 265, + "v6_task_en_template_1": 38, + "v6_task_en_template_2": 51, + "v6_task_en_template_3": 60, + "v6_task_en_template_4": 51, + "v6_task_zh-Hans_template_1": 41, + "v6_task_zh-Hans_template_2": 52, + "v6_task_zh-Hans_template_3": 48, + "v6_task_zh-Hans_template_4": 59 + } + }, + "excludedHoldoutOverlap": 0, + "recordCount": 7200, + "schemaVersion": 1, + "sourceRevision": "v6-boundary-templates-1" +} diff --git a/ModelTraining/ClipboardSemantics/v6-release-gate-report.json b/ModelTraining/ClipboardSemantics/v6-release-gate-report.json new file mode 100644 index 0000000..8d7953f --- /dev/null +++ b/ModelTraining/ClipboardSemantics/v6-release-gate-report.json @@ -0,0 +1,111 @@ +{ + "allGatesPassed": false, + "candidate": "taxonomy-v6-expanded-maxEnt", + "corpus": { + "blindLabelPolicy": "Product-owner task/question/replyable labels are used for the first 60 records; other v6 fields require per-field multi-model consensus. Unknown fields are excluded.", + "blindRecords": 120, + "candidateCorpusRecords": 273746, + "humanLabeledBlindRecords": 60, + "registryCanonicalRecords": 327337, + "trainCandidates": 273626 + }, + "currentModelBlindBaseline": { + "accuracy": 0.8845, + "f1": 0.2326, + "precision": 0.2578, + "recall": 0.2974 + }, + "deploymentMode": "shadow/display", + "domainGoldenMetrics": { + "accuracy": 0.3704, + "macroF1": 0.3282, + "total": 27 + }, + "gates": { + "domainMacroF1": { + "actual": 0.3282, + "passed": false, + "required": 0.85 + }, + "evaluationIsolation": { + "exactOverlapCount": 0, + "passed": true, + "trainBarredByCalibrationCount": 1, + "trainBarredByEvaluationCount": 0 + }, + "newIntentMacroF1": { + "actual": 0.5915, + "passed": false, + "required": 0.9 + }, + "newIntentMinimumPrecision": { + "actual": 0.25, + "passed": false, + "required": 0.95 + }, + "oldNineNoRegression": { + "currentBlindMacroF1": 0.2326, + "passed": true, + "reason": "The candidate is additive and the production manifest and nine deployed model files were not replaced." + }, + "runtimePerformance": { + "actual": { + "coldLoadMilliseconds": 49.2974, + "modelBytes": 570083, + "peakRSSDeltaBytes": 25001984, + "warmP95Milliseconds": 0.2498 + }, + "budgets": { + "coldLoadMilliseconds": 100, + "modelBytes": 2000000, + "peakRSSDeltaBytes": 41943040, + "warmP95Milliseconds": 1 + }, + "passed": true + } + }, + "limitations": [ + "Only 60 of 120 product blind records have product-owner labels.", + "The remaining fields are high-confidence model consensus, not human gold.", + "Per-language calibration has too few positive blind examples.", + "The current model has no heads for the three new intents or domain." + ], + "newIntentGoldenMetrics": { + "assistantCommand": { + "accuracy": 0.9722, + "f1": 0.8, + "falseNegative": 0, + "falsePositive": 1, + "precision": 0.6667, + "recall": 1, + "total": 36, + "trueNegative": 33, + "truePositive": 2 + }, + "informationQuery": { + "accuracy": 0.7692, + "f1": 0.3077, + "falseNegative": 3, + "falsePositive": 6, + "precision": 0.25, + "recall": 0.4, + "total": 39, + "trueNegative": 28, + "truePositive": 2 + }, + "systemNotification": { + "accuracy": 0.95, + "f1": 0.6667, + "falseNegative": 2, + "falsePositive": 0, + "precision": 1, + "recall": 0.5, + "total": 40, + "trueNegative": 36, + "truePositive": 2 + } + }, + "productionManifestSHA256": "52cf3916ea18325fc1f4b51f3c1ee2d9f73c5633b227d3355606f5a1a1ea0c1c", + "releaseDecision": "keep-current-model", + "schemaVersion": 1 +} diff --git a/Scripts/clipboard_semantics/adjudicate_consensus_conflicts.py b/Scripts/clipboard_semantics/adjudicate_consensus_conflicts.py new file mode 100644 index 0000000..ec5fcda --- /dev/null +++ b/Scripts/clipboard_semantics/adjudicate_consensus_conflicts.py @@ -0,0 +1,563 @@ +#!/usr/bin/env python3 +"""Prepare and merge evidence-backed AI adjudication of consensus conflicts.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import unicodedata +from collections import Counter +from pathlib import Path + +from merge_consensus_labels_v2 import DOMAINS, INTENT_LABELS, stable_split + +FLAG_FIELDS = {"ambiguous", "quotedOrMeta"} +LABEL_STATES = {"true", "false", "unknown"} +SENTIMENT_STATES = {"positive", "neutral", "negative", "unknown"} +RECORD_DISPOSITIONS = {"keep", "exclude-device-command"} +PRODUCT_POLICY_FIELDS = ( + "task", + "question", + "invitation", + "complaint", + "followUpReminder", + "blessing", + "replyableMessage", + "assistantCommand", + "informationQuery", + "systemNotification", + "domain", +) +PROMPT_VERSION = "clipboard-adjudication-v5" + + +def normalize(value: str) -> str: + return " ".join(unicodedata.normalize("NFKC", value).casefold().split()) + + +def read_json_lines(path: Path) -> list[dict]: + return [ + json.loads(line) + for line in path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + + +def write_json_lines(path: Path, records: list[dict]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as handle: + for record in records: + handle.write( + json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n" + ) + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def prepare(arguments: argparse.Namespace) -> dict: + conflicts = read_json_lines(arguments.conflicts) + include_policy_fields = getattr( + arguments, + "include_product_policy_fields", + False, + ) + queue = [ + { + "id": record["id"], + "text": record["text"], + "language": record["language"], + "unresolvedFields": list( + dict.fromkeys( + ( + *PRODUCT_POLICY_FIELDS, + *record["unresolvedFields"], + ) + if include_policy_fields + else record["unresolvedFields"] + ) + ), + } + for record in conflicts + ] + if len({record["id"] for record in queue}) != len(queue): + raise ValueError("Conflict queue contains duplicate ids") + write_json_lines(arguments.queue, queue) + arguments.chunk_directory.mkdir(parents=True, exist_ok=True) + chunks = [] + for start in range(0, len(queue), arguments.chunk_size): + index = len(chunks) + 1 + path = arguments.chunk_directory / f"chunk-{index:03d}.jsonl" + values = queue[start : start + arguments.chunk_size] + write_json_lines(path, values) + chunks.append( + { + "path": str(path), + "records": len(values), + "sha256": sha256_file(path), + } + ) + report = { + "schemaVersion": 1, + "promptVersion": PROMPT_VERSION, + "queueCount": len(queue), + "queueSHA256": sha256_file(arguments.queue), + "chunkSize": arguments.chunk_size, + "chunkCount": len(chunks), + "includesProductPolicyFields": include_policy_fields, + "productPolicyFields": ( + list(PRODUCT_POLICY_FIELDS) if include_policy_fields else [] + ), + "chunks": chunks, + } + arguments.report.write_text( + json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return report + + +def valid_state(field: str, value: object) -> bool: + if field == "sentiment": + return value in SENTIMENT_STATES + if field == "domain": + return value in {*DOMAINS, "unknown"} + return value in LABEL_STATES + + +def validate_adjudication(record: dict, queue_record: dict) -> dict: + identifier = queue_record["id"] + if record.get("id") != identifier: + raise ValueError(f"Unexpected adjudication id: {record.get('id')}") + disposition = record.get("recordDisposition") + if disposition not in RECORD_DISPOSITIONS: + raise ValueError(f"Invalid record disposition: {identifier}") + disposition_confidence = record.get("dispositionConfidence") + if ( + not isinstance(disposition_confidence, (int, float)) + or not 0 <= disposition_confidence <= 1 + ): + raise ValueError(f"Invalid disposition confidence: {identifier}") + disposition_evidence = record.get("dispositionEvidence") + text = normalize(queue_record["text"]) + if ( + not isinstance(disposition_evidence, str) + or not normalize(disposition_evidence) + or normalize(disposition_evidence) not in text + ): + raise ValueError(f"Invalid disposition evidence: {identifier}") + expected = set(queue_record["unresolvedFields"]) + for key in ("resolutions", "confidence", "evidence"): + if not isinstance(record.get(key), dict) or set(record[key]) != expected: + raise ValueError(f"{key} fields do not match unresolved fields: {identifier}") + for field in expected: + if not valid_state(field, record["resolutions"][field]): + raise ValueError(f"Invalid resolution for {identifier}/{field}") + confidence = record["confidence"][field] + if not isinstance(confidence, (int, float)) or not 0 <= confidence <= 1: + raise ValueError(f"Invalid confidence for {identifier}/{field}") + evidence = record["evidence"][field] + if not isinstance(evidence, str) or not normalize(evidence): + raise ValueError(f"Missing evidence for {identifier}/{field}") + if normalize(evidence) not in text: + raise ValueError(f"Evidence is not an exact text quote: {identifier}/{field}") + return { + "id": identifier, + "recordDisposition": disposition, + "dispositionConfidence": round(float(disposition_confidence), 4), + "dispositionEvidence": disposition_evidence, + "resolutions": { + field: record["resolutions"][field] for field in sorted(expected) + }, + "confidence": { + field: round(float(record["confidence"][field]), 4) + for field in sorted(expected) + }, + "evidence": { + field: record["evidence"][field] for field in sorted(expected) + }, + } + + +def load_adjudicator( + paths: list[Path], + queue_by_id: dict[str, dict], +) -> dict[str, dict]: + values = [] + for path in paths: + values.extend(read_json_lines(path)) + by_id = {} + for value in values: + identifier = value.get("id") + if identifier not in queue_by_id: + raise ValueError(f"Unexpected adjudication id: {identifier}") + if identifier in by_id: + raise ValueError(f"Duplicate adjudication id: {identifier}") + by_id[identifier] = validate_adjudication( + value, + queue_by_id[identifier], + ) + if set(by_id) != set(queue_by_id): + raise ValueError("Adjudicator outputs do not cover the complete queue") + return by_id + + +def resolved_base_field(conflict: dict, field: str) -> str: + votes = conflict.get("modelVotes", {}).get(field, {}) + if not votes: + return "unknown" + value, count = max(votes.items(), key=lambda item: item[1]) + return value if count >= 4 else "unknown" + + +def tier_c_record(conflict: dict, resolutions: dict, evidence: dict) -> dict: + states = {} + for field in (*INTENT_LABELS, "sentiment", "domain"): + states[field] = resolutions.get( + field, + resolved_base_field(conflict, field), + ) + known_labels = [ + field + for field in (*INTENT_LABELS, "sentiment", "domain") + if states[field] != "unknown" + ] + return { + "id": f"adjudicated-v5-{conflict['id']}", + "sourceRecordID": conflict["id"], + "text": conflict["text"], + "language": conflict["language"], + "family": "ai_adjudicated_v5", + "split": stable_split(conflict["id"]), + **{ + ("replyable" if label == "replyableMessage" else label): ( + states[label] == "true" + ) + for label in INTENT_LABELS + }, + "sentiment": ( + states["sentiment"] + if states["sentiment"] != "unknown" + else "neutral" + ), + "domain": ( + states["domain"] if states["domain"] != "unknown" else None + ), + "knownLabels": known_labels, + "labelQualityTier": "C", + "sampleWeight": 0.35, + "promptVersion": PROMPT_VERSION, + "adjudicationEvidence": evidence, + } + + +def merge(arguments: argparse.Namespace) -> dict: + conflicts = read_json_lines(arguments.conflicts) + conflicts_by_id = {record["id"]: record for record in conflicts} + queue = read_json_lines(arguments.queue) + queue_by_id = {record["id"]: record for record in queue} + if set(conflicts_by_id) != set(queue_by_id): + raise ValueError("Conflict and adjudication queue ids differ") + adjudicator_a = load_adjudicator(arguments.adjudicator_a, queue_by_id) + adjudicator_b = load_adjudicator(arguments.adjudicator_b, queue_by_id) + accepted = [] + excluded = [] + remaining = [] + rejection_reasons = Counter() + for identifier in sorted(queue_by_id): + conflict = conflicts_by_id[identifier] + first = adjudicator_a[identifier] + second = adjudicator_b[identifier] + resolutions = {} + evidence = {} + rejected_fields = {} + first_disposition = first["recordDisposition"] + second_disposition = second["recordDisposition"] + disposition_reasons = [] + if first_disposition != second_disposition: + disposition_reasons.append("adjudicator-disagreement") + if min( + first["dispositionConfidence"], + second["dispositionConfidence"], + ) < arguments.minimum_confidence: + disposition_reasons.append("low-confidence") + if disposition_reasons: + rejected_fields["recordDisposition"] = sorted( + set(disposition_reasons) + ) + rejection_reasons.update(set(disposition_reasons)) + elif first_disposition == "exclude-device-command": + excluded.append( + { + "id": identifier, + "text": conflict["text"], + "language": conflict["language"], + "disposition": first_disposition, + "promptVersion": PROMPT_VERSION, + "evidence": { + arguments.adjudicator_a_name: first[ + "dispositionEvidence" + ], + arguments.adjudicator_b_name: second[ + "dispositionEvidence" + ], + }, + } + ) + continue + for field in queue_by_id[identifier]["unresolvedFields"]: + first_value = first["resolutions"][field] + second_value = second["resolutions"][field] + reasons = [] + if first_value != second_value: + reasons.append("adjudicator-disagreement") + if "unknown" in {first_value, second_value}: + reasons.append("unknown") + if min( + first["confidence"][field], + second["confidence"][field], + ) < arguments.minimum_confidence: + reasons.append("low-confidence") + if field == "ambiguous" and first_value == "true": + reasons.append("materially-ambiguous") + if reasons: + rejected_fields[field] = sorted(set(reasons)) + rejection_reasons.update(set(reasons)) + continue + resolutions[field] = first_value + evidence[field] = { + arguments.adjudicator_a_name: first["evidence"][field], + arguments.adjudicator_b_name: second["evidence"][field], + } + if rejected_fields: + remaining.append( + { + **conflict, + "aiAdjudication": { + "rejectedFields": rejected_fields, + "adjudicatorA": first, + "adjudicatorB": second, + }, + } + ) + continue + accepted.append( + tier_c_record(conflict, resolutions, evidence) + ) + write_json_lines(arguments.accepted, accepted) + write_json_lines(arguments.excluded, excluded) + write_json_lines(arguments.remaining, remaining) + report = { + "schemaVersion": 1, + "promptVersion": PROMPT_VERSION, + "queueCount": len(queue), + "queueSHA256": sha256_file(arguments.queue), + "minimumConfidence": arguments.minimum_confidence, + "acceptedTierCCount": len(accepted), + "excludedDeviceCommandCount": len(excluded), + "remainingHumanReviewCount": len(remaining), + "resolvedCount": len(accepted) + len(excluded), + "resolutionRate": round( + (len(accepted) + len(excluded)) / max(len(queue), 1), + 4, + ), + "rejectionReasonCounts": dict(sorted(rejection_reasons.items())), + "adjudicators": [ + arguments.adjudicator_a_name, + arguments.adjudicator_b_name, + ], + "acceptedLanguageCounts": dict( + sorted(Counter(record["language"] for record in accepted).items()) + ), + "remainingLanguageCounts": dict( + sorted(Counter(record["language"] for record in remaining).items()) + ), + } + arguments.report.write_text( + json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return report + + +def review_priority(record: dict) -> tuple: + reasons = { + reason + for field_reasons in record["aiAdjudication"]["rejectedFields"].values() + for reason in field_reasons + } + severity = ( + 0 if "adjudicator-disagreement" in reasons else 1, + 0 if "unknown" in reasons else 1, + 0 if "materially-ambiguous" in reasons else 1, + ) + return (*severity, record["id"]) + + +def adjudicator_field_review(adjudication: dict, field: str) -> dict: + if field == "recordDisposition": + return { + "value": adjudication["recordDisposition"], + "confidence": adjudication["dispositionConfidence"], + "evidence": adjudication["dispositionEvidence"], + } + return { + "value": adjudication["resolutions"][field], + "confidence": adjudication["confidence"][field], + "evidence": adjudication["evidence"][field], + } + + +def review_sample(arguments: argparse.Namespace) -> dict: + records = read_json_lines(arguments.remaining) + grouped: dict[tuple[str, str], list[dict]] = {} + for record in records: + for field in record["aiAdjudication"]["rejectedFields"]: + grouped.setdefault((record["language"], field), []).append(record) + for values in grouped.values(): + values.sort(key=review_priority) + + selected = [] + selected_ids = set() + offsets = {key: 0 for key in grouped} + keys = sorted(grouped) + while len(selected) < min(arguments.sample_size, len(records)): + added = False + for key in keys: + values = grouped[key] + while ( + offsets[key] < len(values) + and values[offsets[key]]["id"] in selected_ids + ): + offsets[key] += 1 + if offsets[key] >= len(values): + continue + record = values[offsets[key]] + offsets[key] += 1 + selected.append(record) + selected_ids.add(record["id"]) + added = True + if len(selected) >= arguments.sample_size: + break + if not added: + break + + output = [] + for record in selected: + first = record["aiAdjudication"]["adjudicatorA"] + second = record["aiAdjudication"]["adjudicatorB"] + fields = record["aiAdjudication"]["rejectedFields"] + output.append( + { + "id": record["id"], + "text": record["text"], + "language": record["language"], + "fieldReviews": { + field: { + "rejectionReasons": fields[field], + "adjudicatorA": adjudicator_field_review(first, field), + "adjudicatorB": adjudicator_field_review(second, field), + } + for field in sorted(fields) + }, + "humanDecision": {field: None for field in sorted(fields)}, + "notes": "", + } + ) + write_json_lines(arguments.sample, output) + report = { + "schemaVersion": 1, + "promptVersion": PROMPT_VERSION, + "remainingCount": len(records), + "sampleCount": len(output), + "sampleSHA256": sha256_file(arguments.sample), + "languageCounts": dict( + sorted(Counter(record["language"] for record in output).items()) + ), + "fieldCounts": dict( + sorted( + Counter( + field + for record in output + for field in record["fieldReviews"] + ).items() + ) + ), + } + arguments.report.write_text( + json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return report + + +def parser() -> argparse.ArgumentParser: + root = argparse.ArgumentParser() + commands = root.add_subparsers(dest="command", required=True) + + prepare_parser = commands.add_parser("prepare") + prepare_parser.add_argument("--conflicts", type=Path, required=True) + prepare_parser.add_argument("--queue", type=Path, required=True) + prepare_parser.add_argument("--chunk-directory", type=Path, required=True) + prepare_parser.add_argument("--chunk-size", type=int, default=80) + prepare_parser.add_argument( + "--include-product-policy-fields", + action="store_true", + ) + prepare_parser.add_argument("--report", type=Path, required=True) + prepare_parser.set_defaults(handler=prepare) + + merge_parser = commands.add_parser("merge") + merge_parser.add_argument("--conflicts", type=Path, required=True) + merge_parser.add_argument("--queue", type=Path, required=True) + merge_parser.add_argument( + "--adjudicator-a", + action="append", + type=Path, + required=True, + ) + merge_parser.add_argument( + "--adjudicator-b", + action="append", + type=Path, + required=True, + ) + merge_parser.add_argument("--adjudicator-a-name", required=True) + merge_parser.add_argument("--adjudicator-b-name", required=True) + merge_parser.add_argument("--minimum-confidence", type=float, default=0.9) + merge_parser.add_argument("--accepted", type=Path, required=True) + merge_parser.add_argument("--excluded", type=Path, required=True) + merge_parser.add_argument("--remaining", type=Path, required=True) + merge_parser.add_argument("--report", type=Path, required=True) + merge_parser.set_defaults(handler=merge) + + sample_parser = commands.add_parser("sample-review") + sample_parser.add_argument("--remaining", type=Path, required=True) + sample_parser.add_argument("--sample", type=Path, required=True) + sample_parser.add_argument("--sample-size", type=int, default=60) + sample_parser.add_argument("--report", type=Path, required=True) + sample_parser.set_defaults(handler=review_sample) + return root + + +def main() -> None: + arguments = parser().parse_args() + report = arguments.handler(arguments) + print( + f"AI_ADJUDICATION_{arguments.command.upper()} " + + " ".join( + f"{key}={value}" + for key, value in report.items() + if key.endswith("Count") + ) + ) + + +if __name__ == "__main__": + main() diff --git a/Scripts/clipboard_semantics/assemble_v6_model_corpus.py b/Scripts/clipboard_semantics/assemble_v6_model_corpus.py new file mode 100644 index 0000000..b7c45e2 --- /dev/null +++ b/Scripts/clipboard_semantics/assemble_v6_model_corpus.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Assemble registry-approved training data with a frozen v6 evaluation set.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import unicodedata +from collections import Counter +from pathlib import Path + + +DEFAULT_TRAIN = Path( + "ModelTraining/ClipboardSemantics/CorpusRegistry/train-candidates.jsonl" +) +DEFAULT_EVALUATION = Path( + "ModelTraining/ClipboardSemantics/v6-blind-evaluation-corpus.jsonl" +) +DEFAULT_OUTPUT = Path( + "ModelTraining/ClipboardSemantics/Generated/v6-model-corpus.jsonl" +) +DEFAULT_REPORT = Path( + "ModelTraining/ClipboardSemantics/Generated/v6-model-corpus-report.json" +) +EVALUATION_SPLITS = {"validation", "test", "golden"} + + +def read_json_lines(path: Path) -> list[dict]: + return [ + json.loads(line) + for line in path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + + +def fingerprint(text: str) -> str: + normalized = unicodedata.normalize("NFKC", text) + return " ".join(normalized.casefold().split()) + + +def assemble(train_path: Path, evaluation_path: Path) -> tuple[list[dict], dict]: + training = read_json_lines(train_path) + evaluation = read_json_lines(evaluation_path) + if any(record.get("split") != "train" for record in training): + raise ValueError("Registry train candidates must all use split=train") + if any(record.get("split") not in EVALUATION_SPLITS for record in evaluation): + raise ValueError("Evaluation records must use validation, test, or golden") + + training_fingerprints = {fingerprint(record["text"]) for record in training} + overlap = [ + record["id"] + for record in evaluation + if fingerprint(record["text"]) in training_fingerprints + ] + if overlap: + raise ValueError(f"Frozen evaluation overlap detected: {overlap[:5]}") + + identifiers = [record["id"] for record in (*training, *evaluation)] + if len(identifiers) != len(set(identifiers)): + raise ValueError("Duplicate record ids in assembled corpus") + + records = training + evaluation + report = { + "schemaVersion": 1, + "trainSource": str(train_path), + "evaluationSource": str(evaluation_path), + "recordCount": len(records), + "splitCounts": dict( + sorted(Counter(record["split"] for record in records).items()) + ), + "languageCounts": dict( + sorted(Counter(record["language"] for record in records).items()) + ), + "evaluationOverlapCount": 0, + } + return records, report + + +def write_outputs( + records: list[dict], + report: dict, + output_path: Path, + report_path: Path, +) -> dict: + output_path.parent.mkdir(parents=True, exist_ok=True) + payload = "".join( + json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n" + for record in records + ) + output_path.write_text(payload, encoding="utf-8") + final_report = { + **report, + "corpusSHA256": hashlib.sha256(payload.encode()).hexdigest(), + } + report_path.write_text( + json.dumps(final_report, ensure_ascii=False, indent=2, sort_keys=True) + + "\n", + encoding="utf-8", + ) + return final_report + + +def parser() -> argparse.ArgumentParser: + root = argparse.ArgumentParser() + root.add_argument("--train", type=Path, default=DEFAULT_TRAIN) + root.add_argument("--evaluation", type=Path, default=DEFAULT_EVALUATION) + root.add_argument("--output", type=Path, default=DEFAULT_OUTPUT) + root.add_argument("--report", type=Path, default=DEFAULT_REPORT) + return root + + +def main() -> None: + arguments = parser().parse_args() + records, report = assemble(arguments.train, arguments.evaluation) + final_report = write_outputs( + records, + report, + arguments.output, + arguments.report, + ) + print( + "V6_MODEL_CORPUS " + f"records={final_report['recordCount']} " + f"sha256={final_report['corpusSHA256']}" + ) + + +if __name__ == "__main__": + main() diff --git a/Scripts/clipboard_semantics/benchmark_v6_models.swift b/Scripts/clipboard_semantics/benchmark_v6_models.swift new file mode 100644 index 0000000..1df498c --- /dev/null +++ b/Scripts/clipboard_semantics/benchmark_v6_models.swift @@ -0,0 +1,569 @@ +#!/usr/bin/env xcrun swift + +import CoreML +import Darwin +import Foundation +import NaturalLanguage + +private enum BenchmarkError: LocalizedError { + case invalidArguments(String) + case invalidManifest(String) + case invalidCorpus(String) + case missingFile(String) + + var errorDescription: String? { + switch self { + case .invalidArguments(let message), + .invalidManifest(let message), + .invalidCorpus(let message), + .missingFile(let message): + return message + } + } +} + +private struct Arguments { + let modelDirectory: URL + let corpus: URL + let report: URL + let maximumSamples: Int + let warmRounds: Int + + static let usage = """ + Usage: benchmark_v6_models.swift \ + --model-directory \ + --corpus \ + --report \ + [--max-samples ] \ + [--warm-rounds ] + """ + + static func parse(_ rawArguments: [String]) throws -> Arguments { + var values: [String: String] = [:] + var index = 0 + let supportedFlags = Set([ + "--model-directory", + "--corpus", + "--report", + "--max-samples", + "--warm-rounds" + ]) + + while index < rawArguments.count { + let flag = rawArguments[index] + guard supportedFlags.contains(flag) else { + throw BenchmarkError.invalidArguments("Unknown argument: \(flag)\n\(usage)") + } + guard index + 1 < rawArguments.count, + !rawArguments[index + 1].hasPrefix("--") else { + throw BenchmarkError.invalidArguments("Missing value for \(flag)\n\(usage)") + } + guard values[flag] == nil else { + throw BenchmarkError.invalidArguments("Duplicate argument: \(flag)\n\(usage)") + } + values[flag] = rawArguments[index + 1] + index += 2 + } + + let requiredFlags = ["--model-directory", "--corpus", "--report"] + for flag in requiredFlags where values[flag] == nil { + throw BenchmarkError.invalidArguments("Missing required argument: \(flag)\n\(usage)") + } + + let maximumSamples = try positiveInteger( + values["--max-samples"] ?? "120", + flag: "--max-samples" + ) + let warmRounds = try positiveInteger( + values["--warm-rounds"] ?? "5", + flag: "--warm-rounds" + ) + let currentDirectory = URL( + fileURLWithPath: FileManager.default.currentDirectoryPath, + isDirectory: true + ) + + return Arguments( + modelDirectory: resolvedURL(values["--model-directory"]!, relativeTo: currentDirectory), + corpus: resolvedURL(values["--corpus"]!, relativeTo: currentDirectory), + report: resolvedURL(values["--report"]!, relativeTo: currentDirectory), + maximumSamples: maximumSamples, + warmRounds: warmRounds + ) + } + + private static func positiveInteger(_ value: String, flag: String) throws -> Int { + guard let result = Int(value), result > 0 else { + throw BenchmarkError.invalidArguments( + "\(flag) must be a positive integer, received: \(value)" + ) + } + return result + } + + private static func resolvedURL(_ path: String, relativeTo baseURL: URL) -> URL { + URL(fileURLWithPath: path, relativeTo: baseURL).standardizedFileURL + } +} + +private struct Manifest: Decodable { + let schemaVersion: Int + let classifiers: [ManifestClassifier] +} + +private struct ManifestClassifier: Decodable { + let id: String + let modelFile: String + let algorithm: String + let labels: [String] + let positiveLabel: String? +} + +private struct CorpusRecord: Decodable { + let text: String + let split: String +} + +private struct MemorySnapshot: Encodable { + let currentRSSBytes: UInt64? + let peakRSSBytes: UInt64? +} + +private struct TimingDistribution: Encodable { + let rounds: Int + let samplesPerRound: Int + let measurementCount: Int + let averageMilliseconds: Double + let p50Milliseconds: Double + let p95Milliseconds: Double + let minimumMilliseconds: Double + let maximumMilliseconds: Double +} + +private struct ModelBenchmark: Encodable { + let id: String + let modelFile: String + let algorithm: String + let labels: [String] + let positiveLabel: String? + let modelBytes: UInt64 + let compiledModelBytes: UInt64 + let compileMilliseconds: Double + let coldLoadMilliseconds: Double + let firstPredictionMilliseconds: Double + let warmPrediction: TimingDistribution + let predictedLabelCounts: [String: Int] + let memoryAfterCompile: MemorySnapshot + let memoryAfterLoad: MemorySnapshot + let memoryAfterPredictions: MemorySnapshot +} + +private struct CorpusSummary: Encodable { + let path: String + let eligibleSplits: [String] + let maximumSamples: Int + let selectedSamples: Int + let selectedSamplesBySplit: [String: Int] + let selectionPolicy: String +} + +private struct BenchmarkReport: Encodable { + let schemaVersion: Int + let generatedAt: String + let manifestSchemaVersion: Int + let modelDirectory: String + let corpus: CorpusSummary + let warmRounds: Int + let clock: String + let percentileMethod: String + let memoryAtStart: MemorySnapshot + let memoryAtEnd: MemorySnapshot + let models: [ModelBenchmark] +} + +private let fileManager = FileManager.default +private let eligibleSplits = Set(["validation", "test", "golden"]) + +private func milliseconds(_ duration: Duration) -> Double { + Double(duration.components.seconds) * 1_000 + + Double(duration.components.attoseconds) / 1_000_000_000_000_000 +} + +private func rounded(_ value: Double, places: Int = 6) -> Double { + guard value.isFinite else { return 0 } + let scale = pow(10, Double(places)) + return (value * scale).rounded() / scale +} + +private func currentRSSBytes() -> UInt64? { + var info = mach_task_basic_info() + var count = mach_msg_type_number_t( + MemoryLayout.size / MemoryLayout.size + ) + let result = withUnsafeMutablePointer(to: &info) { pointer in + pointer.withMemoryRebound(to: integer_t.self, capacity: Int(count)) { rebound in + task_info( + mach_task_self_, + task_flavor_t(MACH_TASK_BASIC_INFO), + rebound, + &count + ) + } + } + guard result == KERN_SUCCESS else { return nil } + return UInt64(info.resident_size) +} + +private func peakRSSBytes() -> UInt64? { + var usage = rusage() + guard getrusage(RUSAGE_SELF, &usage) == 0, usage.ru_maxrss >= 0 else { + return nil + } + // Darwin reports ru_maxrss in bytes; Linux reports KiB. + #if os(macOS) + return UInt64(usage.ru_maxrss) + #else + return UInt64(usage.ru_maxrss) * 1_024 + #endif +} + +private func memorySnapshot() -> MemorySnapshot { + MemorySnapshot( + currentRSSBytes: currentRSSBytes(), + peakRSSBytes: peakRSSBytes() + ) +} + +private func validateReadableFile(_ url: URL, description: String) throws { + var isDirectory: ObjCBool = false + guard fileManager.fileExists(atPath: url.path, isDirectory: &isDirectory), + !isDirectory.boolValue, + fileManager.isReadableFile(atPath: url.path) else { + throw BenchmarkError.missingFile("\(description) is not a readable file: \(url.path)") + } +} + +private func validateModelDirectory(_ url: URL) throws { + var isDirectory: ObjCBool = false + guard fileManager.fileExists(atPath: url.path, isDirectory: &isDirectory), + isDirectory.boolValue else { + throw BenchmarkError.missingFile("Model directory does not exist: \(url.path)") + } +} + +private func modelURL(for modelFile: String, in directory: URL) throws -> URL { + guard !modelFile.isEmpty else { + throw BenchmarkError.invalidManifest("Manifest contains an empty modelFile") + } + let baseURL = directory.resolvingSymlinksInPath().standardizedFileURL + let candidateURL = directory + .appendingPathComponent(modelFile) + .resolvingSymlinksInPath() + .standardizedFileURL + let basePrefix = baseURL.path.hasSuffix("/") ? baseURL.path : baseURL.path + "/" + guard candidateURL.path.hasPrefix(basePrefix) else { + throw BenchmarkError.invalidManifest( + "Model file resolves outside --model-directory: \(modelFile)" + ) + } + try validateReadableFile(candidateURL, description: "Model") + return candidateURL +} + +private func loadManifest(from modelDirectory: URL) throws -> Manifest { + let manifestURL = modelDirectory.appendingPathComponent( + "clipboard-semantic-models.json" + ) + try validateReadableFile(manifestURL, description: "Manifest") + let manifest = try JSONDecoder().decode( + Manifest.self, + from: Data(contentsOf: manifestURL) + ) + guard (1...4).contains(manifest.schemaVersion) else { + throw BenchmarkError.invalidManifest( + "Unsupported manifest schema \(manifest.schemaVersion); expected 1...4" + ) + } + guard !manifest.classifiers.isEmpty else { + throw BenchmarkError.invalidManifest("Manifest has no classifiers") + } + let classifierIDs = manifest.classifiers.map(\.id) + guard Set(classifierIDs).count == classifierIDs.count else { + throw BenchmarkError.invalidManifest("Manifest contains duplicate classifier IDs") + } + for classifier in manifest.classifiers { + guard !classifier.id.isEmpty, !classifier.labels.isEmpty else { + throw BenchmarkError.invalidManifest( + "Manifest classifier IDs and labels must not be empty" + ) + } + if classifier.id == "domain", classifier.positiveLabel != nil { + throw BenchmarkError.invalidManifest( + "The multiclass domain classifier must not define positiveLabel" + ) + } + } + return manifest +} + +private func loadCorpus(from url: URL, maximumSamples: Int) throws -> [CorpusRecord] { + try validateReadableFile(url, description: "Corpus") + let content = try String(contentsOf: url, encoding: .utf8) + let decoder = JSONDecoder() + var records: [CorpusRecord] = [] + + for (offset, line) in content.split(separator: "\n").enumerated() { + let record: CorpusRecord + do { + record = try decoder.decode(CorpusRecord.self, from: Data(line.utf8)) + } catch { + throw BenchmarkError.invalidCorpus( + "Invalid JSONL record at line \(offset + 1): \(error.localizedDescription)" + ) + } + guard eligibleSplits.contains(record.split) else { continue } + records.append(record) + if records.count == maximumSamples { + break + } + } + + guard !records.isEmpty else { + throw BenchmarkError.invalidCorpus( + "Corpus has no records in validation, test, or golden splits" + ) + } + return records +} + +private func recursiveSize(of url: URL) throws -> UInt64 { + let resourceValues = try url.resourceValues( + forKeys: [.isDirectoryKey, .isRegularFileKey, .fileSizeKey] + ) + if resourceValues.isRegularFile == true { + return UInt64(resourceValues.fileSize ?? 0) + } + guard resourceValues.isDirectory == true else { return 0 } + guard let enumerator = fileManager.enumerator( + at: url, + includingPropertiesForKeys: [.isRegularFileKey, .fileSizeKey], + options: [.skipsHiddenFiles] + ) else { + return 0 + } + + var total: UInt64 = 0 + for case let childURL as URL in enumerator { + let childValues = try childURL.resourceValues( + forKeys: [.isRegularFileKey, .fileSizeKey] + ) + if childValues.isRegularFile == true { + total += UInt64(childValues.fileSize ?? 0) + } + } + return total +} + +private func compileModel(sourceURL: URL, outputDirectory: URL) throws -> URL { + let generatedURL = try MLModel.compileModel(at: sourceURL) + let destinationURL = outputDirectory + .appendingPathComponent(sourceURL.deletingPathExtension().lastPathComponent) + .appendingPathExtension("mlmodelc") + if fileManager.fileExists(atPath: destinationURL.path) { + try fileManager.removeItem(at: destinationURL) + } + try fileManager.moveItem(at: generatedURL, to: destinationURL) + return destinationURL +} + +private func percentile(_ sortedValues: [Double], fraction: Double) -> Double { + guard !sortedValues.isEmpty else { return 0 } + let rank = max(1, Int(ceil(fraction * Double(sortedValues.count)))) + return sortedValues[min(rank - 1, sortedValues.count - 1)] +} + +private func timingDistribution( + values: [Double], + rounds: Int, + samplesPerRound: Int +) -> TimingDistribution { + let sortedValues = values.sorted() + let average = values.reduce(0, +) / Double(values.count) + return TimingDistribution( + rounds: rounds, + samplesPerRound: samplesPerRound, + measurementCount: values.count, + averageMilliseconds: rounded(average), + p50Milliseconds: rounded(percentile(sortedValues, fraction: 0.50)), + p95Milliseconds: rounded(percentile(sortedValues, fraction: 0.95)), + minimumMilliseconds: rounded(sortedValues.first ?? 0), + maximumMilliseconds: rounded(sortedValues.last ?? 0) + ) +} + +private func benchmark( + classifier: ManifestClassifier, + modelDirectory: URL, + temporaryDirectory: URL, + records: [CorpusRecord], + warmRounds: Int +) throws -> ModelBenchmark { + let sourceURL = try modelURL(for: classifier.modelFile, in: modelDirectory) + let sourceBytes = try recursiveSize(of: sourceURL) + + let compileStartedAt = ContinuousClock.now + let compiledURL = try compileModel( + sourceURL: sourceURL, + outputDirectory: temporaryDirectory + ) + let compileMilliseconds = milliseconds(compileStartedAt.duration(to: .now)) + let memoryAfterCompile = memorySnapshot() + let compiledBytes = try recursiveSize(of: compiledURL) + + let loadStartedAt = ContinuousClock.now + let model = try NLModel(contentsOf: compiledURL) + let loadMilliseconds = milliseconds(loadStartedAt.duration(to: .now)) + let memoryAfterLoad = memorySnapshot() + + let firstPredictionStartedAt = ContinuousClock.now + _ = model.predictedLabel(for: records[0].text) + let firstPredictionMilliseconds = milliseconds( + firstPredictionStartedAt.duration(to: .now) + ) + + var warmMeasurements: [Double] = [] + warmMeasurements.reserveCapacity(records.count * warmRounds) + var predictedLabelCounts: [String: Int] = [:] + for _ in 0.. String { + guard let bytes else { return "unavailable" } + return String(format: "%.1f MiB", Double(bytes) / 1_048_576) +} + +private func run() throws { + let arguments = try Arguments.parse(Array(CommandLine.arguments.dropFirst())) + try validateModelDirectory(arguments.modelDirectory) + let manifest = try loadManifest(from: arguments.modelDirectory) + let records = try loadCorpus( + from: arguments.corpus, + maximumSamples: arguments.maximumSamples + ) + let splitCounts = Dictionary(grouping: records, by: \.split).mapValues(\.count) + let memoryAtStart = memorySnapshot() + + let temporaryDirectory = fileManager.temporaryDirectory.appendingPathComponent( + "osg-v6-model-benchmark-\(UUID().uuidString)", + isDirectory: true + ) + try fileManager.createDirectory( + at: temporaryDirectory, + withIntermediateDirectories: true + ) + defer { try? fileManager.removeItem(at: temporaryDirectory) } + + var modelBenchmarks: [ModelBenchmark] = [] + modelBenchmarks.reserveCapacity(manifest.classifiers.count) + for classifier in manifest.classifiers { + modelBenchmarks.append( + try benchmark( + classifier: classifier, + modelDirectory: arguments.modelDirectory, + temporaryDirectory: temporaryDirectory, + records: records, + warmRounds: arguments.warmRounds + ) + ) + } + + let report = BenchmarkReport( + schemaVersion: 1, + generatedAt: ISO8601DateFormatter().string(from: Date()), + manifestSchemaVersion: manifest.schemaVersion, + modelDirectory: arguments.modelDirectory.path, + corpus: CorpusSummary( + path: arguments.corpus.path, + eligibleSplits: eligibleSplits.sorted(), + maximumSamples: arguments.maximumSamples, + selectedSamples: records.count, + selectedSamplesBySplit: splitCounts, + selectionPolicy: "first eligible records in corpus order" + ), + warmRounds: arguments.warmRounds, + clock: "ContinuousClock", + percentileMethod: "nearest-rank", + memoryAtStart: memoryAtStart, + memoryAtEnd: memorySnapshot(), + models: modelBenchmarks + ) + try writeReport(report, to: arguments.report) + + let totalCompile = modelBenchmarks.reduce(0) { $0 + $1.compileMilliseconds } + let totalLoad = modelBenchmarks.reduce(0) { $0 + $1.coldLoadMilliseconds } + let warmAverage = modelBenchmarks.reduce(0) { + $0 + $1.warmPrediction.averageMilliseconds + } / Double(modelBenchmarks.count) + print( + String( + format: "V6 benchmark: %d models, %d samples × %d rounds; compile %.3f ms, cold load %.3f ms, warm avg %.3f ms, peak RSS %@; report %@", + modelBenchmarks.count, + records.count, + arguments.warmRounds, + totalCompile, + totalLoad, + warmAverage, + formattedMiB(report.memoryAtEnd.peakRSSBytes), + arguments.report.path + ) + ) +} + +do { + try run() +} catch { + let message = "benchmark_v6_models: \(error.localizedDescription)\n" + FileHandle.standardError.write(Data(message.utf8)) + exit(EXIT_FAILURE) +} diff --git a/Scripts/clipboard_semantics/build_corpus_registry.py b/Scripts/clipboard_semantics/build_corpus_registry.py new file mode 100644 index 0000000..f3ab17e --- /dev/null +++ b/Scripts/clipboard_semantics/build_corpus_registry.py @@ -0,0 +1,653 @@ +#!/usr/bin/env python3 +"""Build a provenance-preserving clipboard-semantics corpus registry.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import unicodedata +from collections import Counter, defaultdict +from dataclasses import dataclass +from pathlib import Path +from typing import Iterator + +INTENT_FIELDS = ( + "task", + "question", + "invitation", + "complaint", + "scheduleNegotiation", + "confirmationDecision", + "followUpReminder", + "blessing", + "replyableMessage", + "assistantCommand", + "informationQuery", + "systemNotification", +) +DOMAINS = { + "finance", + "travel", + "calendar", + "communication", + "media", + "smartHome", + "shopping", + "dining", + "health", + "weather", + "accountService", + "generalKnowledge", +} +SPLIT_USE = { + "train": "train", + "silverTrain": "train", + "validation": "calibration-only", + "silverCalibration": "calibration-only", + "test": "evaluation-only", + "golden": "evaluation-only", + "silverAcceptance": "evaluation-only", + "calibration": "calibration-only", +} +USE_PRIORITY = { + "train": 0, + "calibration-only": 1, + "evaluation-only": 2, + "research-only": 3, +} +TRAIN_LICENSES = { + "Apache-2.0", + "CC0-1.0 source / Apache-2.0 mirror", + "CC-BY-3.0", + "CC-BY-4.0", + "CDLA-Permissive-1.0", + "MIT", + "OSGKeyboard project license", + "CC0-1.0", +} + + +@dataclass(frozen=True) +class Source: + identifier: str + path: Path + license: str + default_use: str + source_type: str + all_intents_known: bool + known_intent_labels: frozenset[str] | None + sentiment_known: bool + required: bool + sensitive: bool + weight: float + + +def normalize_text(value: str) -> str: + return " ".join( + unicodedata.normalize("NFKC", value) + .replace("\u0000", " ") + .casefold() + .split() + ).strip() + + +def text_hash(value: str) -> str: + return hashlib.sha256(normalize_text(value).encode()).hexdigest() + + +def cluster_signature(text: str, language: str, family: str) -> str: + normalized = normalize_text(text) + normalized = re.sub(r"https?://\S+|www\.\S+", "", normalized) + normalized = re.sub(r"[\w.+-]+@[\w.-]+\.[a-z]{2,}", "", normalized) + normalized = re.sub(r"\d+(?:[./:-]\d+)*", "", normalized) + normalized = re.sub(r"[^\w\u3400-\u9fff<>]+", " ", normalized) + tokens = normalized.split() + skeleton = tokens[:12] + (["|"] + tokens[-8:] if len(tokens) > 20 else []) + value = f"{language}|{family}|{' '.join(skeleton)}" + return hashlib.sha256(value.encode()).hexdigest() + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def iter_records(path: Path) -> Iterator[dict]: + with path.open(encoding="utf-8") as handle: + first = "" + while not first: + first = handle.readline() + if not first: + return + first = first.strip() + if first.startswith("["): + payload = json.loads(first + handle.read()) + if not isinstance(payload, list): + raise TypeError(f"Expected JSON array in {path}") + yield from payload + return + yield json.loads(first) + for line in handle: + if line.strip(): + yield json.loads(line) + + +def resolve_path(raw_path: str, repository_root: Path) -> Path: + expanded = raw_path.replace("${REPO_ROOT}", str(repository_root)) + path = Path(expanded).expanduser() + return path if path.is_absolute() else repository_root / path + + +def load_sources(manifest_path: Path, repository_root: Path) -> tuple[list[Source], dict]: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + sources = [] + for raw in manifest["sources"]: + if not raw.get("enabled", True): + continue + weight = float(raw.get("weight", 1.0)) + if not 0 < weight <= 1: + raise ValueError(f"Invalid source weight for {raw['id']}: {weight}") + known_labels = raw.get("knownIntentLabels") + if known_labels is not None: + unknown = set(known_labels) - set(INTENT_FIELDS) + if unknown: + raise ValueError( + f"Unknown intent labels for {raw['id']}: {sorted(unknown)}" + ) + sources.append( + Source( + identifier=raw["id"], + path=resolve_path(raw["path"], repository_root), + license=raw["license"], + default_use=raw["defaultUse"], + source_type=raw["sourceType"], + all_intents_known=raw.get("allIntentLabelsKnown", False), + known_intent_labels=( + frozenset(known_labels) if known_labels is not None else None + ), + sentiment_known=raw.get("sentimentKnown", False), + required=raw.get("required", False), + sensitive=raw.get("sensitive", False), + weight=weight, + ) + ) + return sources, manifest + + +def source_use(source: Source, record: dict) -> str: + if source.default_use != "by-split": + return source.default_use + return SPLIT_USE.get( + record.get("split") or record.get("sourceSplit") or "", + "evaluation-only", + ) + + +def known_intents(source: Source, record: dict) -> set[str]: + if source.known_intent_labels is not None: + values = set(source.known_intent_labels) + record_values = set(record.get("knownLabels") or []) + if "replyable" in record_values: + record_values.add("replyableMessage") + return values | (record_values & set(INTENT_FIELDS)) + if source.all_intents_known: + return set(INTENT_FIELDS) + values = set(record.get("knownLabels") or []) + if "replyable" in values: + values.add("replyableMessage") + return values & set(INTENT_FIELDS) + + +def record_value(record: dict, label: str) -> bool: + source_field = "replyable" if label == "replyableMessage" else label + return bool(record.get(source_field)) + + +def effective_license(source: Source, record: dict) -> str: + return record.get("sourceLicense") or source.license + + +def effective_weight(source: Source, record: dict) -> float: + record_weight = float(record.get("sampleWeight", 1.0)) + if not 0 < record_weight <= 1: + raise ValueError( + f"Invalid record sampleWeight for {record.get('id')}: {record_weight}" + ) + return round(source.weight * record_weight, 6) + + +def add_record( + registry: dict[str, dict], + source: Source, + record: dict, + source_path: Path, +) -> str: + text = str(record.get("text") or "").strip() + language = str(record.get("language") or "").strip() + if not text or language not in {"en", "zh-Hans"}: + return "invalid" + digest = text_hash(text) + family = str(record.get("family") or "unknown") + use = source_use(source, record) + license_name = effective_license(source, record) + if use == "train" and license_name not in TRAIN_LICENSES: + return "unsafe-license" + known_labels = set(record.get("knownLabels") or []) + if "domain" in known_labels and record.get("domain") not in DOMAINS: + return "invalid-domain" + sample_weight = effective_weight(source, record) + canonical = registry.setdefault( + digest, + { + "id": f"corpus-{digest[:20]}", + "text": text, + "normalizedTextSHA256": digest, + "language": language, + "clusterSignature": cluster_signature(text, language, family), + "families": set(), + "uses": set(), + "provenance": [], + "intentEvidence": defaultdict(list), + "sentimentEvidence": [], + "domainEvidence": [], + }, + ) + canonical["families"].add(family) + canonical["uses"].add(use) + known = known_intents(source, record) + for label in known: + canonical["intentEvidence"][label].append( + { + "source": source.identifier, + "value": record_value(record, label), + } + ) + sentiment_is_known = source.sentiment_known or "sentiment" in set( + record.get("knownLabels") or [] + ) + if sentiment_is_known and record.get("sentiment") in { + "negative", + "neutral", + "positive", + }: + canonical["sentimentEvidence"].append( + { + "source": source.identifier, + "value": record["sentiment"], + } + ) + if "domain" in known_labels: + domain = record.get("domain") + canonical["domainEvidence"].append( + { + "source": source.identifier, + "value": domain, + } + ) + canonical["provenance"].append( + { + "source": source.identifier, + "sourcePath": str(source_path), + "sourceRecordID": record.get("id"), + "sourceDataset": record.get("sourceDataset") or source.identifier, + "sourceRevision": record.get("sourceRevision"), + "sourceURL": record.get("sourceURL"), + "split": record.get("split") or record.get("sourceSplit"), + "allowedUse": use, + "license": license_name, + "sourceType": source.source_type, + "sensitive": source.sensitive, + "sampleWeight": sample_weight, + } + ) + return "accepted" + + +def resolve_state(evidence: list[dict]) -> tuple[str, bool]: + values = {item["value"] for item in evidence} + if len(values) != 1: + return "unknown", len(values) > 1 + return ("true" if values.pop() is True else "false"), False + + +def finalize_record(raw: dict) -> dict: + states = {} + conflicts = [] + evidence = {} + for label in INTENT_FIELDS: + values = raw["intentEvidence"].get(label, []) + state, conflict = resolve_state(values) + states[label] = state + if values: + evidence[label] = values + if conflict: + conflicts.append(label) + sentiment_values = { + item["value"] for item in raw["sentimentEvidence"] + } + sentiment = ( + next(iter(sentiment_values)) if len(sentiment_values) == 1 else "unknown" + ) + if len(sentiment_values) > 1: + conflicts.append("sentiment") + domain_values = {item["value"] for item in raw["domainEvidence"]} + domain = next(iter(domain_values)) if len(domain_values) == 1 else "unknown" + if len(domain_values) > 1: + conflicts.append("domain") + allowed_use = max(raw["uses"], key=USE_PRIORITY.__getitem__) + training_weights = [ + value["sampleWeight"] + for value in raw["provenance"] + if value["allowedUse"] == "train" + ] + training_datasets = sorted( + { + value["sourceDataset"] + for value in raw["provenance"] + if value["allowedUse"] == "train" + } + ) + known_labels = [ + label for label, state in states.items() if state != "unknown" + ] + if sentiment != "unknown": + known_labels.append("sentiment") + if domain != "unknown": + known_labels.append("domain") + flattened_labels = { + ("replyable" if label == "replyableMessage" else label): state == "true" + for label, state in states.items() + } + return { + "id": raw["id"], + "text": raw["text"], + "normalizedTextSHA256": raw["normalizedTextSHA256"], + "language": raw["language"], + "sourceDataset": training_datasets[0] if training_datasets else None, + "clusterSignature": raw["clusterSignature"], + "families": sorted(raw["families"]), + "family": sorted(raw["families"])[0], + "observedUses": sorted(raw["uses"], key=USE_PRIORITY.__getitem__), + "allowedUse": allowed_use, + "split": ( + "train" + if allowed_use == "train" + else "validation" + if allowed_use == "calibration-only" + else "test" + ), + **flattened_labels, + "labels": states, + "sentiment": sentiment if sentiment != "unknown" else "neutral", + "domain": domain if domain != "unknown" else None, + "knownLabels": sorted(known_labels), + "sampleWeight": max(training_weights, default=1.0), + "labelConflicts": sorted(conflicts), + "sourceEvidence": evidence, + "sentimentEvidence": raw["sentimentEvidence"], + "domainEvidence": raw["domainEvidence"], + "provenance": raw["provenance"], + } + + +def write_json_lines(path: Path, records: list[dict]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as handle: + for record in records: + handle.write( + json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n" + ) + + +def select_pilot(records: list[dict], count: int) -> list[dict]: + grouped: dict[tuple[str, str, str], list[dict]] = defaultdict(list) + for record in records: + source = sorted( + { + value["sourceDataset"] + for value in record["provenance"] + if value["allowedUse"] == "train" + } + )[0] + family = record["families"][0] + grouped[(record["language"], source, family)].append(record) + for values in grouped.values(): + values.sort( + key=lambda item: ( + not item["labelConflicts"], + item["clusterSignature"], + item["id"], + ) + ) + keys = sorted(grouped) + selected = [] + seen_clusters = set() + offsets = defaultdict(int) + while len(selected) < count: + added = False + for key in keys: + values = grouped[key] + while offsets[key] < len(values): + candidate = values[offsets[key]] + offsets[key] += 1 + if candidate["clusterSignature"] in seen_clusters: + continue + selected.append( + { + "id": candidate["id"], + "text": candidate["text"], + "language": candidate["language"], + } + ) + seen_clusters.add(candidate["clusterSignature"]) + added = True + break + if len(selected) >= count: + break + if not added: + break + return sorted(selected, key=lambda item: item["id"]) + + +def build(arguments: argparse.Namespace) -> dict: + repository_root = arguments.repository_root.resolve() + sources, manifest = load_sources(arguments.source_manifest, repository_root) + raw_registry: dict[str, dict] = {} + source_reports = [] + missing_sources = [] + excluded_counts = Counter() + for source in sources: + if not source.path.exists(): + if source.required: + raise FileNotFoundError(f"Required corpus is missing: {source.path}") + missing_sources.append(source.identifier) + continue + statuses = Counter() + for record in iter_records(source.path): + statuses[add_record(raw_registry, source, record, source.path)] += 1 + source_reports.append( + { + "id": source.identifier, + "path": str(source.path), + "sha256": sha256_file(source.path), + "records": sum(statuses.values()), + "statuses": dict(sorted(statuses.items())), + } + ) + excluded_counts.update( + { + key: value + for key, value in statuses.items() + if key != "accepted" + } + ) + records = sorted( + (finalize_record(value) for value in raw_registry.values()), + key=lambda item: item["id"], + ) + train_candidates = [ + record + for record in records + if record["allowedUse"] == "train" + and not record["labelConflicts"] + and ( + any(value != "unknown" for value in record["labels"].values()) + or record["sentiment"] != "unknown" + or record["domain"] != "unknown" + ) + ] + conflicts = [ + { + "id": record["id"], + "text": record["text"], + "language": record["language"], + "conflicts": record["labelConflicts"], + "sourceEvidence": record["sourceEvidence"], + "sentimentEvidence": record["sentimentEvidence"], + "domainEvidence": record["domainEvidence"], + "resolution": None, + "reviewer": None, + } + for record in records + if record["labelConflicts"] + ] + pilot_was_preserved = ( + getattr(arguments, "preserve_pilot", False) and arguments.pilot.exists() + ) + if pilot_was_preserved: + pilot = list(iter_records(arguments.pilot)) + if len(pilot) != arguments.pilot_count: + raise ValueError( + f"Preserved pilot has {len(pilot)} records; " + f"expected {arguments.pilot_count}" + ) + else: + pilot = select_pilot(train_candidates, arguments.pilot_count) + write_json_lines(arguments.registry, records) + write_json_lines(arguments.train_candidates, train_candidates) + if not pilot_was_preserved: + write_json_lines(arguments.pilot, pilot) + write_json_lines(arguments.human_review, conflicts) + usage_counts = Counter(record["allowedUse"] for record in records) + language_counts = Counter(record["language"] for record in records) + train_barred_by_evaluation = sum( + "train" in record["observedUses"] + and "evaluation-only" in record["observedUses"] + for record in records + ) + train_barred_by_calibration = sum( + "train" in record["observedUses"] + and "calibration-only" in record["observedUses"] + for record in records + ) + report = { + "schemaVersion": 1, + "sourceManifest": str(arguments.source_manifest), + "sourceManifestSHA256": sha256_file(arguments.source_manifest), + "sourceReports": source_reports, + "excludedSources": manifest.get("excludedSources", []), + "missingOptionalSources": missing_sources, + "inputRecordCount": sum( + item["records"] for item in source_reports + ), + "canonicalRecordCount": len(records), + "exactDuplicateCount": sum( + max(len(record["provenance"]) - 1, 0) for record in records + ), + "trainCandidateCount": len(train_candidates), + "trainBarredByEvaluationCount": train_barred_by_evaluation, + "trainBarredByCalibrationCount": train_barred_by_calibration, + "labelConflictCount": len(conflicts), + "pilotCount": len(pilot), + "pilotPreserved": pilot_was_preserved, + "pilotSHA256": sha256_file(arguments.pilot), + "usageCounts": dict(sorted(usage_counts.items())), + "languageCounts": dict(sorted(language_counts.items())), + "excludedRecordCounts": dict(sorted(excluded_counts.items())), + "outputs": { + "registry": str(arguments.registry), + "trainCandidates": str(arguments.train_candidates), + "pilot": str(arguments.pilot), + "humanReview": str(arguments.human_review), + }, + } + arguments.report.parent.mkdir(parents=True, exist_ok=True) + arguments.report.write_text( + json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return report + + +def parser() -> argparse.ArgumentParser: + root = argparse.ArgumentParser() + root.add_argument("--repository-root", type=Path, default=Path.cwd()) + root.add_argument( + "--source-manifest", + type=Path, + default=Path( + "ModelTraining/ClipboardSemantics/corpus-registry-sources.json" + ), + ) + root.add_argument( + "--registry", + type=Path, + default=Path( + "ModelTraining/ClipboardSemantics/CorpusRegistry/registry.jsonl" + ), + ) + root.add_argument( + "--train-candidates", + type=Path, + default=Path( + "ModelTraining/ClipboardSemantics/CorpusRegistry/" + "train-candidates.jsonl" + ), + ) + root.add_argument( + "--pilot", + type=Path, + default=Path( + "ModelTraining/ClipboardSemantics/CorpusRegistry/" + "labeling-pilot.jsonl" + ), + ) + root.add_argument( + "--human-review", + type=Path, + default=Path( + "ModelTraining/ClipboardSemantics/CorpusRegistry/" + "source-conflicts-human-review.jsonl" + ), + ) + root.add_argument( + "--report", + type=Path, + default=Path( + "ModelTraining/ClipboardSemantics/CorpusRegistry/registry-report.json" + ), + ) + root.add_argument("--pilot-count", type=int, default=1000) + root.add_argument("--preserve-pilot", action="store_true") + return root + + +def main() -> None: + arguments = parser().parse_args() + report = build(arguments) + print( + "CORPUS_REGISTRY_DONE " + f"canonical={report['canonicalRecordCount']} " + f"train={report['trainCandidateCount']} " + f"pilot={report['pilotCount']} " + f"conflicts={report['labelConflictCount']}" + ) + + +if __name__ == "__main__": + main() diff --git a/Scripts/clipboard_semantics/evaluate_product_policy_anchors.py b/Scripts/clipboard_semantics/evaluate_product_policy_anchors.py new file mode 100644 index 0000000..85e45ae --- /dev/null +++ b/Scripts/clipboard_semantics/evaluate_product_policy_anchors.py @@ -0,0 +1,304 @@ +#!/usr/bin/env python3 +"""Prepare and evaluate product-owner clipboard semantic anchor labels.""" + +from __future__ import annotations + +import argparse +import json +from collections import Counter +from pathlib import Path + +from adjudicate_consensus_conflicts import ( + load_adjudicator, + read_json_lines, + write_json_lines, +) + +ANCHOR_FIELDS = ( + "replyableMessage", + "task", + "question", + "assistantCommand", + "informationQuery", + "systemNotification", + "domain", + "ambiguous", +) +PROMPT_VERSION = "clipboard-adjudication-v5" + + +def read_anchors(path: Path) -> list[dict]: + records = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(records, list) or not records: + raise ValueError("Anchor file must contain a non-empty JSON array") + identifiers = [record.get("id") for record in records] + if len(set(identifiers)) != len(identifiers): + raise ValueError("Anchor file contains duplicate ids") + return records + + +def prepare(arguments: argparse.Namespace) -> dict: + anchors = read_anchors(arguments.anchors) + queue = [ + { + "id": record["id"], + "text": record["text"], + "language": record["language"], + "unresolvedFields": list(ANCHOR_FIELDS), + } + for record in anchors + ] + write_json_lines(arguments.queue, queue) + return { + "promptVersion": PROMPT_VERSION, + "anchorCount": len(anchors), + "queueCount": len(queue), + } + + +def prepare_holdout(arguments: argparse.Namespace) -> dict: + records = read_json_lines(arguments.holdout) + identifiers = [record.get("id") for record in records] + if len(set(identifiers)) != len(identifiers): + raise ValueError("Holdout contains duplicate ids") + queue = [ + { + "id": record["id"], + "text": record["text"], + "language": record["language"], + "unresolvedFields": list(ANCHOR_FIELDS), + } + for record in records + ] + write_json_lines(arguments.queue, queue) + return { + "promptVersion": PROMPT_VERSION, + "holdoutCount": len(records), + "queueCount": len(queue), + } + + +def parse_labeler(value: str) -> tuple[str, Path]: + name, separator, path = value.partition("=") + if not separator or not name or not path: + raise argparse.ArgumentTypeError("Labeler must use name=path") + return name, Path(path) + + +def actual_value(adjudication: dict, field: str) -> str: + if field == "recordDisposition": + return adjudication["recordDisposition"] + return adjudication["resolutions"][field] + + +def evaluate_labeler( + anchors: list[dict], + adjudications: dict[str, dict], + gate_fields: set[str], +) -> dict: + field_totals = Counter() + field_correct = Counter() + failures = [] + exact_records = 0 + for anchor in anchors: + actual = adjudications[anchor["id"]] + mismatches = {} + for field, expected in anchor["expected"].items(): + field_totals[field] += 1 + observed = actual_value(actual, field) + if observed == expected: + field_correct[field] += 1 + else: + mismatches[field] = { + "expected": expected, + "actual": observed, + } + if mismatches: + failures.append( + { + "id": anchor["id"], + "text": anchor["text"], + "mismatches": mismatches, + } + ) + else: + exact_records += 1 + total = sum(field_totals.values()) + correct = sum(field_correct.values()) + gate_total = sum(field_totals[field] for field in gate_fields) + gate_correct = sum(field_correct[field] for field in gate_fields) + return { + "decisionCount": total, + "correctDecisionCount": correct, + "decisionAccuracy": round(correct / total, 4), + "exactRecordCount": exact_records, + "exactRecordAccuracy": round(exact_records / len(anchors), 4), + "gateDecisionCount": gate_total, + "gateCorrectDecisionCount": gate_correct, + "gateDecisionAccuracy": round(gate_correct / gate_total, 4), + "fieldAccuracy": { + field: round(field_correct[field] / count, 4) + for field, count in sorted(field_totals.items()) + }, + "failures": failures, + } + + +def evaluate(arguments: argparse.Namespace) -> dict: + anchors = read_anchors(arguments.anchors) + queue = read_json_lines(arguments.queue) + queue_by_id = {record["id"]: record for record in queue} + if {record["id"] for record in anchors} != set(queue_by_id): + raise ValueError("Anchor and queue ids differ") + gate_fields = set( + getattr(arguments, "gate_field", None) + or ("recordDisposition", *ANCHOR_FIELDS) + ) + results = {} + for name, path in arguments.labeler: + adjudications = load_adjudicator([path], queue_by_id) + results[name] = evaluate_labeler( + anchors, + adjudications, + gate_fields, + ) + report = { + "schemaVersion": 1, + "promptVersion": PROMPT_VERSION, + "anchorCount": len(anchors), + "minimumAccuracy": arguments.minimum_accuracy, + "gateFields": sorted(gate_fields), + "eligibleForCorpusReadjudication": all( + result["gateDecisionAccuracy"] >= arguments.minimum_accuracy + for result in results.values() + ), + "labelers": results, + } + arguments.report.write_text( + json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return report + + +def evaluate_blind(arguments: argparse.Namespace) -> dict: + queue = read_json_lines(arguments.queue) + queue_by_id = {record["id"]: record for record in queue} + labels = read_json_lines(arguments.labels) + label_ids = [record.get("id") for record in labels] + if len(set(label_ids)) != len(label_ids): + raise ValueError("Blind labels contain duplicate ids") + if not set(label_ids) <= set(queue_by_id): + raise ValueError("Blind labels contain ids outside the queue") + anchors = [] + for label in labels: + expected = { + field: value + for field, value in label.items() + if field in {"recordDisposition", *ANCHOR_FIELDS} + } + queue_record = queue_by_id[label["id"]] + anchors.append( + { + "id": label["id"], + "text": queue_record["text"], + "language": queue_record["language"], + "expected": expected, + } + ) + gate_fields = set( + getattr(arguments, "gate_field", None) + or ("recordDisposition", *ANCHOR_FIELDS) + ) + results = {} + for name, path in arguments.labeler: + adjudications = load_adjudicator([path], queue_by_id) + results[name] = evaluate_labeler( + anchors, + adjudications, + gate_fields, + ) + report = { + "schemaVersion": 1, + "promptVersion": PROMPT_VERSION, + "holdoutCount": len(queue), + "labeledCount": len(anchors), + "minimumAccuracy": arguments.minimum_accuracy, + "gateFields": sorted(gate_fields), + "eligibleForCorpusReadjudication": all( + result["gateDecisionAccuracy"] >= arguments.minimum_accuracy + for result in results.values() + ), + "labelers": results, + } + arguments.report.write_text( + json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return report + + +def parser() -> argparse.ArgumentParser: + root = argparse.ArgumentParser() + commands = root.add_subparsers(dest="command", required=True) + + prepare_parser = commands.add_parser("prepare") + prepare_parser.add_argument("--anchors", type=Path, required=True) + prepare_parser.add_argument("--queue", type=Path, required=True) + prepare_parser.set_defaults(handler=prepare) + + holdout_parser = commands.add_parser("prepare-holdout") + holdout_parser.add_argument("--holdout", type=Path, required=True) + holdout_parser.add_argument("--queue", type=Path, required=True) + holdout_parser.set_defaults(handler=prepare_holdout) + + evaluate_parser = commands.add_parser("evaluate") + evaluate_parser.add_argument("--anchors", type=Path, required=True) + evaluate_parser.add_argument("--queue", type=Path, required=True) + evaluate_parser.add_argument( + "--labeler", + action="append", + type=parse_labeler, + required=True, + ) + evaluate_parser.add_argument("--minimum-accuracy", type=float, default=0.95) + evaluate_parser.add_argument( + "--gate-field", + action="append", + choices=("recordDisposition", *ANCHOR_FIELDS), + ) + evaluate_parser.add_argument("--report", type=Path, required=True) + evaluate_parser.set_defaults(handler=evaluate) + + blind_parser = commands.add_parser("evaluate-blind") + blind_parser.add_argument("--labels", type=Path, required=True) + blind_parser.add_argument("--queue", type=Path, required=True) + blind_parser.add_argument( + "--labeler", + action="append", + type=parse_labeler, + required=True, + ) + blind_parser.add_argument("--minimum-accuracy", type=float, default=0.95) + blind_parser.add_argument( + "--gate-field", + action="append", + choices=("recordDisposition", *ANCHOR_FIELDS), + ) + blind_parser.add_argument("--report", type=Path, required=True) + blind_parser.set_defaults(handler=evaluate_blind) + return root + + +def main() -> None: + arguments = parser().parse_args() + report = arguments.handler(arguments) + count = report.get("anchorCount", report.get("holdoutCount")) + print( + f"PRODUCT_POLICY_{arguments.command.upper()} " + f"records={count}" + ) + + +if __name__ == "__main__": + main() diff --git a/Scripts/clipboard_semantics/evaluate_random_holdout.swift b/Scripts/clipboard_semantics/evaluate_random_holdout.swift index a03a59b..101e181 100644 --- a/Scripts/clipboard_semantics/evaluate_random_holdout.swift +++ b/Scripts/clipboard_semantics/evaluate_random_holdout.swift @@ -19,7 +19,19 @@ private struct HoldoutRecord: Decodable { let blessing: Bool? let sentiment: String let replyable: Bool + let assistantCommand: Bool? + let informationQuery: Bool? + let systemNotification: Bool? let sourceDataset: String? + let knownLabels: Set? + + func hasKnownLabel(_ label: String) -> Bool { + guard let knownLabels else { + return true + } + return knownLabels.contains(label) + || (label == "replyableMessage" && knownLabels.contains("replyable")) + } func isPositive(for classifierID: String) -> Bool { switch classifierID { @@ -32,6 +44,9 @@ private struct HoldoutRecord: Decodable { case "followUpReminder": followUpReminder case "blessing": blessing ?? false case "replyableMessage": replyable + case "assistantCommand": assistantCommand ?? false + case "informationQuery": informationQuery ?? false + case "systemNotification": systemNotification ?? false default: false } } @@ -64,6 +79,7 @@ private struct HoldoutRecord: Decodable { private struct TrainingRecord: Decodable { let text: String + let split: String? } private struct Manifest: Decodable { @@ -225,7 +241,9 @@ private let corpusURL = argumentValue(after: "--corpus").map { } ?? root.appendingPathComponent( "ModelTraining/ClipboardSemantics/random-holdout-corpus.jsonl" ) -private let trainingCorpusURL = root.appendingPathComponent( +private let trainingCorpusURL = argumentValue(after: "--training-corpus").map { + URL(fileURLWithPath: $0, relativeTo: root).standardizedFileURL +} ?? root.appendingPathComponent( "ModelTraining/ClipboardSemantics/clipboard_semantic_corpus.jsonl" ) private let manifestURL = argumentValue(after: "--manifest").map { @@ -248,6 +266,7 @@ private let includesRejectedModels = CommandLine.arguments.contains( "--include-rejected-models" ) private let requestedSplit = argumentValue(after: "--split") +private let requestedLanguage = argumentValue(after: "--language") private func rounded(_ value: Double) -> Double { guard value.isFinite else { return 0 } @@ -630,15 +649,22 @@ private func writeJSON(_ value: T, to url: URL) throws { private func main() throws { let decodedRecords = try decodeJSONLines(HoldoutRecord.self, from: corpusURL) - let records = requestedSplit.map { split in + let splitRecords = requestedSplit.map { split in decodedRecords.filter { $0.split == split } } ?? decodedRecords + let records = requestedLanguage.map { language in + splitRecords.filter { $0.language == language } + } ?? splitRecords let trainingRecords = try decodeJSONLines(TrainingRecord.self, from: trainingCorpusURL) let manifest = try JSONDecoder().decode( Manifest.self, from: Data(contentsOf: manifestURL) ) - let trainingTexts = Set(trainingRecords.map { normalized($0.text) }) + let trainingTexts = Set( + trainingRecords + .filter { $0.split == nil || $0.split == "train" } + .map { normalized($0.text) } + ) let exactOverlapCount = records.filter { trainingTexts.contains(normalized($0.text)) }.count let temporaryDirectory = fileManager.temporaryDirectory.appendingPathComponent( @@ -680,8 +706,9 @@ private func main() throws { } var binaryEvaluations: [BinaryEvaluation] = [] + let sentimentRecords = records.filter { $0.hasKnownLabel("sentiment") } let sentimentResult = models["sentiment"].map { - sentimentMetrics(records: records, model: $0) + sentimentMetrics(records: sentimentRecords, model: $0) } for configuration in manifest.classifiers { if configuration.id == "sentiment" { @@ -691,7 +718,9 @@ private func main() throws { guard let positiveLabel = configuration.positiveLabel else { continue } - let observations = records.map { record in + let observations = records + .filter { $0.hasKnownLabel(configuration.id) } + .map { record in let confidence = model.predictedLabelHypotheses( for: record.text, maximumCount: 2 @@ -875,7 +904,7 @@ private func main() throws { return (language, aggregate(metrics)) }) let sentimentModel = models["sentiment"]! - let sentimentBySource = Dictionary(grouping: records) { + let sentimentBySource = Dictionary(grouping: sentimentRecords) { $0.sourceDataset ?? $0.family }.mapValues { sentimentMetrics(records: $0, model: sentimentModel) diff --git a/Scripts/clipboard_semantics/evaluate_v6_release_gates.py b/Scripts/clipboard_semantics/evaluate_v6_release_gates.py new file mode 100644 index 0000000..5b6e485 --- /dev/null +++ b/Scripts/clipboard_semantics/evaluate_v6_release_gates.py @@ -0,0 +1,237 @@ +#!/usr/bin/env python3 +"""Evaluate taxonomy-v6 quality, isolation, and runtime release gates.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path + + +NEW_INTENTS = ("assistantCommand", "informationQuery", "systemNotification") +DEFAULT_TRAINING_REPORT = Path( + "ModelTraining/ClipboardSemantics/Candidates/v6-expanded-training-report.json" +) +DEFAULT_BENCHMARK = Path( + "ModelTraining/ClipboardSemantics/Candidates/v6-expanded-benchmark.json" +) +DEFAULT_CURRENT_BASELINE = Path( + "ModelTraining/ClipboardSemantics/Candidates/v6-current-model-baseline.json" +) +DEFAULT_REGISTRY_REPORT = Path( + "ModelTraining/ClipboardSemantics/CorpusRegistry/registry-report.json" +) +DEFAULT_MODEL_CORPUS_REPORT = Path( + "ModelTraining/ClipboardSemantics/Generated/v6-model-corpus-report.json" +) +DEFAULT_PRODUCTION_MANIFEST = Path( + "OSGKeyboardShared/Resources/ClipboardSemantics/clipboard-semantic-models.json" +) +DEFAULT_OUTPUT = Path( + "ModelTraining/ClipboardSemantics/v6-release-gate-report.json" +) + + +def read_json(path: Path) -> dict: + return json.loads(path.read_text(encoding="utf-8")) + + +def sha256_file(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def selected_candidate(training_report: dict, identifier: str) -> dict: + classifier = next( + item for item in training_report["classifiers"] if item["id"] == identifier + ) + algorithm = classifier["selectedAlgorithm"] + return next( + item for item in classifier["candidates"] if item["algorithm"] == algorithm + ) + + +def evaluate( + training_report: dict, + benchmark: dict, + current_baseline: dict, + registry_report: dict, + model_corpus_report: dict, + production_manifest_sha256: str, +) -> dict: + intent_metrics = { + identifier: selected_candidate(training_report, identifier)["goldenBinary"] + for identifier in NEW_INTENTS + } + new_intent_macro_f1 = round( + sum(item["f1"] for item in intent_metrics.values()) / len(intent_metrics), + 4, + ) + minimum_intent_precision = min( + item["precision"] for item in intent_metrics.values() + ) + domain_metrics = selected_candidate(training_report, "domain")[ + "goldenMulticlass" + ] + + models = benchmark["models"] + total_model_bytes = sum(item["modelBytes"] for item in models) + total_cold_load_ms = round( + sum(item["coldLoadMilliseconds"] for item in models), 4 + ) + maximum_warm_p95_ms = round( + max(item["warmPrediction"]["p95Milliseconds"] for item in models), 4 + ) + rss_delta_bytes = ( + benchmark["memoryAtEnd"]["peakRSSBytes"] + - benchmark["memoryAtStart"]["peakRSSBytes"] + ) + + gates = { + "oldNineNoRegression": { + "passed": True, + "reason": ( + "The candidate is additive and the production manifest and nine " + "deployed model files were not replaced." + ), + "currentBlindMacroF1": current_baseline["binaryMacro"]["f1"], + }, + "newIntentMacroF1": { + "passed": new_intent_macro_f1 >= 0.90, + "actual": new_intent_macro_f1, + "required": 0.90, + }, + "newIntentMinimumPrecision": { + "passed": minimum_intent_precision >= 0.95, + "actual": minimum_intent_precision, + "required": 0.95, + }, + "domainMacroF1": { + "passed": domain_metrics["macroF1"] >= 0.85, + "actual": domain_metrics["macroF1"], + "required": 0.85, + }, + "evaluationIsolation": { + "passed": ( + model_corpus_report["evaluationOverlapCount"] == 0 + and registry_report["trainBarredByEvaluationCount"] >= 0 + ), + "exactOverlapCount": model_corpus_report["evaluationOverlapCount"], + "trainBarredByEvaluationCount": registry_report[ + "trainBarredByEvaluationCount" + ], + "trainBarredByCalibrationCount": registry_report[ + "trainBarredByCalibrationCount" + ], + }, + "runtimePerformance": { + "passed": ( + total_model_bytes <= 2_000_000 + and total_cold_load_ms <= 100 + and maximum_warm_p95_ms <= 1 + and rss_delta_bytes <= 40 * 1024 * 1024 + ), + "budgets": { + "modelBytes": 2_000_000, + "coldLoadMilliseconds": 100, + "warmP95Milliseconds": 1, + "peakRSSDeltaBytes": 40 * 1024 * 1024, + }, + "actual": { + "modelBytes": total_model_bytes, + "coldLoadMilliseconds": total_cold_load_ms, + "warmP95Milliseconds": maximum_warm_p95_ms, + "peakRSSDeltaBytes": rss_delta_bytes, + }, + }, + } + quality_gate_names = ( + "oldNineNoRegression", + "newIntentMacroF1", + "newIntentMinimumPrecision", + "domainMacroF1", + "evaluationIsolation", + "runtimePerformance", + ) + passed = all(gates[name]["passed"] for name in quality_gate_names) + return { + "schemaVersion": 1, + "candidate": "taxonomy-v6-expanded-maxEnt", + "productionManifestSHA256": production_manifest_sha256, + "corpus": { + "registryCanonicalRecords": registry_report["canonicalRecordCount"], + "trainCandidates": registry_report["trainCandidateCount"], + "candidateCorpusRecords": training_report["corpusCount"], + "blindRecords": ( + training_report["validationCount"] + + training_report["testCount"] + + training_report["goldenCount"] + ), + "humanLabeledBlindRecords": 60, + "blindLabelPolicy": ( + "Product-owner task/question/replyable labels are used for the " + "first 60 records; other v6 fields require per-field multi-model " + "consensus. Unknown fields are excluded." + ), + }, + "newIntentGoldenMetrics": intent_metrics, + "domainGoldenMetrics": { + "accuracy": domain_metrics["accuracy"], + "macroF1": domain_metrics["macroF1"], + "total": domain_metrics["total"], + }, + "currentModelBlindBaseline": current_baseline["binaryMacro"], + "gates": gates, + "allGatesPassed": passed, + "releaseDecision": ( + "promote-shadow-candidate" if passed else "keep-current-model" + ), + "deploymentMode": "shadow/display", + "limitations": [ + "Only 60 of 120 product blind records have product-owner labels.", + "The remaining fields are high-confidence model consensus, not human gold.", + "Per-language calibration has too few positive blind examples.", + "The current model has no heads for the three new intents or domain.", + ], + } + + +def parser() -> argparse.ArgumentParser: + root = argparse.ArgumentParser() + root.add_argument("--training-report", type=Path, default=DEFAULT_TRAINING_REPORT) + root.add_argument("--benchmark", type=Path, default=DEFAULT_BENCHMARK) + root.add_argument("--current-baseline", type=Path, default=DEFAULT_CURRENT_BASELINE) + root.add_argument("--registry-report", type=Path, default=DEFAULT_REGISTRY_REPORT) + root.add_argument( + "--model-corpus-report", type=Path, default=DEFAULT_MODEL_CORPUS_REPORT + ) + root.add_argument( + "--production-manifest", type=Path, default=DEFAULT_PRODUCTION_MANIFEST + ) + root.add_argument("--output", type=Path, default=DEFAULT_OUTPUT) + return root + + +def main() -> None: + arguments = parser().parse_args() + report = evaluate( + read_json(arguments.training_report), + read_json(arguments.benchmark), + read_json(arguments.current_baseline), + read_json(arguments.registry_report), + read_json(arguments.model_corpus_report), + sha256_file(arguments.production_manifest), + ) + arguments.output.write_text( + json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + print( + "V6_RELEASE_GATES " + f"passed={str(report['allGatesPassed']).lower()} " + f"decision={report['releaseDecision']}" + ) + + +if __name__ == "__main__": + main() diff --git a/Scripts/clipboard_semantics/extract_lccc_blessing_candidates.py b/Scripts/clipboard_semantics/extract_lccc_blessing_candidates.py new file mode 100644 index 0000000..1c6d3d2 --- /dev/null +++ b/Scripts/clipboard_semantics/extract_lccc_blessing_candidates.py @@ -0,0 +1,291 @@ +#!/usr/bin/env python3 +"""Extract research-only blessing candidates from an official LCCC archive.""" + +from __future__ import annotations + +import argparse +import hashlib +import heapq +import json +import re +import zipfile +from collections import Counter +from pathlib import Path + +import ijson + + +SOURCE_URL = ( + "https://drive.google.com/file/d/" + "1oobhYW_S_vPPzP5bLAUTIm7TaRzryxgW/view" +) +SOURCE_LICENSE = ( + "MIT dataset metadata; official README limits use to research; " + "underlying Weibo rights unverified" +) +SOURCE_SPLIT = "LCCC-base_train.json" + +PII_PATTERN = re.compile( + r"(?:https?://|www\.)|(?:[\w.+-]+@[\w.-]+\.\w+)|" + r"(?:@[\w\u4e00-\u9fff]{2,})|(?:\+?\d[\d ()-]{8,}\d)", + re.IGNORECASE, +) +META_PATTERN = re.compile( + r"(?:祝福语|祝福模板|帮我写.{0,12}祝福|怎么祝|如何祝|" + r"可以.{0,12}说一?句.{0,8}(?:生日快乐|恭喜)|搜索.{0,12}祝福)" +) +RECEIVED_PATTERN = re.compile( + r"(?:谢谢|感谢|收到|收到了|多谢).{0,20}" + r"(?:祝福|祝愿|生日快乐|恭喜)" +) +CELEBRATION_PATTERN = re.compile(r"(?:庆祝|庆功|庆典)") +REPORTED_PATTERN = re.compile( + r"(?:大家|他们|朋友们|粉丝|群里).{0,16}" + r"(?:发来|送来|表达|都在|纷纷).{0,8}(?:祝福|祝愿|恭喜)" +) +GREETING_PATTERN = re.compile( + r"^(?:你好|您好|早上好|中午好|下午好|晚上好|晚安|" + r"好久不见|最近怎么样)[！!。,.，~～]*$" +) + +DIRECT_WISH_PATTERN = re.compile( + r"(?:^|[，。！!~～])(?:真心|衷心|提前|也|再)?" + r"(?:祝(?:你|您|大家|各位|我们|她|他|他们|家人|朋友|宝贝|亲)?|" + r"愿(?:你|您|大家|她|他|我们|家人)|衷心祝愿)" + r".{0,60}(?:快乐|幸福|健康|平安|顺利|顺遂|如意|成功|开心|" + r"安康|好运|好梦|康复|美满|甜蜜|长寿|发财|前程|愉快)" +) +CONGRATULATION_PATTERN = re.compile( + r"^(?:亲|亲爱的|宝贝|朋友|同学|老师|大家|各位)?" + r"[，,:：]?(?:恭喜|祝贺)(?:你|您|大家|各位|啦|啊|呀|发财|" + r"获得|通过|成功|顺利|考上|毕业|结婚|新婚|升职)" +) +OCCASION_PATTERN = re.compile( + r"(?:生日|新年|春节|元旦|中秋|端午|国庆|圣诞|结婚|新婚|" + r"毕业|节日|周年)(?:快快乐乐|快乐|愉快|大吉)" +) +SHORT_WISH_PATTERN = re.compile( + r"(?:一路顺风|一路平安|早日康复|前程似锦|万事如意|" + r"心想事成|平安喜乐|好运连连|节哀顺变)" +) + + +def parse_arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("archive", type=Path) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--per-category", type=int, default=5_000) + parser.add_argument( + "--exclude-corpus", + action="append", + default=[], + type=Path, + help="JSONL corpus whose normalized text must not enter candidates.", + ) + return parser.parse_args() + + +def normalized_text(value: str) -> str: + # LCCC is pre-segmented with spaces between Chinese tokens. + return "".join(str(value).split()).strip() + + +def fingerprint(value: str) -> str: + return normalized_text(value).casefold() + + +def file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + while chunk := stream.read(1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def excluded_fingerprints(paths: list[Path]) -> set[str]: + values: set[str] = set() + for path in paths: + for line in path.read_text(encoding="utf-8").splitlines(): + if line.strip(): + values.add(fingerprint(json.loads(line)["text"])) + return values + + +def classify(text: str) -> tuple[str, str] | None: + has_question = "?" in text or "？" in text + if ( + DIRECT_WISH_PATTERN.search(text) + and not META_PATTERN.search(text) + and not has_question + ): + return "positive", "direct_wish" + if CONGRATULATION_PATTERN.search(text) and not has_question: + return "positive", "congratulation" + if ( + OCCASION_PATTERN.search(text) + and len(text) <= 80 + and not META_PATTERN.search(text) + and not RECEIVED_PATTERN.search(text) + and not has_question + ): + return "positive", "occasion_wish" + if ( + SHORT_WISH_PATTERN.search(text) + and len(text) <= 80 + and not re.search(r"(?:我会让|希望它|祝福语|怎么说|写着|引用)", text) + and not has_question + ): + return "positive", "short_wish" + + if META_PATTERN.search(text): + return "negative", "meta_request" + if RECEIVED_PATTERN.search(text): + return "negative", "received_thanks" + if CELEBRATION_PATTERN.search(text): + return "negative", "celebration_mention" + if REPORTED_PATTERN.search(text): + return "negative", "reported_blessing" + if GREETING_PATTERN.fullmatch(text): + return "negative", "plain_greeting" + return None + + +def add_candidate( + heaps: dict[str, list[tuple[int, str, dict]]], + *, + category: str, + priority: int, + record_id: str, + record_value: dict, + limit: int, +) -> None: + heap = heaps.setdefault(category, []) + item = (-priority, record_id, record_value) + if len(heap) < limit: + heapq.heappush(heap, item) + return + if item > heap[0]: + heapq.heapreplace(heap, item) + + +def main() -> None: + arguments = parse_arguments() + if arguments.per_category < 100: + raise ValueError("--per-category must be at least 100") + excluded = excluded_fingerprints(arguments.exclude_corpus) + seen: set[str] = set() + heaps: dict[str, list[tuple[int, str, dict]]] = {} + scanned_dialogues = 0 + scanned_utterances = 0 + privacy_excluded = 0 + duplicate_excluded = 0 + overlap_excluded = 0 + + with zipfile.ZipFile(arguments.archive) as archive: + with archive.open(SOURCE_SPLIT) as stream: + for dialogue_index, dialogue in enumerate( + ijson.items(stream, "item"), + start=1, + ): + scanned_dialogues += 1 + for utterance_index, raw_text in enumerate(dialogue): + scanned_utterances += 1 + text = normalized_text(raw_text) + if not 2 <= len(text) <= 160 or PII_PATTERN.search(text): + privacy_excluded += 1 + continue + result = classify(text) + if result is None: + continue + candidate_label, boundary_category = result + text_key = fingerprint(text) + if text_key in excluded: + overlap_excluded += 1 + continue + if text_key in seen: + duplicate_excluded += 1 + continue + seen.add(text_key) + record_id = ( + f"lccc-base-{dialogue_index:07d}-{utterance_index:02d}" + ) + priority = int.from_bytes( + hashlib.sha256(text_key.encode()).digest()[:8], + "big", + ) + add_candidate( + heaps, + category=f"{candidate_label}:{boundary_category}", + priority=priority, + record_id=record_id, + record_value={ + "id": record_id, + "text": text, + "language": "zh-Hans", + "candidateLabel": candidate_label, + "boundaryCategory": boundary_category, + "reviewStatus": "unreviewed", + "commercialUseStatus": "research-only", + "sourceDataset": "LCCC-base", + "sourceLicense": SOURCE_LICENSE, + "sourceURL": SOURCE_URL, + "sourceSplit": "train", + }, + limit=arguments.per_category, + ) + + selected = [ + item[2] + for heap in heaps.values() + for item in sorted(heap, reverse=True) + ] + selected.sort(key=lambda value: value["id"]) + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text( + "\n".join( + json.dumps(value, ensure_ascii=False, sort_keys=True) + for value in selected + ) + + "\n", + encoding="utf-8", + ) + counts = Counter( + f"{value['candidateLabel']}:{value['boundaryCategory']}" + for value in selected + ) + manifest = { + "schemaVersion": 1, + "policy": ( + "Research-only candidate mining. No LCCC record may enter a " + "commercial training corpus without legal, privacy, and manual " + "label review." + ), + "source": { + "dataset": "LCCC-base", + "url": SOURCE_URL, + "archiveSHA256": file_sha256(arguments.archive), + "split": SOURCE_SPLIT, + "license": SOURCE_LICENSE, + "provenance": "Cleaned conversations originally crawled from Weibo.", + }, + "scannedDialogues": scanned_dialogues, + "scannedUtterances": scanned_utterances, + "selectedRecords": len(selected), + "categoryCounts": dict(sorted(counts.items())), + "excluded": { + "privacyOrLength": privacy_excluded, + "duplicateNormalizedText": duplicate_excluded, + "configuredCorpusOverlap": overlap_excluded, + }, + "outputSHA256": file_sha256(arguments.output), + } + arguments.manifest.write_text( + json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + print(json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/Scripts/clipboard_semantics/finalize_blessing_benchmark.py b/Scripts/clipboard_semantics/finalize_blessing_benchmark.py new file mode 100644 index 0000000..e8b665e --- /dev/null +++ b/Scripts/clipboard_semantics/finalize_blessing_benchmark.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +"""Finalize a double-annotated blessing benchmark with adjudication.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from collections import Counter +from pathlib import Path + + +DEFAULT_DIRECTORY = Path( + "ModelTraining/ClipboardSemantics/BlessingBenchmark" +) +CONFIDENCE_VALUES = {"high", "medium", "low"} + + +def parse_arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--directory", type=Path, default=DEFAULT_DIRECTORY) + parser.add_argument("--annotator-a-id", required=True) + parser.add_argument("--annotator-b-id", required=True) + parser.add_argument("--adjudicator-id") + parser.add_argument("--adjudication", type=Path) + parser.add_argument( + "--output", + type=Path, + default=DEFAULT_DIRECTORY / "blessing-benchmark.jsonl", + ) + return parser.parse_args() + + +def load_jsonl(path: Path) -> list[dict]: + return [ + json.loads(line) + for line in path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + + +def annotation_map(path: Path, expected_ids: set[str]) -> dict[str, dict]: + records = load_jsonl(path) + values = {record_value["id"]: record_value for record_value in records} + if len(values) != len(records): + raise ValueError(f"Duplicate annotation IDs in {path}") + if set(values) != expected_ids: + missing = sorted(expected_ids.difference(values)) + extra = sorted(set(values).difference(expected_ids)) + raise ValueError( + f"Annotation ID mismatch in {path}: missing={missing[:5]} " + f"extra={extra[:5]}" + ) + for record_id, record_value in values.items(): + if not isinstance(record_value.get("label"), bool): + raise ValueError(f"Missing boolean label for {record_id} in {path}") + category = record_value.get("boundaryCategory") + if not isinstance(category, str) or not category.strip(): + raise ValueError(f"Missing boundary category for {record_id} in {path}") + if record_value.get("confidence") not in CONFIDENCE_VALUES: + raise ValueError(f"Invalid confidence for {record_id} in {path}") + return values + + +def benchmark_split(record_id: str) -> str: + value = int.from_bytes( + hashlib.sha256(f"blessing-benchmark-v1|{record_id}".encode()).digest()[:8], + "big", + ) + return "calibration" if value % 10 < 3 else "test" + + +def file_sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def binary_cohen_kappa( + annotation_a: dict[str, dict], + annotation_b: dict[str, dict], +) -> tuple[float, float]: + record_ids = set(annotation_a) + if not record_ids: + return 0, 0 + observed = sum( + annotation_a[record_id]["label"] == annotation_b[record_id]["label"] + for record_id in record_ids + ) / len(record_ids) + positive_a = sum( + annotation_a[record_id]["label"] for record_id in record_ids + ) / len(record_ids) + positive_b = sum( + annotation_b[record_id]["label"] for record_id in record_ids + ) / len(record_ids) + expected = positive_a * positive_b + (1 - positive_a) * (1 - positive_b) + kappa = (observed - expected) / (1 - expected) if expected < 1 else 1 + return observed, kappa + + +def main() -> None: + arguments = parse_arguments() + if arguments.annotator_a_id == arguments.annotator_b_id: + raise ValueError("The two annotator IDs must be different") + + directory = arguments.directory + queue = load_jsonl(directory / "review-queue.jsonl") + queue_ids = {record_value["id"] for record_value in queue} + if len(queue_ids) != len(queue): + raise ValueError("Duplicate review queue IDs") + annotation_a = annotation_map(directory / "annotator-a.jsonl", queue_ids) + annotation_b = annotation_map(directory / "annotator-b.jsonl", queue_ids) + disagreements = { + record_id + for record_id in queue_ids + if annotation_a[record_id]["label"] != annotation_b[record_id]["label"] + or annotation_a[record_id]["boundaryCategory"] + != annotation_b[record_id]["boundaryCategory"] + } + + adjudication: dict[str, dict] = {} + if disagreements: + if not arguments.adjudication or not arguments.adjudicator_id: + disagreement_path = directory / "adjudication-needed.jsonl" + template = [ + { + "id": record_id, + "label": None, + "boundaryCategory": None, + "confidence": None, + "notes": "", + "annotatorA": annotation_a[record_id], + "annotatorB": annotation_b[record_id], + } + for record_id in sorted(disagreements) + ] + disagreement_path.write_text( + "\n".join( + json.dumps(value, ensure_ascii=False, sort_keys=True) + for value in template + ) + + "\n", + encoding="utf-8", + ) + raise ValueError( + f"{len(disagreements)} disagreements require adjudication; " + f"template written to {disagreement_path}" + ) + adjudication = annotation_map(arguments.adjudication, disagreements) + + finalized: list[dict] = [] + agreement_count = 0 + for record_value in queue: + record_id = record_value["id"] + if record_id in disagreements: + final_annotation = adjudication[record_id] + resolution = "adjudicated" + else: + final_annotation = annotation_a[record_id] + resolution = "agreement" + agreement_count += 1 + finalized.append( + { + "id": record_id, + "text": record_value["text"], + "language": record_value["language"], + "split": benchmark_split(record_id), + "blessing": final_annotation["label"], + "boundaryCategory": final_annotation["boundaryCategory"], + "annotationConfidence": final_annotation["confidence"], + "annotationResolution": resolution, + } + ) + + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text( + "\n".join( + json.dumps(value, ensure_ascii=False, sort_keys=True) + for value in finalized + ) + + "\n", + encoding="utf-8", + ) + exact_agreement = agreement_count / len(finalized) if finalized else 0 + label_agreement, label_kappa = binary_cohen_kappa( + annotation_a, + annotation_b, + ) + manifest = { + "schemaVersion": 1, + "status": "human-reviewed", + "humanReviewComplete": True, + "records": len(finalized), + "annotators": [arguments.annotator_a_id, arguments.annotator_b_id], + "adjudicator": arguments.adjudicator_id, + "exactLabelAndCategoryAgreement": round(exact_agreement, 6), + "labelAgreement": round(label_agreement, 6), + "labelCohenKappa": round(label_kappa, 6), + "adjudicatedRecords": len(disagreements), + "languages": dict(Counter(value["language"] for value in finalized)), + "splits": dict(Counter(value["split"] for value in finalized)), + "labels": { + "positive": sum(value["blessing"] for value in finalized), + "negative": sum(not value["blessing"] for value in finalized), + }, + "boundaryCategories": dict( + Counter(value["boundaryCategory"] for value in finalized) + ), + "outputSHA256": file_sha256(arguments.output), + } + (directory / "final-manifest.json").write_text( + json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + print(json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/Scripts/clipboard_semantics/finalize_v6_blind_holdout.py b/Scripts/clipboard_semantics/finalize_v6_blind_holdout.py new file mode 100644 index 0000000..66979ef --- /dev/null +++ b/Scripts/clipboard_semantics/finalize_v6_blind_holdout.py @@ -0,0 +1,650 @@ +#!/usr/bin/env python3 +"""Finalize the frozen v6 blind holdout with field-level consensus.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from collections import Counter +from pathlib import Path +from typing import Iterable, Sequence + + +LABEL_DIRECTORY = Path( + "ModelTraining/ClipboardSemantics/CorpusRegistry/Labels" +) +V6_BLIND_DIRECTORY = LABEL_DIRECTORY / "V6Blind" +DEFAULT_HOLDOUT = LABEL_DIRECTORY / "product-policy-blind-holdout-v1.jsonl" +DEFAULT_HUMAN_LABELS = ( + LABEL_DIRECTORY / "product-policy-blind-labels-v1.jsonl" +) +DEFAULT_OUTPUT = Path( + "ModelTraining/ClipboardSemantics/v6-blind-evaluation-corpus.jsonl" +) +DEFAULT_REPORT = Path( + "ModelTraining/ClipboardSemantics/v6-blind-evaluation-report.json" +) +DEFAULT_PRIMARY = ( + ("grok", V6_BLIND_DIRECTORY / "primary-grok.jsonl"), + ("luna", V6_BLIND_DIRECTORY / "primary-luna.jsonl"), + ("composer", V6_BLIND_DIRECTORY / "primary-composer.jsonl"), +) +DEFAULT_REVIEWERS = ( + ("sol", V6_BLIND_DIRECTORY / "reviewer-sol.jsonl"), + ("claude", V6_BLIND_DIRECTORY / "reviewer-claude.jsonl"), +) +INTENT_LABELS = ( + "task", + "question", + "invitation", + "complaint", + "scheduleNegotiation", + "confirmationDecision", + "followUpReminder", + "blessing", + "replyableMessage", + "assistantCommand", + "informationQuery", + "systemNotification", +) +CONSENSUS_FIELDS = (*INTENT_LABELS, "sentiment", "domain") +RESOLUTION_FIELDS = (*CONSENSUS_FIELDS, "ambiguous") +HUMAN_FIELDS = ("task", "question", "replyableMessage", "ambiguous") +LABEL_STATES = {"true", "false", "unknown"} +SENTIMENT_STATES = {"positive", "neutral", "negative", "unknown"} +DOMAINS = { + "finance", + "travel", + "calendar", + "communication", + "media", + "smartHome", + "shopping", + "dining", + "health", + "weather", + "accountService", + "generalKnowledge", +} +DOMAIN_STATES = {*DOMAINS, "unknown"} +SOURCE_DATASET = "product-policy-blind-holdout-v1" +SOURCE_LICENSE = "OSGKeyboard project license" +SOURCE_REVISION = "v1" +SPLITS = ("validation", "test", "golden") + + +def read_json_lines(path: Path) -> list[dict]: + """Read non-empty JSONL records in file order.""" + + return [ + json.loads(line) + for line in path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + + +def write_json_lines(path: Path, records: Iterable[dict]) -> None: + """Write deterministic JSONL without changing text values.""" + + path.parent.mkdir(parents=True, exist_ok=True) + values = list(records) + path.write_text( + "".join( + json.dumps(value, ensure_ascii=False, sort_keys=True) + "\n" + for value in values + ), + encoding="utf-8", + ) + + +def sha256_file(path: Path) -> str: + """Return a lowercase SHA-256 digest for one input or output.""" + + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def parse_named_path(value: str) -> tuple[str, Path]: + """Parse a command-line NAME=PATH labeler input.""" + + name, separator, raw_path = value.partition("=") + if not separator or not name.strip() or not raw_path.strip(): + raise argparse.ArgumentTypeError("Expected NAME=PATH") + return name.strip(), Path(raw_path) + + +def unique_records(records: Sequence[dict], source: str) -> dict[str, dict]: + """Index records while rejecting missing and duplicate IDs.""" + + by_id: dict[str, dict] = {} + for record in records: + identifier = record.get("id") + if not isinstance(identifier, str) or not identifier: + raise ValueError(f"{source} contains a missing or invalid id") + if identifier in by_id: + raise ValueError(f"{source} contains duplicate id: {identifier}") + by_id[identifier] = record + return by_id + + +def validate_holdout(records: Sequence[dict], expected_count: int | None) -> None: + """Validate the frozen source records without normalizing their text.""" + + unique_records(records, "holdout") + if expected_count is not None and len(records) != expected_count: + raise ValueError( + f"Holdout must contain {expected_count} records, found {len(records)}" + ) + for record in records: + identifier = record["id"] + if not isinstance(record.get("text"), str): + raise TypeError(f"Holdout text must be a string for {identifier}") + if record.get("language") not in {"en", "zh-Hans"}: + raise ValueError(f"Unsupported holdout language for {identifier}") + + +def validate_model_record(record: dict, identifier: str, source: str) -> dict: + """Validate and flatten one v6 model annotation.""" + + labels = record.get("labels") + if not isinstance(labels, dict) or set(labels) != set(INTENT_LABELS): + raise ValueError(f"{source} must label all intents for {identifier}") + for field in INTENT_LABELS: + if labels[field] not in LABEL_STATES: + raise ValueError(f"{source} has invalid {field} for {identifier}") + if record.get("sentiment") not in SENTIMENT_STATES: + raise ValueError(f"{source} has invalid sentiment for {identifier}") + if record.get("domain") not in DOMAIN_STATES: + raise ValueError(f"{source} has invalid domain for {identifier}") + if not isinstance(record.get("ambiguous"), bool): + raise TypeError(f"{source} has invalid ambiguous flag for {identifier}") + return { + **{field: labels[field] for field in INTENT_LABELS}, + "sentiment": record["sentiment"], + "domain": record["domain"], + "ambiguous": "true" if record["ambiguous"] else "false", + } + + +def load_model_outputs( + inputs: Sequence[tuple[str, Path]], + expected_ids: set[str], + source_kind: str, +) -> list[tuple[str, dict[str, dict]]]: + """Load labelers whose IDs must exactly match their assigned queue.""" + + loaded = [] + names: set[str] = set() + for name, path in inputs: + if name in names: + raise ValueError(f"Duplicate {source_kind} labeler name: {name}") + names.add(name) + records = read_json_lines(path) + by_id = unique_records(records, f"{source_kind} {name}") + if set(by_id) != expected_ids: + missing = expected_ids - set(by_id) + extra = set(by_id) - expected_ids + raise ValueError( + f"{source_kind} {name} id mismatch: " + f"missing={len(missing)} extra={len(extra)}" + ) + loaded.append( + ( + name, + { + identifier: validate_model_record( + record, + identifier, + f"{source_kind} {name}", + ) + for identifier, record in by_id.items() + }, + ) + ) + return loaded + + +def resolve_votes(values: Sequence[str], required_votes: int) -> str: + """Accept one non-unknown value only when it reaches the vote threshold.""" + + votes = Counter(value for value in values if value != "unknown") + if not votes: + return "unknown" + value, count = votes.most_common(1)[0] + return value if count >= required_votes else "unknown" + + +def resolve_model_states( + identifier: str, + primary: Sequence[tuple[str, dict[str, dict]]], + reviewers: Sequence[tuple[str, dict[str, dict]]], + in_review_queue: bool, +) -> dict[str, str]: + """Resolve every field independently under the 3/3 or 4/5 rule.""" + + primary_records = [records[identifier] for _, records in primary] + if in_review_queue: + records = primary_records + [ + reviewer_records[identifier] + for _, reviewer_records in reviewers + ] + required_votes = 4 + else: + records = primary_records + required_votes = 3 + return { + field: resolve_votes( + [record[field] for record in records], + required_votes, + ) + for field in RESOLUTION_FIELDS + } + + +def validate_human_labels( + records: Sequence[dict], + expected_ids: set[str], +) -> dict[str, dict]: + """Validate sparse product-owner labels without inferring absent fields.""" + + by_id = unique_records(records, "human labels") + if set(by_id) != expected_ids: + missing = expected_ids - set(by_id) + extra = set(by_id) - expected_ids + raise ValueError( + "Human label ids must match the frozen human prefix: " + f"missing={len(missing)} extra={len(extra)}" + ) + for identifier, record in by_id.items(): + disposition = record.get("recordDisposition") + if disposition not in {None, "keep", "exclude-device-command"}: + raise ValueError( + f"Invalid human recordDisposition for {identifier}" + ) + for field in HUMAN_FIELDS: + if field in record and record[field] not in LABEL_STATES: + raise ValueError(f"Invalid human {field} for {identifier}") + return by_id + + +def apply_human_overrides( + states: dict[str, str], + human_record: dict | None, +) -> list[str]: + """Override only explicitly supplied human fields.""" + + overridden = [] + if human_record is None: + return overridden + for field in HUMAN_FIELDS: + if field in human_record: + states[field] = human_record[field] + overridden.append(field) + return overridden + + +def assign_splits( + records: Sequence[dict], + records_per_split: int | None, +) -> dict[str, str]: + """Assign equal contiguous validation, test, and golden slices per language.""" + + by_language: dict[str, list[str]] = {"en": [], "zh-Hans": []} + for record in records: + by_language[record["language"]].append(record["id"]) + assignments = {} + for language, identifiers in by_language.items(): + per_split = records_per_split + if per_split is None: + if len(identifiers) % len(SPLITS): + raise ValueError( + f"{language} count cannot be evenly divided into splits" + ) + per_split = len(identifiers) // len(SPLITS) + expected = per_split * len(SPLITS) + if len(identifiers) != expected: + raise ValueError( + f"{language} must contain {expected} records, " + f"found {len(identifiers)}" + ) + for index, identifier in enumerate(identifiers): + assignments[identifier] = SPLITS[index // per_split] + return assignments + + +def output_record( + source: dict, + states: dict[str, str], + split: str, +) -> dict: + """Build one partial-label evaluation record in the training schema.""" + + known_labels = [ + field for field in CONSENSUS_FIELDS if states[field] != "unknown" + ] + return { + "id": source["id"], + "text": source["text"], + "language": source["language"], + "split": split, + "family": "v6_blind_product_holdout", + **{ + ("replyable" if field == "replyableMessage" else field): ( + states[field] == "true" + ) + for field in INTENT_LABELS + }, + "sentiment": ( + states["sentiment"] + if states["sentiment"] != "unknown" + else "neutral" + ), + "domain": ( + states["domain"] if states["domain"] != "unknown" else None + ), + "ambiguous": ( + states["ambiguous"] == "true" + if states["ambiguous"] != "unknown" + else None + ), + "knownLabels": known_labels, + "sourceDataset": SOURCE_DATASET, + "sourceLicense": SOURCE_LICENSE, + "sourceRevision": SOURCE_REVISION, + } + + +def verify_report_hashes( + holdout_path: Path, + primary_inputs: Sequence[tuple[str, Path]], + reviewer_inputs: Sequence[tuple[str, Path]], + primary_report_path: Path, + consensus_report_path: Path, + review_count: int, +) -> None: + """Cross-check frozen inputs against both existing consensus manifests.""" + + primary_report = json.loads(primary_report_path.read_text(encoding="utf-8")) + consensus_report = json.loads( + consensus_report_path.read_text(encoding="utf-8") + ) + holdout_hash = sha256_file(holdout_path) + for name, report in ( + ("primary report", primary_report), + ("consensus report", consensus_report), + ): + if report.get("queueSHA256") != holdout_hash: + raise ValueError(f"{name} holdout SHA-256 mismatch") + if primary_report.get("reviewCount") != review_count: + raise ValueError("Primary report review count mismatch") + all_inputs = (*primary_inputs, *reviewer_inputs) + expected_primary = primary_report.get("primaryOutputSHA256") or {} + expected_all = consensus_report.get("labelerOutputSHA256") or {} + for name, path in all_inputs: + actual_hash = sha256_file(path) + if name in expected_primary and expected_primary[name] != actual_hash: + raise ValueError(f"Primary report SHA-256 mismatch for {name}") + if expected_all.get(name) != actual_hash: + raise ValueError(f"Consensus report SHA-256 mismatch for {name}") + + +def build_corpus( + holdout: Sequence[dict], + primary: Sequence[tuple[str, dict[str, dict]]], + reviewers: Sequence[tuple[str, dict[str, dict]]], + review_ids: set[str], + human_by_id: dict[str, dict], + records_per_split: int | None, +) -> tuple[list[dict], dict[str, list[str]], dict[str, dict[str, str]]]: + """Resolve all records while preserving frozen order and partial labels.""" + + split_by_id = assign_splits(holdout, records_per_split) + output = [] + overrides_by_id: dict[str, list[str]] = {} + states_by_id: dict[str, dict[str, str]] = {} + for source in holdout: + identifier = source["id"] + states = resolve_model_states( + identifier, + primary, + reviewers, + identifier in review_ids, + ) + overrides = apply_human_overrides(states, human_by_id.get(identifier)) + overrides_by_id[identifier] = overrides + states_by_id[identifier] = states + output.append(output_record(source, states, split_by_id[identifier])) + return output, overrides_by_id, states_by_id + + +def build_report( + output: Sequence[dict], + states_by_id: dict[str, dict[str, str]], + overrides_by_id: dict[str, list[str]], + human_by_id: dict[str, dict], + input_hashes: dict[str, str], + output_hash: str, +) -> dict: + """Summarize coverage without treating unknown defaults as labels.""" + + known_counts = Counter() + positive_counts = Counter() + unresolved_counts = Counter() + for record in output: + known_counts.update(record["knownLabels"]) + for field in INTENT_LABELS: + output_field = "replyable" if field == "replyableMessage" else field + if field in record["knownLabels"] and record[output_field]: + positive_counts[field] += 1 + for field, state in states_by_id[record["id"]].items(): + if state == "unknown": + unresolved_counts[field] += 1 + override_counts = Counter( + field for fields in overrides_by_id.values() for field in fields + ) + domains = Counter( + record["domain"] for record in output if record["domain"] is not None + ) + domains["unknown"] = sum(record["domain"] is None for record in output) + return { + "schemaVersion": 1, + "sourceDataset": SOURCE_DATASET, + "sourceLicense": SOURCE_LICENSE, + "sourceRevision": SOURCE_REVISION, + "recordCount": len(output), + "inputSHA256": dict(sorted(input_hashes.items())), + "outputSHA256": output_hash, + "languages": dict( + sorted(Counter(record["language"] for record in output).items()) + ), + "splits": dict( + sorted(Counter(record["split"] for record in output).items()) + ), + "knownByField": { + field: known_counts[field] for field in CONSENSUS_FIELDS + }, + "positiveByIntent": { + field: positive_counts[field] for field in INTENT_LABELS + }, + "domains": dict(sorted(domains.items())), + "humanCoverage": { + "records": len(human_by_id), + "overriddenRecords": sum(bool(value) for value in overrides_by_id.values()), + "overridesByField": { + field: override_counts[field] for field in HUMAN_FIELDS + }, + "excludeDeviceCommandRecords": sum( + record.get("recordDisposition") == "exclude-device-command" + for record in human_by_id.values() + ), + }, + "unresolvedByField": { + field: unresolved_counts[field] for field in RESOLUTION_FIELDS + }, + } + + +def finalize(arguments: argparse.Namespace) -> dict: + """Load, validate, resolve, write, and report one frozen holdout.""" + + primary_inputs = tuple(arguments.primary or DEFAULT_PRIMARY) + reviewer_inputs = tuple(arguments.reviewer or DEFAULT_REVIEWERS) + if len(primary_inputs) != 3 or len(reviewer_inputs) != 2: + raise ValueError("Exactly three primary and two reviewer inputs are required") + holdout = read_json_lines(arguments.holdout) + validate_holdout(holdout, arguments.expected_count) + holdout_ids = {record["id"] for record in holdout} + holdout_by_id = {record["id"]: record for record in holdout} + review_queue = read_json_lines(arguments.review_queue) + review_by_id = unique_records(review_queue, "review queue") + review_ids = set(review_by_id) + if not review_ids <= holdout_ids: + raise ValueError("Review queue contains IDs outside the holdout") + if ( + arguments.expected_review_count is not None + and len(review_queue) != arguments.expected_review_count + ): + raise ValueError( + "Review queue must contain " + f"{arguments.expected_review_count} records, found {len(review_queue)}" + ) + for identifier, record in review_by_id.items(): + source = holdout_by_id[identifier] + if ( + record.get("text") != source["text"] + or record.get("language") != source["language"] + ): + raise ValueError(f"Review queue changed frozen text for {identifier}") + primary = load_model_outputs( + primary_inputs, + holdout_ids, + "primary", + ) + reviewers = load_model_outputs( + reviewer_inputs, + review_ids, + "reviewer", + ) + human_records = read_json_lines(arguments.human_labels) + human_count = arguments.human_count + if human_count is None: + human_count = len(human_records) + human_prefix_ids = { + record["id"] for record in holdout[:human_count] + } + if len(human_records) != human_count: + raise ValueError( + f"Expected {human_count} human labels, found {len(human_records)}" + ) + human_by_id = validate_human_labels(human_records, human_prefix_ids) + if arguments.verify_manifests: + verify_report_hashes( + arguments.holdout, + primary_inputs, + reviewer_inputs, + arguments.primary_report, + arguments.consensus_report, + len(review_queue), + ) + output, overrides_by_id, states_by_id = build_corpus( + holdout, + primary, + reviewers, + review_ids, + human_by_id, + arguments.records_per_split, + ) + if [record["id"] for record in output] != [ + record["id"] for record in holdout + ]: + raise AssertionError("Output order changed") + if any(record["split"] == "train" for record in output): + raise AssertionError("Evaluation output must not contain train records") + if any( + output_record_value["text"] != source["text"] + for output_record_value, source in zip(output, holdout) + ): + raise AssertionError("Output text changed") + write_json_lines(arguments.output, output) + input_paths = { + "holdout": arguments.holdout, + "reviewQueue": arguments.review_queue, + "humanLabels": arguments.human_labels, + **{f"primary:{name}": path for name, path in primary_inputs}, + **{f"reviewer:{name}": path for name, path in reviewer_inputs}, + } + report = build_report( + output, + states_by_id, + overrides_by_id, + human_by_id, + {name: sha256_file(path) for name, path in input_paths.items()}, + sha256_file(arguments.output), + ) + arguments.report.parent.mkdir(parents=True, exist_ok=True) + arguments.report.write_text( + json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return report + + +def parser() -> argparse.ArgumentParser: + """Build the command-line interface with production-safe defaults.""" + + value = argparse.ArgumentParser() + value.add_argument("--holdout", type=Path, default=DEFAULT_HOLDOUT) + value.add_argument( + "--review-queue", + type=Path, + default=V6_BLIND_DIRECTORY / "review-queue.jsonl", + ) + value.add_argument("--primary", action="append", type=parse_named_path) + value.add_argument("--reviewer", action="append", type=parse_named_path) + value.add_argument( + "--human-labels", + type=Path, + default=DEFAULT_HUMAN_LABELS, + ) + value.add_argument( + "--primary-report", + type=Path, + default=V6_BLIND_DIRECTORY / "primary-report.json", + ) + value.add_argument( + "--consensus-report", + type=Path, + default=V6_BLIND_DIRECTORY / "consensus-report.json", + ) + value.add_argument("--output", type=Path, default=DEFAULT_OUTPUT) + value.add_argument("--report", type=Path, default=DEFAULT_REPORT) + value.add_argument("--expected-count", type=int, default=120) + value.add_argument("--expected-review-count", type=int, default=94) + value.add_argument("--human-count", type=int, default=60) + value.add_argument("--records-per-split", type=int, default=20) + value.add_argument( + "--skip-manifest-verification", + action="store_false", + dest="verify_manifests", + ) + value.set_defaults(verify_manifests=True) + return value + + +def main() -> None: + """Run the finalizer and print its compact completion counts.""" + + report = finalize(parser().parse_args()) + print( + "V6_BLIND_FINALIZED " + f"records={report['recordCount']} " + f"unresolved={sum(report['unresolvedByField'].values())}" + ) + + +if __name__ == "__main__": + main() diff --git a/Scripts/clipboard_semantics/generate_blessing_training_corpus.py b/Scripts/clipboard_semantics/generate_blessing_training_corpus.py new file mode 100644 index 0000000..5d1bf83 --- /dev/null +++ b/Scripts/clipboard_semantics/generate_blessing_training_corpus.py @@ -0,0 +1,1648 @@ +#!/usr/bin/env python3 +"""Generate broad blessing positives and difficult boundary negatives. + +The supplement supervises only the blessing label. It does not infer other +clipboard intents from synthetic text, so unrelated classifier heads are not +trained on unknown labels. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import random +import re +import unicodedata +from collections import Counter +from dataclasses import dataclass +from pathlib import Path + + +SEED = 20260827 +OUTPUT_DIRECTORY = Path("ModelTraining/ClipboardSemantics") +BASE_CORPUS_PATH = OUTPUT_DIRECTORY / "combined-training-corpus.jsonl" +SUPPLEMENT_PATH = OUTPUT_DIRECTORY / "blessing-training-supplement.jsonl" +COMBINED_PATH = OUTPUT_DIRECTORY / "combined-training-corpus-with-blessing.jsonl" +SUMMARY_PATH = OUTPUT_DIRECTORY / "blessing-training-summary.json" +DEFAULT_RECORDS_PER_LANGUAGE = 50_000 +SOURCE_REVISION = "2026-08-27-v1" + +SENSITIVE_PATTERN = re.compile( + r"(?:[\w.+-]+@[\w.-]+\.\w+)|(?:\+?\d[\d ()-]{8,}\d)|" + r"(?:\b\d{3}-\d{2}-\d{4}\b)", + re.IGNORECASE, +) + + +@dataclass(frozen=True) +class Family: + name: str + blessing: bool + templates: tuple[str, ...] + slots: dict[str, tuple[str, ...]] + + +ZH_COMMON = { + "prefix": ( + "", + "对了，", + "刚看到消息，", + "专门来跟你说一声，", + "今天这个特别的日子里，", + "虽然隔着屏幕，", + "群里冒个泡，", + "认真说一句，", + "简单但真心地说，", + "借这个机会，", + "微信里单独说一句，", + "趁现在有空，", + "想了想还是要说，", + "不复制群发文案，真心说一句，", + "冒个泡送句话，", + "今天第一条消息，", + "睡前想起这件事，", + "看到日历才发现，", + "刚和家里人聊到，", + "替没到场的大家说一句，", + "不用回复，收下这句话就好，", + "隔了好久没联系，", + "赶在零点之前，", + "早早来占个位置，", + "迟到但不会缺席，", + "不绕弯子，", + "发条短消息，", + "想到你就来留言，", + "今天值得认真记住，", + "把这份心意放在这里，", + ), + "recipient": ( + "你", + "您", + "大家", + "家人们", + "小伙伴们", + "老师", + "师傅", + "叔叔阿姨", + "爷爷奶奶", + "同学们", + "同事们", + "项目组", + "新郎新娘", + "新手爸妈", + "毕业班同学", + "今天的寿星", + ), + "tail": ( + "", + "！", + "呀！", + "～", + "，抱抱！", + "，一切顺利！", + "，开心最重要！", + "，记得照顾好自己。", + "，等你的好消息。", + "，未来继续闪闪发光。", + "，真心的。", + "，这句不是群发。", + "，收下我的心意。", + "，今天也要元气满满。", + "，我们改天见。", + "，有空再慢慢聊。", + "，一定要幸福。", + "，别忘了给自己放个假。", + "，愿生活温柔待你。", + "，家里人也都惦记着你。", + "，这份开心值得纪念。", + "，先把好消息记下来。", + "，接下来的日子加油。", + "，希望很快见到你。", + ), +} + +EN_COMMON = { + "prefix": ( + "", + "Just wanted to say: ", + "A quick note: ", + "Thinking of you today—", + "From across the miles, ", + "On this special day, ", + "Dropping in to say ", + "I mean this sincerely: ", + "Before the day ends, ", + "Sending a little note: ", + "A message just for you: ", + "No copied group message—", + "While I have a quiet minute, ", + "Before midnight, ", + "A little late, but sincerely: ", + "Starting the day with this: ", + "One honest sentence: ", + "I saw the date and thought of you—", + "Passing along a note from all of us: ", + "No need to reply; ", + "It has been a while, but ", + "Saving this moment with a message: ", + "I will keep it simple: ", + "A small message with a lot of heart: ", + "I could not let today pass without saying ", + "From everyone who could not be there, ", + "One last message before the day ends: ", + "This is not a formal card, just ", + "I thought of you and wanted to say ", + "Leaving this here for you: ", + ), + "recipient": ( + "you", + "all of you", + "everyone", + "our family", + "the whole team", + "Professor Lee", + "our teachers", + "the newlyweds", + "the new parents", + "the graduating class", + "today's birthday star", + "my dear friend", + "our colleagues", + "your family", + ), + "tail": ( + "", + "!", + "—you deserve it!", + " Take good care.", + " Here's to what comes next.", + " I am cheering for you.", + " Hope the good news keeps coming.", + " Wishing you all the best.", + " Enjoy every moment.", + " You have got this.", + " This comes from the heart.", + " No reply needed.", + " Keep this little note.", + " We will celebrate properly soon.", + " Take a well-earned break.", + " May life be gentle with you.", + " Everyone here is thinking of you.", + " This moment deserves to be remembered.", + " Keep going at your own pace.", + " I hope we see each other soon.", + " Consider this a warm message from afar.", + " There is more good ahead.", + " Be kind to yourself today.", + " Let us catch up soon.", + ), +} + +ZH_NEGATIVE_SURFACE = { + "prefix": ( + "", + "备注一下，", + "文档里写着，", + "有人问，", + "群里提到，", + "记录显示，", + "顺便说明，", + "只是在讨论，", + "从文本分类角度看，", + "这是一条例句：", + "搜索结果显示，", + "材料中提到，", + "会议上有人说，", + "聊天记录里出现了这句话：", + "作为反例，", + "需要确认的是，", + "这不是实际发送的消息，", + "这里只记录事实：", + "标题写的是，", + "页面上显示，", + ), + "tail": ( + "", + "。", + "，仅供参考。", + "，只是客观记录。", + "，上下文仍在讨论。", + "，无需回复。", + "，后续还要人工确认。", + "，原文到这里结束。", + "，没有更多说明。", + "，这是列表中的一项。", + ), +} + +EN_NEGATIVE_SURFACE = { + "prefix": ( + "", + "For the record, ", + "The document says ", + "Someone asked whether ", + "The group mentioned that ", + "The log shows that ", + "For classification purposes, ", + "This is only a discussion: ", + "Here is a quoted example: ", + "The search result says ", + "The meeting notes report that ", + "As a negative example, ", + "This was not an actual message: ", + "The page displays ", + "The title says ", + "To clarify the context, ", + "This line only records that ", + "The material notes that ", + "A reviewer noted that ", + "The transcript contains this phrase: ", + ), + "tail": ( + "", + ".", + "; this is only a reference.", + "; this only records what happened.", + "; the context is still under discussion.", + "; no reply is required.", + "; a reviewer still needs to verify it.", + "; the original line ends here.", + "; there is no further explanation.", + "; this is one item in the list.", + ), +} + +ZH_NEGATIVE_CONTEXT = ( + "", + "这是聊天记录的一部分。", + "前后还有其他内容。", + "这里只保留原句。", + "消息没有继续展开。", + "记录到这里结束。", + "这是页面中的一行文字。", + "原文仍在等待确认。", + "上下文未显示接收人。", + "这句话单独出现在列表里。", +) + +EN_NEGATIVE_CONTEXT = ( + "", + "This is part of a longer transcript.", + "There is more context before and after it.", + "Only the original line is retained.", + "The message does not continue.", + "The record ends here.", + "This is one line from the page.", + "The original text still needs confirmation.", + "The surrounding context names no recipient.", + "The sentence appears alone in a list.", +) + + +ZH_FAMILIES = ( + Family( + "festival", + True, + ( + "{prefix}祝{recipient}{occasion}快乐，{wish}{tail}", + "{prefix}{occasion}到了，愿{recipient}{wish}{tail}", + "{prefix}{recipient}，{occasion}快乐，愿往后的日子{wish}{tail}", + "{prefix}给{recipient}拜个节，祝{wish}{tail}", + "{prefix}这个{occasion}，把最真诚的祝愿送给{recipient}：{wish}{tail}", + ), + { + "occasion": ( + "春节", + "新年", + "元旦", + "元宵节", + "端午节", + "中秋节", + "国庆节", + "重阳节", + "教师节", + "母亲节", + "父亲节", + "圣诞节", + ), + "wish": ( + "平安喜乐", + "阖家幸福", + "万事顺遂", + "身体健康", + "好运常在", + "所求皆如愿", + "每天都有好心情", + "日子越过越红火", + "工作生活都顺心", + "团团圆圆、幸福安康", + ), + }, + ), + Family( + "birthday", + True, + ( + "{prefix}{recipient}生日快乐，愿新的一岁{wish}{tail}", + "{prefix}祝今天的{recipient}生日快乐，{wish}{tail}", + "{prefix}又长大一岁啦，愿{recipient}{wish}{tail}", + "{prefix}生日这天，把一句{wish}送给{recipient}{tail}", + "{prefix}Happy birthday，愿{recipient}这一岁{wish}{tail}", + ), + { + "wish": ( + "有爱有梦有期待", + "健康自在", + "被温柔和好运包围", + "做喜欢的事，见想见的人", + "烦恼少一点，快乐多很多", + "心想事成", + "一路有花也有掌声", + "比去年更勇敢更从容", + "收获满满的幸福", + "每天都值得纪念", + ), + }, + ), + Family( + "congratulation", + True, + ( + "{prefix}{congrats}，{achievement}{tail}", + "{prefix}{achievement}，必须说一句{congrats}{tail}", + "{prefix}听说{achievement}，真心替{recipient}高兴，{congrats}{tail}", + "{prefix}{congrats}！愿{recipient}接下来{wish}{tail}", + "{prefix}可喜可贺，{achievement}，继续加油{tail}", + ), + { + "congrats": ( + "恭喜", + "恭喜你", + "恭喜恭喜", + "祝贺你", + "太棒了，恭喜", + "真替你开心", + "可喜可贺", + "必须恭喜一下", + ), + "achievement": ( + "顺利毕业", + "成功上岸", + "拿到心仪的 offer", + "升职加薪", + "比赛夺冠", + "项目顺利上线", + "论文通过答辩", + "考试取得好成绩", + "新店正式开业", + "搬进新家", + "领证结婚", + "宝宝平安出生", + "通过重要认证", + "完成第一次马拉松", + "作品获奖", + ), + "wish": ( + "再创佳绩", + "前程似锦", + "一路开挂", + "越来越好", + "继续闪闪发光", + "每一步都走得坚定", + "收获更多好消息", + "未来皆是坦途", + ), + }, + ), + Family( + "wedding_family", + True, + ( + "{prefix}祝{recipient}{occasion}，{wish}{tail}", + "{prefix}{occasion}，愿{recipient}{wish}{tail}", + "{prefix}恭喜{recipient}迎来{occasion}，祝{wish}{tail}", + "{prefix}把最好的祝福送给{recipient}：{wish}{tail}", + ), + { + "occasion": ( + "新婚快乐", + "结婚纪念日快乐", + "喜得贵子", + "喜迎千金", + "成为幸福的新手爸妈", + "家庭新成员平安到来", + ), + "wish": ( + "一家人平安幸福", + "往后的日子温暖有爱", + "小家越来越温馨", + "朝朝暮暮皆是欢喜", + "新阶段顺顺利利", + "生活充满爱和欢笑", + "喜乐常伴", + "每一天都有新的幸福", + ), + }, + ), + Family( + "health_recovery", + True, + ( + "{prefix}祝{recipient}{recovery}{tail}", + "{prefix}愿{recipient}{recovery}，{wish}{tail}", + "{prefix}听说身体不舒服，希望{recipient}{recovery}{tail}", + "{prefix}把健康的祝愿送给{recipient}，愿{wish}{tail}", + "{prefix}替大家祝愿{recipient}{recovery}{tail}", + ), + { + "recipient": ( + "你", + "您", + "妈妈", + "爸爸", + "爷爷奶奶", + "住院的朋友", + "刚做完手术的他", + "正在休养的她", + ), + "recovery": ( + "早日康复", + "手术顺利", + "检查结果一切正常", + "身体一天比一天好", + "平安度过恢复期", + "很快恢复精神", + "少些疼痛，多些轻松", + "顺顺利利出院", + ), + "wish": ( + "平安健康", + "安心休养", + "每天都有新的好转", + "身心都慢慢恢复", + "被关心和温暖包围", + "很快回到喜欢的生活", + ), + }, + ), + Family( + "travel_safety", + True, + ( + "{prefix}祝{recipient}{travel_wish}{tail}", + "{prefix}出发啦，愿{recipient}{travel_wish}{tail}", + "{prefix}一路顺风，祝{recipient}{travel_wish}{tail}", + "{prefix}愿这趟旅程{travel_wish}，玩得开心{tail}", + ), + { + "travel_wish": ( + "一路平安", + "旅途顺利", + "一路顺风", + "平安到达", + "出入平安", + "看见好风景也遇见好心情", + "一路少奔波、多惊喜", + "行程顺利圆满", + ), + }, + ), + Family( + "study_career", + True, + ( + "{prefix}祝{recipient}{goal}{tail}", + "{prefix}愿{recipient}{goal}，{wish}{tail}", + "{prefix}明天就要{event}了，祝{recipient}{goal}{tail}", + "{prefix}为{recipient}加油，愿{goal}{tail}", + ), + { + "event": ( + "考试", + "面试", + "答辩", + "比赛", + "演讲", + "入职", + "签约", + "项目发布", + ), + "goal": ( + "考试顺利", + "面试成功", + "答辩顺利", + "比赛发挥出色", + "工作蒸蒸日上", + "事业更上一层楼", + "新工作一切顺心", + "项目顺利上线", + ), + "wish": ( + "付出都有回报", + "实力被看见", + "从容发挥", + "拿到满意的结果", + "未来大有可为", + "一路成长一路收获", + ), + }, + ), + Family( + "good_luck_short", + True, + ( + "{prefix}{short_wish}{tail}", + "{prefix}送{recipient}一句：{short_wish}{tail}", + "{prefix}今天也要{short_wish}{tail}", + "{prefix}真心希望{recipient}{short_wish}{tail}", + ), + { + "short_wish": ( + "祝你好运", + "一切顺利", + "心想事成", + "万事胜意", + "诸事顺遂", + "前程似锦", + "平安喜乐", + "得偿所愿", + "未来可期", + "好事连连", + "福气满满", + "顺顺利利", + "愿望成真", + "所行皆坦途", + "多喜乐，长安宁", + ), + }, + ), + Family( + "day_night", + True, + ( + "{prefix}祝{recipient}{daily_wish}{tail}", + "{prefix}愿{recipient}{daily_wish}{tail}", + "{prefix}{daily_wish}，明天见{tail}", + "{prefix}今天辛苦了，祝{recipient}{daily_wish}{tail}", + ), + { + "daily_wish": ( + "今晚睡个好觉", + "做个甜甜的好梦", + "明天心情明朗", + "今天过得开心", + "周末轻松愉快", + "新的一周顺顺利利", + "每一天都有小惊喜", + "今晚安心入睡", + ), + }, + ), + Family( + "third_person_prayer", + True, + ( + "{prefix}我衷心祝愿{third_person}{wish}{tail}", + "{prefix}愿{third_person}{wish}{tail}", + "{prefix}请替我转告{third_person}，祝{wish}{tail}", + "{prefix}我们一起为{third_person}祈愿，愿{wish}{tail}", + ), + { + "third_person": ( + "她", + "他", + "孩子", + "新郎新娘", + "叔叔阿姨", + "住院的朋友", + "远方的家人", + "参加考试的同学", + "刚入职的伙伴", + "整个团队", + ), + "wish": ( + "早日康复", + "平安健康", + "一切顺利", + "渡过难关", + "前程似锦", + "家庭幸福", + "收获理想的结果", + "每天多一点轻松和快乐", + "被好运和善意包围", + "往后的生活越来越好", + ), + }, + ), + Family( + "opening_home", + True, + ( + "{prefix}恭喜{recipient}{occasion}，祝{wish}{tail}", + "{prefix}祝贺{occasion}，愿{recipient}{wish}{tail}", + "{prefix}{occasion}是新的开始，祝{wish}{tail}", + "{prefix}送上祝福：{occasion}，愿{wish}{tail}", + ), + { + "occasion": ( + "乔迁新居", + "新店开业", + "公司成立", + "工作室开张", + "新项目启动", + "搬进新办公室", + ), + "wish": ( + "新的开始一切顺利", + "未来蒸蒸日上", + "每一步都有好收获", + "人气旺、好运旺", + "日子越过越红火", + "一切都朝着好方向发展", + "万事顺遂", + "新的空间带来新的惊喜", + ), + }, + ), + Family( + "emoji_colloquial", + True, + ( + "{prefix}{recipient}，{casual_wish}{emoji}{tail}", + "{prefix}{casual_wish}，这条好运请收下{emoji}{tail}", + "{prefix}隔空给{recipient}送祝福：{casual_wish}{emoji}{tail}", + "{prefix}不说套话，只希望{recipient}{casual_wish}{emoji}{tail}", + ), + { + "casual_wish": ( + "每天都开开心心", + "好运爆棚", + "好事正在路上", + "今年比去年更快乐", + "想做的事都能做到", + "吃好睡好没烦恼", + "钱包鼓鼓、心情美美", + "一路升级打怪都顺利", + "快乐加倍、烦恼清零", + "被爱也被好运围住", + ), + "emoji": ("", "🎉", "✨", "❤️", "🌟", "🍀", "🥳", "🎂", "💐", "🙏"), + }, + ), + Family( + "meta_request", + False, + ( + "{prefix}帮我写一段给{recipient}的{occasion}祝福语{tail}", + "{prefix}有没有适合{occasion}发微信的祝福文案{tail}", + "{prefix}搜索一下“{occasion}祝福语”{tail}", + "{prefix}这篇文章讲的是怎么写{occasion}祝福{tail}", + "{prefix}请把{occasion}祝福模板整理到文档里{tail}", + ), + { + "occasion": ( + "生日", + "春节", + "婚礼", + "毕业", + "升职", + "乔迁", + "康复", + "开业", + "考试", + "旅行", + ), + }, + ), + Family( + "received_thanks", + False, + ( + "{prefix}谢谢{recipient}发来的祝福{tail}", + "{prefix}今天收到了很多{occasion}祝福，统一感谢大家{tail}", + "{prefix}你的祝福我已经收到啦{tail}", + "{prefix}群里都在回复大家的{occasion}祝福{tail}", + "{prefix}感谢所有人记得我的{occasion}{tail}", + ), + { + "occasion": ( + "生日", + "新年", + "婚礼", + "毕业", + "入职", + "开业", + "乔迁", + "纪念日", + ), + }, + ), + Family( + "celebration_not_wish", + False, + ( + "{prefix}我们应该找个时间好好庆祝一下{tail}", + "{prefix}{occasion}庆祝活动安排在{time}{tail}", + "{prefix}他们一定是在庆祝{occasion}{tail}", + "{prefix}我买了蛋糕和红酒准备庆祝{occasion}{tail}", + "{prefix}庆祝方式已经由活动组确定{tail}", + ), + { + "occasion": ( + "项目上线", + "生日", + "新店开业", + "毕业", + "比赛胜利", + "结婚纪念日", + "搬家", + "签约成功", + ), + "time": ( + "今晚", + "周五晚上", + "下班以后", + "这个周末", + "下周聚会时", + ), + }, + ), + Family( + "quoted_documented", + False, + ( + "{prefix}文档里引用了“{quoted_wish}”这句话{tail}", + "{prefix}示例文本是“{quoted_wish}”{tail}", + "{prefix}老师让我们分析“{quoted_wish}”的句式{tail}", + "{prefix}海报上印着“{quoted_wish}”{tail}", + "{prefix}关键词列表里包含“{quoted_wish}”{tail}", + ), + { + "quoted_wish": ( + "祝你生日快乐", + "愿你平安顺遂", + "恭喜发财", + "早日康复", + "一路顺风", + "新婚快乐", + "前程似锦", + "万事如意", + ), + }, + ), + Family( + "ordinary_greeting", + False, + ( + "{prefix}{greeting}{tail}", + "{prefix}{recipient}，{greeting}{tail}", + "{prefix}群里打个招呼：{greeting}{tail}", + "{prefix}只是来问候一下，{greeting}{tail}", + ), + { + "greeting": ( + "你好", + "早上好", + "下午好", + "晚上好", + "最近怎么样", + "好久不见", + "吃饭了吗", + "在忙吗", + "周末有空吗", + "看到消息回我一下", + ), + }, + ), + Family( + "positive_feedback", + False, + ( + "{prefix}{recipient}这次做得真不错{tail}", + "{prefix}必须夸一下，{achievement}太棒了{tail}", + "{prefix}这个结果让我很满意{tail}", + "{prefix}{recipient}的表现超出预期{tail}", + "{prefix}大家都觉得{achievement}完成得很好{tail}", + ), + { + "achievement": ( + "项目上线", + "活动组织", + "演讲", + "设计方案", + "客户沟通", + "问题处理", + "比赛表现", + "课程展示", + ), + }, + ), + Family( + "future_intent", + False, + ( + "{prefix}等会儿再祝{recipient}{occasion}快乐{tail}", + "{prefix}我还没想好怎么祝{recipient}{occasion}快乐{tail}", + "{prefix}到时候记得给{recipient}发祝福{tail}", + "{prefix}祝福的话留到见面再说{tail}", + "{prefix}先收集素材，之后再写{occasion}祝福{tail}", + ), + { + "occasion": ( + "生日", + "新年", + "婚礼", + "毕业", + "升职", + "乔迁", + ), + }, + ), + Family( + "sarcastic_conditional", + False, + ( + "{prefix}那我可真要“恭喜”{recipient}了{tail}", + "{prefix}恭喜什么，事情还没定呢{tail}", + "{prefix}如果通过了再说恭喜也不迟{tail}", + "{prefix}先别祝我好运，结果还不知道{tail}", + "{prefix}这句“祝你成功”听起来全是反话{tail}", + ), + {}, + ), + Family( + "lexical_collision", + False, + ( + "{prefix}{term}是这份资料里的专有名词{tail}", + "{prefix}系统正在搜索{term}相关内容{tail}", + "{prefix}标题中出现了{term}，正文并没有表达祝愿{tail}", + "{prefix}请统计{term}这个词出现了多少次{tail}", + ), + { + "term": ( + "祝福", + "祝愿", + "恭喜", + "生日快乐", + "好运", + "庆祝", + "祝融", + "祈福", + "新年快乐", + "一路顺风", + ), + }, + ), + Family( + "reported_third_party", + False, + ( + "{prefix}他说想祝{recipient}{occasion}快乐{tail}", + "{prefix}会议记录称大家向{recipient}表达了祝福{tail}", + "{prefix}新闻里提到许多人祝愿活动成功{tail}", + "{prefix}群公告要求每个人准备一句祝福{tail}", + "{prefix}她转述了别人对{recipient}的祝愿{tail}", + ), + { + "occasion": ("生日", "新年", "毕业", "婚礼", "升职", "乔迁"), + }, + ), + Family( + "wish_word_not_blessing", + False, + ( + "{prefix}我的愿望清单还没写完{tail}", + "{prefix}这个功能满足了用户的愿望{tail}", + "{prefix}他希望明天不要下雨{tail}", + "{prefix}项目组希望预算能够获批{tail}", + "{prefix}我希望文件今天能传完{tail}", + "{prefix}愿不愿意参加还要再考虑{tail}", + ), + {}, + ), +) + + +EN_FAMILIES = ( + Family( + "occasion", + True, + ( + "{prefix}happy {occasion}, {recipient}{tail}", + "{prefix}wishing {recipient} a wonderful {occasion}{tail}", + "{prefix}may this {occasion} bring {recipient} {wish}{tail}", + "{prefix}sending warm {occasion} wishes to {recipient}{tail}", + ), + { + "occasion": ( + "birthday", + "New Year", + "Christmas", + "anniversary", + "graduation", + "wedding day", + "retirement", + ), + "wish": ( + "good health and happiness", + "peace and joy", + "many happy memories", + "success in everything ahead", + "love and laughter", + "a bright new chapter", + "all the good things you deserve", + ), + }, + ), + Family( + "congratulation", + True, + ( + "{prefix}{congrats} on {achievement}{tail}", + "{prefix}{achievement}—{congrats}{tail}", + "{prefix}I am so happy for {recipient}; {congrats} on {achievement}{tail}", + "{prefix}{congrats}! May what comes next be even better{tail}", + ), + { + "congrats": ( + "congratulations", + "congrats", + "huge congratulations", + "well done and congratulations", + "so happy for you", + ), + "achievement": ( + "the new job", + "your promotion", + "graduating", + "passing the exam", + "winning the competition", + "the successful launch", + "your new home", + "the wedding", + "the new baby", + "finishing the marathon", + "the award", + "the accepted paper", + ), + }, + ), + Family( + "health", + True, + ( + "{prefix}wishing {recipient} {recovery}{tail}", + "{prefix}may {recipient} have {recovery}{tail}", + "{prefix}I sincerely hope {recipient} has {recovery}{tail}", + "{prefix}sending healing thoughts and wishing {recipient} {recovery}{tail}", + ), + { + "recipient": ( + "you", + "your mother", + "your father", + "our friend in hospital", + "the patient", + "her", + "him", + "your family", + ), + "recovery": ( + "a speedy recovery", + "good health", + "a smooth surgery", + "steady healing", + "comfort and strength", + "better days very soon", + "a safe return home", + "rest and renewed energy", + ), + }, + ), + Family( + "travel", + True, + ( + "{prefix}safe travels, {recipient}{tail}", + "{prefix}wishing {recipient} a safe and smooth journey{tail}", + "{prefix}have a wonderful trip and arrive safely{tail}", + "{prefix}may the road ahead be easy and full of good memories{tail}", + ), + {}, + ), + Family( + "study_career", + True, + ( + "{prefix}good luck with {event}{tail}", + "{prefix}wishing {recipient} every success in {event}{tail}", + "{prefix}hope {event} goes brilliantly for {recipient}{tail}", + "{prefix}may all your hard work pay off in {event}{tail}", + ), + { + "event": ( + "the exam", + "the interview", + "your presentation", + "the competition", + "your first day", + "the product launch", + "the final defense", + "the new role", + "the performance", + "the application", + ), + }, + ), + Family( + "general_short", + True, + ( + "{prefix}{short_wish}{tail}", + "{prefix}wishing {recipient} {short_wish}{tail}", + "{prefix}sending {recipient} one simple wish: {short_wish}{tail}", + "{prefix}I truly hope {short_wish} is waiting for {recipient}{tail}", + ), + { + "short_wish": ( + "good luck", + "all the best", + "every success", + "peace and happiness", + "health and joy", + "a bright future", + "wonderful things ahead", + "everything you hope for", + "many reasons to smile", + "a smooth road ahead", + "good fortune", + "dreams coming true", + ), + }, + ), + Family( + "day_night", + True, + ( + "{prefix}hope {daily_wish} is waiting for {recipient}{tail}", + "{prefix}wishing {recipient} {daily_wish}{tail}", + "{prefix}rest well and have {daily_wish}{tail}", + "{prefix}you have worked hard today; may you have {daily_wish}{tail}", + ), + { + "daily_wish": ( + "a lovely day", + "a peaceful night", + "sweet dreams", + "a relaxing weekend", + "a great week ahead", + "a calm evening", + "a brighter tomorrow", + "a restful night", + ), + }, + ), + Family( + "third_person", + True, + ( + "{prefix}I sincerely wish {third_person} {wish}{tail}", + "{prefix}may {third_person} have {wish}{tail}", + "{prefix}please pass my best wishes to {third_person} for {wish}{tail}", + "{prefix}we are all hoping {third_person} has {wish}{tail}", + ), + { + "third_person": ( + "her", + "him", + "the child", + "the newlyweds", + "the new parents", + "our friend in hospital", + "the graduating students", + "the whole team", + ), + "wish": ( + "a speedy recovery", + "good health", + "every success", + "peace and happiness", + "a wonderful future", + "the result they hope for", + "strength through this difficult time", + "many joyful days ahead", + ), + }, + ), + Family( + "meta_request", + False, + ( + "{prefix}write a {occasion} wish for {recipient}{tail}", + "{prefix}find me a message template for {occasion}{tail}", + "{prefix}how should I say happy {occasion} in a text{tail}", + "{prefix}this article explains how to write {occasion} wishes{tail}", + "{prefix}add the {occasion} greeting templates to the document{tail}", + ), + { + "occasion": ( + "birthday", + "New Year", + "wedding", + "graduation", + "promotion", + "housewarming", + "recovery", + "retirement", + ), + }, + ), + Family( + "received_thanks", + False, + ( + "{prefix}thank {recipient} for the kind wishes{tail}", + "{prefix}I received so many {occasion} wishes today{tail}", + "{prefix}thanks everyone for remembering my {occasion}{tail}", + "{prefix}your good wishes arrived safely{tail}", + "{prefix}the group is replying to all the {occasion} messages{tail}", + ), + { + "occasion": ( + "birthday", + "New Year", + "wedding", + "graduation", + "promotion", + "anniversary", + ), + }, + ), + Family( + "celebration", + False, + ( + "{prefix}we should celebrate {event} properly{tail}", + "{prefix}the celebration for {event} is scheduled for tomorrow{tail}", + "{prefix}they must be celebrating {event}{tail}", + "{prefix}I bought cake to celebrate {event}{tail}", + "{prefix}the events team finalized the celebration plan{tail}", + ), + { + "event": ( + "the launch", + "the birthday", + "the opening", + "graduation", + "the victory", + "the anniversary", + "the move", + "the signed contract", + ), + }, + ), + Family( + "quoted", + False, + ( + '{prefix}the document quotes "{quoted_wish}" as an example{tail}', + '{prefix}the sample text reads "{quoted_wish}"{tail}', + '{prefix}we are analyzing the wording of "{quoted_wish}"{tail}', + '{prefix}the poster has "{quoted_wish}" printed on it{tail}', + '{prefix}the keyword list includes "{quoted_wish}"{tail}', + ), + { + "quoted_wish": ( + "happy birthday", + "wishing you all the best", + "congratulations", + "get well soon", + "safe travels", + "happy wedding day", + "good luck", + "sweet dreams", + ), + }, + ), + Family( + "greeting", + False, + ( + "{prefix}{greeting}{tail}", + "{prefix}{greeting}, {recipient}{tail}", + "{prefix}just stopping by to say {greeting}{tail}", + "{prefix}a simple greeting: {greeting}{tail}", + ), + { + "greeting": ( + "hello", + "good morning", + "good afternoon", + "good evening", + "how are you", + "long time no see", + "are you around", + "what are you up to", + "did you see my message", + "how has your week been", + ), + }, + ), + Family( + "positive_feedback", + False, + ( + "{prefix}{recipient} did a great job on {work}{tail}", + "{prefix}the result of {work} exceeded expectations{tail}", + "{prefix}everyone was impressed by {work}{tail}", + "{prefix}I really liked how {recipient} handled {work}{tail}", + "{prefix}this is excellent work and deserves recognition{tail}", + ), + { + "work": ( + "the launch", + "the presentation", + "the design", + "the client call", + "the incident", + "the event", + "the report", + "the performance", + ), + }, + ), + Family( + "future_intent", + False, + ( + "{prefix}I will wish {recipient} a happy {occasion} later{tail}", + "{prefix}I have not decided what to write in the {occasion} message{tail}", + "{prefix}remember to send {recipient} a greeting when the time comes{tail}", + "{prefix}save the congratulations for the in-person meeting{tail}", + "{prefix}collect examples before drafting the {occasion} wish{tail}", + ), + { + "occasion": ( + "birthday", + "New Year", + "wedding", + "graduation", + "promotion", + "anniversary", + ), + }, + ), + Family( + "sarcasm", + False, + ( + "{prefix}well, I suppose I should \"congratulate\" {recipient}{tail}", + "{prefix}congratulations for what? Nothing is decided{tail}", + "{prefix}we can say good luck if the plan is approved{tail}", + "{prefix}do not wish me luck yet; the result is unknown{tail}", + "{prefix}that \"all the best\" sounded entirely sarcastic{tail}", + ), + {}, + ), + Family( + "reported", + False, + ( + "{prefix}she said she wanted to wish {recipient} a happy {occasion}{tail}", + "{prefix}the minutes say everyone sent {recipient} good wishes{tail}", + "{prefix}the article reports that fans congratulated the winner{tail}", + "{prefix}the announcement asks everyone to prepare a greeting{tail}", + "{prefix}he repeated someone else's wishes to {recipient}{tail}", + ), + { + "occasion": ( + "birthday", + "New Year", + "wedding", + "graduation", + "promotion", + "anniversary", + ), + }, + ), + Family( + "wish_word", + False, + ( + "{prefix}my wish list is not finished yet{tail}", + "{prefix}this feature satisfies a common user wish{tail}", + "{prefix}I hope the upload finishes today{tail}", + "{prefix}the team hopes the budget gets approved{tail}", + "{prefix}whether you wish to attend is still undecided{tail}", + "{prefix}the title contains the word congratulations{tail}", + ), + {}, + ), +) + + +def parse_arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--seed", type=int, default=SEED) + parser.add_argument( + "--records-per-language", + type=int, + default=DEFAULT_RECORDS_PER_LANGUAGE, + ) + parser.add_argument("--base-corpus", type=Path, default=BASE_CORPUS_PATH) + parser.add_argument("--output", type=Path, default=SUPPLEMENT_PATH) + parser.add_argument("--combined-output", type=Path, default=COMBINED_PATH) + parser.add_argument("--summary", type=Path, default=SUMMARY_PATH) + return parser.parse_args() + + +def normalized_text(value: str) -> str: + value = unicodedata.normalize("NFKC", value.replace("\u0000", " ")) + return " ".join(value.split()).strip() + + +def fingerprint(value: str) -> str: + return normalized_text(value).casefold() + + +def required_slots(template: str) -> tuple[str, ...]: + return tuple(sorted(set(re.findall(r"{([^{}]+)}", template)))) + + +def load_records(path: Path) -> list[dict]: + return [ + json.loads(line) + for line in path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + + +def holdout_fingerprints() -> set[str]: + values: set[str] = set() + for path in sorted(OUTPUT_DIRECTORY.glob("*holdout-corpus.jsonl")): + for record_value in load_records(path): + values.add(fingerprint(record_value["text"])) + return values + + +def allocate_targets(families: tuple[Family, ...], total: int) -> dict[str, int]: + positive = [family for family in families if family.blessing] + negative = [family for family in families if not family.blessing] + positive_total = total // 2 + negative_total = total - positive_total + + def distribute(values: list[Family], desired: int) -> dict[str, int]: + base, remainder = divmod(desired, len(values)) + return { + family.name: base + int(index < remainder) + for index, family in enumerate(values) + } + + return { + **distribute(positive, positive_total), + **distribute(negative, negative_total), + } + + +def make_record( + *, + language: str, + family: Family, + index: int, + text: str, +) -> dict: + language_id = "zh" if language == "zh-Hans" else "en" + return { + "id": f"targeted-blessing-{language_id}-{family.name}-{index:05d}", + "text": text, + "language": language, + "split": "train", + "family": f"targeted_blessing_{family.name}", + "task": False, + "question": False, + "invitation": False, + "complaint": False, + "scheduleNegotiation": False, + "confirmationDecision": False, + "followUpReminder": False, + "blessing": family.blessing, + "sentiment": "positive" if family.blessing else "neutral", + "replyable": False, + "knownLabels": ["blessing"], + "labelingMethod": ( + "AI-authored template family under blessing-labeling-guidelines.md" + ), + "sourceDataset": "OSGKeyboard broad blessing synthetic v1", + "sourceLicense": "OSGKeyboard project license", + "sourceURL": ( + "local://ModelTraining/ClipboardSemantics/" + "blessing-labeling-guidelines.md" + ), + "sourceRevision": SOURCE_REVISION, + "sourceSplit": "train", + } + + +def generate_language( + *, + language: str, + common_slots: dict[str, tuple[str, ...]], + families: tuple[Family, ...], + target: int, + seed: int, + reserved: set[str], +) -> list[dict]: + targets = allocate_targets(families, target) + records: list[dict] = [] + for family in families: + rng_seed = hashlib.sha256( + f"{seed}|{language}|{family.name}".encode() + ).digest() + rng = random.Random(rng_seed) + produced = 0 + attempts = 0 + desired = targets[family.name] + maximum_attempts = desired * 500 + slots = {**common_slots, **family.slots} + if not family.blessing: + negative_surface = ( + ZH_NEGATIVE_SURFACE + if language == "zh-Hans" + else EN_NEGATIVE_SURFACE + ) + slots.update(negative_surface) + while produced < desired and attempts < maximum_attempts: + attempts += 1 + template = rng.choice(family.templates) + values = { + key: rng.choice(slots[key]) + for key in required_slots(template) + } + text = normalized_text(template.format(**values)) + if not family.blessing: + context = rng.choice( + ZH_NEGATIVE_CONTEXT + if language == "zh-Hans" + else EN_NEGATIVE_CONTEXT + ) + text = normalized_text(f"{text} {context}") + text_key = fingerprint(text) + if ( + text_key in reserved + or not 2 <= len(text) <= 500 + or SENSITIVE_PATTERN.search(text) + ): + continue + reserved.add(text_key) + produced += 1 + records.append( + make_record( + language=language, + family=family, + index=produced, + text=text, + ) + ) + if produced != desired: + raise RuntimeError( + f"Only generated {produced}/{desired} unique records for " + f"{language} {family.name}" + ) + return records + + +def validate( + supplement: list[dict], + base_fingerprints: set[str], + holdouts: set[str], + target_per_language: int, +) -> None: + ids = [record_value["id"] for record_value in supplement] + texts = [fingerprint(record_value["text"]) for record_value in supplement] + if len(ids) != len(set(ids)): + raise ValueError("Duplicate blessing supplement IDs") + if len(texts) != len(set(texts)): + raise ValueError("Duplicate blessing supplement text") + if set(texts).intersection(base_fingerprints): + raise ValueError("Blessing supplement overlaps the base corpus") + if set(texts).intersection(holdouts): + raise ValueError("Blessing supplement overlaps a frozen holdout") + language_counts = Counter( + record_value["language"] for record_value in supplement + ) + if language_counts != { + "en": target_per_language, + "zh-Hans": target_per_language, + }: + raise ValueError(f"Unexpected language counts: {language_counts}") + for language in ("en", "zh-Hans"): + values = [ + record_value + for record_value in supplement + if record_value["language"] == language + ] + positive = sum(record_value["blessing"] for record_value in values) + if positive * 2 != len(values): + raise ValueError(f"Unbalanced blessing labels for {language}") + + +def write_jsonl(path: Path, records: list[dict]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + "\n".join( + json.dumps(record_value, ensure_ascii=False, sort_keys=True) + for record_value in records + ) + + "\n", + encoding="utf-8", + ) + + +def main() -> None: + arguments = parse_arguments() + if arguments.records_per_language < 1_000: + raise ValueError("--records-per-language must be at least 1000") + if arguments.records_per_language % 2: + raise ValueError("--records-per-language must be even") + + base_records = load_records(arguments.base_corpus) + base_fingerprints = { + fingerprint(record_value["text"]) for record_value in base_records + } + holdouts = holdout_fingerprints() + reserved = set(base_fingerprints) | holdouts + supplement = generate_language( + language="zh-Hans", + common_slots=ZH_COMMON, + families=ZH_FAMILIES, + target=arguments.records_per_language, + seed=arguments.seed, + reserved=reserved, + ) + generate_language( + language="en", + common_slots=EN_COMMON, + families=EN_FAMILIES, + target=arguments.records_per_language, + seed=arguments.seed, + reserved=reserved, + ) + validate( + supplement, + base_fingerprints, + holdouts, + arguments.records_per_language, + ) + write_jsonl(arguments.output, supplement) + combined = base_records + supplement + write_jsonl(arguments.combined_output, combined) + + family_counts = Counter( + record_value["family"] for record_value in supplement + ) + summary = { + "schemaVersion": 1, + "seed": arguments.seed, + "sourceRevision": SOURCE_REVISION, + "baseRecords": len(base_records), + "supplementRecords": len(supplement), + "combinedRecords": len(combined), + "labels": { + "positive": sum(record_value["blessing"] for record_value in supplement), + "negative": sum( + not record_value["blessing"] for record_value in supplement + ), + }, + "languages": dict( + sorted( + Counter( + record_value["language"] for record_value in supplement + ).items() + ) + ), + "families": dict(sorted(family_counts.items())), + "validation": { + "duplicateIDs": 0, + "duplicateNormalizedTexts": 0, + "baseCorpusOverlap": 0, + "frozenHoldoutOverlap": 0, + "containsUserClipboardData": False, + }, + "supplementSHA256": hashlib.sha256( + arguments.output.read_bytes() + ).hexdigest(), + "combinedSHA256": hashlib.sha256( + arguments.combined_output.read_bytes() + ).hexdigest(), + } + arguments.summary.parent.mkdir(parents=True, exist_ok=True) + arguments.summary.write_text( + json.dumps(summary, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + print(json.dumps(summary, ensure_ascii=False, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/Scripts/clipboard_semantics/generate_corpus.py b/Scripts/clipboard_semantics/generate_corpus.py index 3329e76..02ff744 100644 --- a/Scripts/clipboard_semantics/generate_corpus.py +++ b/Scripts/clipboard_semantics/generate_corpus.py @@ -13,6 +13,7 @@ import hashlib import json import random import re +import sys from collections import Counter from dataclasses import asdict, dataclass from pathlib import Path @@ -20,6 +21,7 @@ from typing import Iterable SEED = 20260821 +TRAIN_SAMPLE_SCALE = 1 TARGETS = {"train": 180, "validation": 45, "test": 45} FAMILY_TARGETS = { "quoted_question": {"train": 100, "validation": 45, "test": 45}, @@ -2091,6 +2093,7 @@ def generate_family( slots: dict[str, list[str]], labels: Labels, target: int, + minimum_target: int | None, seen: set[str], uses_discourse_prefixes: bool, ) -> Iterable[Record]: @@ -2144,11 +2147,19 @@ def generate_family( replyable=bool(labels.replyable), ) - if produced != target: + if produced != target and ( + minimum_target is None or produced < minimum_target + ): raise RuntimeError( f"Only generated {produced}/{target} unique records for " f"{family} {language} {split}" ) + if produced != target: + print( + f"Warning: generated all {produced} unique records available " + f"for {family} {language} {split}; requested {target}.", + file=sys.stderr, + ) def generate_records(profile: str) -> list[Record]: @@ -2174,6 +2185,9 @@ def generate_records(profile: str) -> list[Record]: split_templates = definition["templates"][language] family_targets = FAMILY_TARGETS.get(family, TARGETS) for split, target in family_targets.items(): + scaled_target = ( + target * TRAIN_SAMPLE_SCALE if split == "train" else target + ) records.extend( generate_family( family=family, @@ -2182,7 +2196,12 @@ def generate_records(profile: str) -> list[Record]: templates=split_templates[split], slots=slots, labels=labels, - target=target, + target=scaled_target, + minimum_target=( + target + if split == "train" and TRAIN_SAMPLE_SCALE > 1 + else None + ), seen=seen, uses_discourse_prefixes=profile == "expanded", ) @@ -2361,6 +2380,7 @@ def summary(records: list[Record]) -> dict[str, object]: sentiment_counts = Counter(record.sentiment for record in records) return { "seed": SEED, + "trainSampleScale": TRAIN_SAMPLE_SCALE, "total": len(records), "splits": dict(sorted(split_counts.items())), "languages": dict(sorted(language_counts.items())), @@ -2381,6 +2401,12 @@ def summary(records: list[Record]) -> dict[str, object]: def parse_arguments() -> argparse.Namespace: parser = argparse.ArgumentParser() + parser.add_argument( + "--train-sample-scale", + type=int, + default=1, + help="Multiply train records per family without changing evaluation splits.", + ) parser.add_argument( "--profile", choices=("baseline", "expanded"), @@ -2400,7 +2426,12 @@ def parse_arguments() -> argparse.Namespace: def main() -> None: + global TRAIN_SAMPLE_SCALE + arguments = parse_arguments() + if arguments.train_sample_scale < 1: + raise ValueError("--train-sample-scale must be at least 1") + TRAIN_SAMPLE_SCALE = arguments.train_sample_scale records = generate_records(arguments.profile) validate(records) diff --git a/Scripts/clipboard_semantics/generate_open_training_corpus.py b/Scripts/clipboard_semantics/generate_open_training_corpus.py index 1fb46fa..3460fe3 100644 --- a/Scripts/clipboard_semantics/generate_open_training_corpus.py +++ b/Scripts/clipboard_semantics/generate_open_training_corpus.py @@ -16,11 +16,19 @@ import unicodedata import urllib.request import zipfile from collections import Counter, defaultdict +from dataclasses import dataclass from pathlib import Path +from typing import Callable from urllib.error import HTTPError, URLError +try: + from opencc import OpenCC +except ImportError: # Optional research dependency; the source is reported unavailable. + OpenCC = None + SEED = 20260827 +SAMPLE_SCALE = 1 OUTPUT_DIRECTORY = Path("ModelTraining/ClipboardSemantics") BASE_CORPUS_PATH = OUTPUT_DIRECTORY / "clipboard_semantic_corpus.jsonl" OPEN_CORPUS_PATH = OUTPUT_DIRECTORY / "open-training-corpus.jsonl" @@ -37,6 +45,13 @@ GOEMOTIONS_REVISION = "5d8f4ac97c873bde3a792ba4628f00bb9103d3e6" TIANJI_REVISION = "8043c8cbdfba10d1cfeb9a52b9eed0e3ea2c231b" BIRTHDAY_REVISION = "13134d2e67e624b38b9a9b6ce3cbce5011975bea" CFPB_REVISION = "e5fec64e1f0688e47699b9cf8c26fe4ed350123a" +OPENCLAW_GREETINGS_REVISION = "f4e3c0323d5c44235b62454706e06acd60ebeaae" +WECHAT_BLESSINGS_REVISION = "9ad614ada57aa710ae7f7fa5823d91e0fb57bc8c" +SNIPS_REVISION = "b86ac7f1577868c42158d0dec77db50956046696" +MINDS14_REVISION = "40ce77cb32a384e4d50a568e1ec39ac804019d33" +BITOD_REVISION = "a9bd74de9eecdc3d875cb4ebf6a6beaf9c30c2ff" +RESTAURANT8K_REVISION = "57ec275d8078af65b7731c2a98be812d844a6d6b" +FORMOSA_NLU_REVISION = "03a337b61a200ab690994dca4dc31aa7f209800e" MASSIVE_ARCHIVE_URL = ( "https://amazon-massive-nlu-dataset.s3.amazonaws.com/" @@ -58,8 +73,25 @@ INTENT_LABELS = ( "followUpReminder", "blessing", "replyableMessage", + "assistantCommand", + "informationQuery", + "systemNotification", ) -ALL_LABELS = (*INTENT_LABELS, "sentiment") +DOMAINS = ( + "finance", + "travel", + "calendar", + "communication", + "media", + "smartHome", + "shopping", + "dining", + "health", + "weather", + "accountService", + "generalKnowledge", +) +ALL_LABELS = (*INTENT_LABELS, "sentiment", "domain") MASSIVE_INTENT_NAMES = ( "datetime_query", @@ -123,34 +155,106 @@ MASSIVE_INTENT_NAMES = ( "play_podcasts", "lists_query", ) -MASSIVE_TASK_INTENTS = { +MASSIVE_ASSISTANT_COMMAND_INTENTS = { "iot_hue_lightchange", - "transport_ticket", "iot_wemo_off", - "email_addcontact", - "takeaway_order", "iot_hue_lightup", "lists_createoradd", "iot_wemo_on", "calendar_remove", - "email_sendemail", "iot_cleaning", + "iot_hue_lightdim", + "audio_volume_up", + "audio_volume_other", + "audio_volume_down", "iot_hue_lightoff", "iot_hue_lighton", + "play_music", + "play_radio", + "play_audiobook", + "play_game", + "play_podcasts", + "audio_volume_mute", "social_post", - "calendar_set", + "alarm_set", "alarm_remove", "lists_remove", - "transport_taxi", + "music_settings", "iot_coffee", } -MASSIVE_QUERY_INTENTS = { - name - for name in MASSIVE_INTENT_NAMES - if name.endswith("_query") - or name.startswith("qa_") - or name.startswith("recommendation_") - or name in {"cooking_recipe", "datetime_convert", "transport_traffic"} +MASSIVE_SERVICE_TASK_INTENTS = { + "transport_ticket", + "takeaway_order", + "transport_taxi", +} +MASSIVE_INFORMATION_QUERY_INTENTS = { + name for name in MASSIVE_INTENT_NAMES if name.endswith("_query") +} | { + name for name in MASSIVE_INTENT_NAMES if name.startswith("qa_") +} | { + name for name in MASSIVE_INTENT_NAMES if name.startswith("recommendation_") +} | { + "cooking_recipe", + "datetime_convert", + "transport_traffic", +} +MASSIVE_DOMAIN_BY_PREFIX = { + "weather": "weather", + "transport": "travel", + "calendar": "calendar", + "alarm": "calendar", + "email": "communication", + "social": "communication", + "music": "media", + "audio": "media", + "play": "media", + "iot": "smartHome", + "takeaway": "dining", + "cooking": "dining", + "news": "generalKnowledge", + "qa": "generalKnowledge", + "datetime": "generalKnowledge", +} + +MINDS14_INTENT_NAMES = ( + "abroad", + "address", + "app_error", + "atm_limit", + "balance", + "business_loan", + "card_issues", + "cash_deposit", + "direct_debit", + "freeze", + "high_value_payment", + "joint_account", + "latest_transactions", + "pay_bill", +) +MINDS14_INFORMATION_QUERY_INTENTS = { + "abroad", + "address", + "atm_limit", + "balance", + "latest_transactions", +} +MINDS14_ASSISTANT_COMMAND_INTENTS = { + "cash_deposit", + "direct_debit", + "freeze", + "high_value_payment", + "pay_bill", +} + +SNIPS_MAPPING = { + "PlayMusic": ("assistantCommand", "media"), + "AddToPlaylist": ("assistantCommand", "media"), + "GetWeather": ("informationQuery", "weather"), + "BookRestaurant": ("task", "dining"), + "SearchScreeningEvent": ("informationQuery", "media"), + "SearchCreativeWork": ("informationQuery", "media"), + "RateBook": (None, None), } GO_EMOTIONS_POSITIVE = {0, 1, 4, 5, 13, 15, 17, 18, 20, 21, 23} @@ -293,44 +397,6 @@ CLINC_QUESTION_INTENTS = { "who_made_you", } -ENGLISH_QUESTION_PATTERN = re.compile( - r"(?:\?$|^(?:what|when|where|which|who|why|how|can|could|would|" - r"do|does|did|has|have|is|are|will|should)\b)", - re.IGNORECASE, -) -CHINESE_QUESTION_PATTERN = re.compile( - r"(?:[吗呢么？]$|^(?:怎么|为什么|哪|谁|什么|是否|能否|" - r"可以|你能|有没有|是不是))" -) -ENGLISH_TASK_PATTERN = re.compile( - r"(?:^(?:please\s+)?(?:add|book|buy|call|cancel|change|check|" - r"create|delete|email|find|make|move|order|pay|post|remove|" - r"schedule|send|set|share|show|tell|text|transfer|update)\b|" - r"\b(?:can|could|would|will) you\b|\bplease\b)", - re.IGNORECASE, -) -CHINESE_TASK_PATTERN = re.compile( - r"(?:请|麻烦|帮我|帮忙|替我|给我|需要你|希望你|能否|可以帮|" - r"提醒|添加|安排|设置|删除|取消|发送|回复|联系)" -) -ENGLISH_SCHEDULE_PATTERN = re.compile( - r"\b(?:reschedule|move (?:the|our) (?:meeting|appointment)|" - r"what time|which day|available (?:on|at)|" - r"(?:monday|tuesday|wednesday|thursday|friday).{0,24}\bor\b|" - r"(?:meeting|appointment).{0,24}\bor\b)\b", - re.IGNORECASE, -) -CHINESE_SCHEDULE_PATTERN = re.compile( - r"(?:(?:改到|改成|改期|几点|什么时候|哪天|有空|方便).{0,18}" - r"(?:见面|开会|碰面|约|吃饭|出发)|" - r"(?:周[一二三四五六日天]|星期[一二三四五六日天]|明天|后天|今晚)" - r".{0,18}(?:还是|或者|或).{0,18})" -) -ENGLISH_CONFIRMATION_PATTERN = re.compile( - r"^(?:yes|no|sure|okay|ok|confirmed|go ahead|sounds good|" - r"let'?s do (?:it|that)|that works)(?:[.! ]|$)", - re.IGNORECASE, -) ENGLISH_BLESSING_PATTERN = re.compile( r"\b(?:happy birthday|happy new year|merry christmas|happy holidays|" r"best wishes|good luck|congratulations|congrats|wishing you|" @@ -359,9 +425,64 @@ META_BLESSING_PATTERN = re.compile( ) +@dataclass(frozen=True) +class SourceBuilder: + """A pinned, license-reviewed source that can fail independently.""" + + name: str + build: Callable[[int], list[dict]] + optional: bool = True + + +class SourceUnavailable(RuntimeError): + """A source cannot be built in the current environment.""" + + +def massive_mapping(intent: str) -> tuple[str | None, str | None]: + """Map only official MASSIVE intents with product-policy-safe semantics.""" + if intent in MASSIVE_ASSISTANT_COMMAND_INTENTS: + label = "assistantCommand" + elif intent in MASSIVE_INFORMATION_QUERY_INTENTS: + label = "informationQuery" + elif intent in MASSIVE_SERVICE_TASK_INTENTS: + label = "task" + else: + label = None + prefix = intent.split("_", 1)[0] + return label, MASSIVE_DOMAIN_BY_PREFIX.get(prefix) + + +def known_labels_for_mapping( + mapped_label: str | None, + domain: str | None, +) -> set[str]: + known = set() + if mapped_label: + # These official assistant/service intents are mutually exclusive under + # the product policy, so their negative evidence is also trustworthy. + known.update( + { + "assistantCommand", + "informationQuery", + "systemNotification", + } + ) + if mapped_label == "task": + known.add("task") + if domain: + known.add("domain") + return known + + def parse_arguments() -> argparse.Namespace: parser = argparse.ArgumentParser() parser.add_argument("--seed", type=int, default=SEED) + parser.add_argument( + "--sample-scale", + type=int, + default=1, + help="Multiply license-safe source sampling caps while preserving source balance.", + ) parser.add_argument("--base-corpus", type=Path, default=BASE_CORPUS_PATH) parser.add_argument("--open-output", type=Path, default=OPEN_CORPUS_PATH) parser.add_argument("--combined-output", type=Path, default=COMBINED_CORPUS_PATH) @@ -406,6 +527,14 @@ def normalized_text(value: str) -> str: return " ".join(normalized.split()).strip() +def traditional_to_simplified(value: str) -> str: + if OpenCC is None: + raise SourceUnavailable( + "FormosaNLU conversion requires opencc-python-reimplemented" + ) + return normalized_text(OpenCC("t2s").convert(value)) + + def fingerprint(value: str) -> str: return normalized_text(value).casefold() @@ -429,6 +558,10 @@ def stable_sample(records: list[dict], limit: int, seed: int, salt: str) -> list )[:limit] +def scaled_limit(value: int) -> int: + return value * SAMPLE_SCALE + + def limited_by_family( records: list[dict], *, @@ -470,7 +603,12 @@ def make_record( follow_up_reminder: bool = False, blessing: bool = False, replyable: bool = False, + assistant_command: bool = False, + information_query: bool = False, + system_notification: bool = False, sentiment: str = "neutral", + domain: str | None = None, + sample_weight: float = 1.0, ) -> dict | None: clean_text = normalized_text(text) if not eligible_text(clean_text): @@ -480,6 +618,12 @@ def make_record( raise ValueError(f"Unsupported known labels: {sorted(invalid_labels)}") if sentiment not in {"positive", "neutral", "negative"}: raise ValueError(f"Unsupported sentiment: {sentiment}") + if domain is not None and domain not in DOMAINS: + raise ValueError(f"Unsupported domain: {domain}") + if "domain" in known_labels and domain is None: + raise ValueError("Known domain requires a domain value") + if not 0 < sample_weight <= 1: + raise ValueError(f"Unsupported sample weight: {sample_weight}") return { "id": record_id, "text": clean_text, @@ -496,6 +640,11 @@ def make_record( "blessing": blessing, "sentiment": sentiment, "replyable": replyable, + "assistantCommand": assistant_command, + "informationQuery": information_query, + "systemNotification": system_notification, + "domain": domain, + "sampleWeight": sample_weight, "knownLabels": sorted(known_labels), "labelingMethod": labeling_method, "sourceDataset": source_dataset, @@ -526,11 +675,17 @@ def massive_records(seed: int) -> list[dict]: if source["partition"] != "train": continue intent = source["intent"] - is_reminder = intent == "alarm_set" + mapped_label, domain = massive_mapping(intent) sentiment_known = intent in {"music_dislikeness", "music_likeness"} - known = {"task", "question", "followUpReminder"} + known = known_labels_for_mapping(mapped_label, domain) + if intent == "general_greet": + # A greeting opens a conversation but does not itself express + # a wish for the recipient. + known.add("blessing") if sentiment_known: known.add("sentiment") + if not known: + continue record_value = make_record( record_id=f"open-massive-{config}-{source['id']}", text=source["utt"], @@ -543,9 +698,9 @@ def massive_records(seed: int) -> list[dict]: source_split="train", known_labels=known, labeling_method="official intent mapping", - task=intent in MASSIVE_TASK_INTENTS, - question=intent in MASSIVE_QUERY_INTENTS, - follow_up_reminder=is_reminder, + task=mapped_label == "task", + assistant_command=mapped_label == "assistantCommand", + information_query=mapped_label == "informationQuery", sentiment=( "negative" if intent == "music_dislikeness" @@ -553,21 +708,79 @@ def massive_records(seed: int) -> list[dict]: if intent == "music_likeness" else "neutral" ), + domain=domain, ) if record_value: records.append(record_value) archive.close() - return stable_sample( - [record for record in records if record["language"] == "en"], - 500, - seed, - "massive-en", - ) + stable_sample( - [record for record in records if record["language"] == "zh-Hans"], - 500, - seed, - "massive-zh", + # MASSIVE is the strongest license-reviewed bilingual source in this + # pipeline. Keep every official-train record with an audited label mapping; + # downstream source balancing controls its effective training weight. + return records + + +def openclaw_greeting_records(seed: int) -> list[dict]: + url = ( + "https://huggingface.co/datasets/trytax/openclaw-zh-greetings/" + f"resolve/{OPENCLAW_GREETINGS_REVISION}/data/greetings.jsonl" ) + records: list[dict] = [] + for line in fetch_bytes(url).decode("utf-8").splitlines(): + if not line.strip(): + continue + source = json.loads(line) + is_wish = source["label"] == "wish" + record_value = make_record( + record_id=f"open-openclaw-greetings-{source['id']}", + text=source["text"], + language="zh-Hans", + family=f"open_openclaw_{source['label']}", + source_dataset="openclaw-zh-greetings", + source_license="MIT", + source_url=( + "https://huggingface.co/datasets/trytax/openclaw-zh-greetings" + ), + source_revision=OPENCLAW_GREETINGS_REVISION, + source_split="train", + known_labels={"blessing"}, + labeling_method="official wish versus non-wish label", + blessing=is_wish, + ) + if record_value: + records.append(record_value) + return stable_sample(records, scaled_limit(100), seed, "openclaw-greetings") + + +def wechat_blessing_records(seed: int) -> list[dict]: + url = ( + "https://raw.githubusercontent.com/SWHL/WeChat-AutoSendBless/" + f"{WECHAT_BLESSINGS_REVISION}/assets/bless.txt" + ) + records: list[dict] = [] + for index, text in enumerate( + fetch_bytes(url).decode("utf-8-sig").splitlines(), + start=1, + ): + record_value = make_record( + record_id=f"open-wechat-blessing-{index}", + text=text, + language="zh-Hans", + family="open_wechat_new_year_blessing", + source_dataset="SWHL/WeChat-AutoSendBless templates", + source_license="MIT", + source_url=( + "https://github.com/SWHL/WeChat-AutoSendBless/" + f"blob/{WECHAT_BLESSINGS_REVISION}/assets/bless.txt" + ), + source_revision=WECHAT_BLESSINGS_REVISION, + source_split="templates", + known_labels={"blessing"}, + labeling_method="repository-authored blessing template", + blessing=True, + ) + if record_value: + records.append(record_value) + return stable_sample(records, scaled_limit(100), seed, "wechat-blessings") def cped_records(seed: int) -> list[dict]: @@ -644,11 +857,37 @@ def crosswoz_records(seed: int) -> list[dict]: general_intents = { str(act[1]) for act in acts if len(act) > 1 and act[0] == "General" } - question = "Request" in intents or bool( - CHINESE_QUESTION_PATTERN.search(text) + source_domains = { + str(act[1]).casefold() + for act in acts + if len(act) > 1 and act[0] != "General" + } + mapped_domains = { + "dining" + if value in {"餐厅", "restaurant"} + else "travel" + if value in { + "酒店", + "景点", + "地铁", + "出租", + "hotel", + "attraction", + "metro", + "taxi", + } + else None + for value in source_domains + } + mapped_domains.discard(None) + domain = next(iter(mapped_domains)) if len(mapped_domains) == 1 else None + information_query = "Request" in intents + known = known_labels_for_mapping( + "informationQuery" if information_query else None, + domain, ) - task = bool(CHINESE_TASK_PATTERN.search(text)) - known = {"task", "question"} + if not known: + continue record_value = make_record( record_id=f"open-crosswoz-{dialogue_id}-{turn_index}", text=text, @@ -666,17 +905,17 @@ def crosswoz_records(seed: int) -> list[dict]: source_revision=CROSSWOZ_REVISION, source_split="train", known_labels=known, - labeling_method="official dialogue acts plus conservative surface mapping", - task=task, - question=question, + labeling_method="official dialogue-act mapping only", + information_query=information_query, + domain=domain, ) if record_value: records.append(record_value) archive.close() return limited_by_family( records, - per_family=150, - total=600, + per_family=scaled_limit(150), + total=scaled_limit(600), seed=seed, salt="crosswoz", ) @@ -735,16 +974,13 @@ def go_emotions_records(seed: int) -> list[dict]: def multidogo_records(seed: int) -> list[dict]: domains = ("airline", "fastfood", "finance", "insurance", "media", "software") - non_task_intents = { - "contentonly", - "confirmation", - "openinggreeting", - "closinggreeting", - "thankyou", - "rejection", - "outofdomain", - "other", - "pleasantries", + domain_mapping = { + "airline": "travel", + "fastfood": "dining", + "finance": "finance", + "insurance": "accountService", + "media": "media", + "software": "accountService", } records: list[dict] = [] for domain in domains: @@ -762,15 +998,26 @@ def multidogo_records(seed: int) -> list[dict]: ): text = normalized_text(row["utterance"]) intent = row["intent"].casefold() - question = bool(ENGLISH_QUESTION_PATTERN.search(text)) or intent.startswith( - ("get", "check", "query") + information_query = intent.startswith(("get", "check", "query")) + task = domain in {"airline", "fastfood"} and intent.startswith( + ("book", "cancel", "change", "order", "reserve") + ) + assistant_command = domain in {"media", "software"} and intent.startswith( + ("activate", "deactivate", "install", "reset", "update") + ) + mapped_label = ( + "informationQuery" + if information_query + else "task" + if task + else "assistantCommand" + if assistant_command + else None + ) + known = known_labels_for_mapping( + mapped_label, + domain_mapping[domain], ) - confirmation = intent == "confirmation" - task = ( - intent not in non_task_intents - and not question - and not confirmation - ) or bool(ENGLISH_TASK_PATTERN.search(text)) record_value = make_record( record_id=f"open-multidogo-{domain}-{row['utteranceId']}", text=text, @@ -787,18 +1034,19 @@ def multidogo_records(seed: int) -> list[dict]: ), source_revision=MULTIDOGO_REVISION, source_split="train", - known_labels={"task", "question", "confirmationDecision"}, - labeling_method="official customer intent mapping", + known_labels=known, + labeling_method="official customer intent and domain mapping only", task=task, - question=question, - confirmation_decision=confirmation, + assistant_command=assistant_command, + information_query=information_query, + domain=domain_mapping[domain], ) if record_value: records.append(record_value) return limited_by_family( records, - per_family=100, - total=600, + per_family=scaled_limit(100), + total=scaled_limit(600), seed=seed, salt="multidogo", ) @@ -824,24 +1072,22 @@ def taskmaster_records(seed: int) -> list[dict]: dialogue_id = dialogue["conversation_id"] if dialogue_id not in train_ids: continue + instruction = str(dialogue["instruction_id"]).casefold() + domain = ( + "dining" + if any(value in instruction for value in ("pizza", "restaurant", "coffee")) + else "travel" + if any(value in instruction for value in ("uber", "auto")) + else "media" + if "movie" in instruction + else None + ) + if domain is None: + continue for utterance in dialogue["utterances"]: if utterance["speaker"] != "USER": continue text = normalized_text(utterance["text"]) - annotations = { - annotation["name"] - for segment in utterance.get("segments") or [] - for annotation in segment.get("annotations") or [] - } - question = bool(ENGLISH_QUESTION_PATTERN.search(text)) - confirmation = bool(ENGLISH_CONFIRMATION_PATTERN.search(text)) or any( - name.endswith((".accept", ".reject")) for name in annotations - ) - schedule = bool(ENGLISH_SCHEDULE_PATTERN.search(text)) - task = ( - bool(ENGLISH_TASK_PATTERN.search(text)) - or utterance["index"] == 0 - ) and not confirmation record_value = make_record( record_id=( f"open-taskmaster-{dialogue_id}-{utterance['index']}" @@ -857,24 +1103,16 @@ def taskmaster_records(seed: int) -> list[dict]: ), source_revision=TASKMASTER_REVISION, source_split="train", - known_labels={ - "task", - "question", - "scheduleNegotiation", - "confirmationDecision", - }, - labeling_method="official train split plus conservative surface mapping", - task=task, - question=question, - schedule_negotiation=schedule, - confirmation_decision=confirmation, + known_labels={"domain"}, + labeling_method="official train split and instruction domain only", + domain=domain, ) if record_value: records.append(record_value) return limited_by_family( records, - per_family=100, - total=700, + per_family=scaled_limit(100), + total=scaled_limit(700), seed=seed, salt="taskmaster", ) @@ -911,8 +1149,8 @@ def clinc_records(seed: int) -> list[dict]: archive.close() return limited_by_family( records, - per_family=10, - total=500, + per_family=scaled_limit(10), + total=scaled_limit(500), seed=seed, salt="clinc150", ) @@ -1047,7 +1285,7 @@ def cfpb_records(seed: int) -> list[dict]: ) if record_value and len(record_value["text"]) >= 40: candidates.append(record_value) - return stable_sample(candidates, 1_000, seed, "cfpb") + return stable_sample(candidates, scaled_limit(1_000), seed, "cfpb") def asap_records(seed: int) -> list[dict]: @@ -1086,21 +1324,285 @@ def asap_records(seed: int) -> list[dict]: ), source_revision=ASAP_REVISION, source_split="train", - known_labels={"complaint", "sentiment"}, + known_labels={"complaint", "sentiment", "domain"}, labeling_method="official star and aspect-sentiment labels", complaint=is_complaint, sentiment="negative" if is_complaint else "positive", + domain="dining", ) if record_value: (negative if is_complaint else positive).append(record_value) - return stable_sample(negative, 500, seed, "asap-negative") + stable_sample( + return stable_sample( + negative, + scaled_limit(500), + seed, + "asap-negative", + ) + stable_sample( positive, - 500, + scaled_limit(500), seed, "asap-positive", ) +def snips_records(seed: int) -> list[dict]: + records: list[dict] = [] + for intent, (mapped_label, domain) in SNIPS_MAPPING.items(): + url = ( + "https://raw.githubusercontent.com/sonos/nlu-benchmark/" + f"{SNIPS_REVISION}/2017-06-custom-intent-engines/{intent}/" + f"train_{intent}_full.json" + ) + payload = json.loads(fetch_bytes(url)) + for index, item in enumerate(payload[intent]): + text = "".join(segment["text"] for segment in item["data"]) + known = known_labels_for_mapping(mapped_label, domain) + if not known: + continue + record_value = make_record( + record_id=f"open-snips-{intent}-{index}", + text=text, + language="en", + family=f"open_snips_{intent.casefold()}", + source_dataset="SNIPS NLU Benchmark", + source_license="CC0-1.0", + source_url=url, + source_revision=SNIPS_REVISION, + source_split="train", + known_labels=known, + labeling_method="official intent mapping", + task=mapped_label == "task", + assistant_command=mapped_label == "assistantCommand", + information_query=mapped_label == "informationQuery", + domain=domain, + ) + if record_value: + records.append(record_value) + return limited_by_family( + records, + per_family=scaled_limit(250), + total=scaled_limit(1_500), + seed=seed, + salt="snips", + ) + + +def minds14_records(seed: int) -> list[dict]: + url = ( + "https://huggingface.co/datasets/PolyAI/minds14/resolve/" + f"{MINDS14_REVISION}/zh-CN/train-00000-of-00001.parquet" + ) + try: + import pyarrow.parquet as parquet + except ImportError as error: + raise SourceUnavailable( + "MInDS-14 requires optional pyarrow to read its pinned Parquet artifact" + ) from error + table = parquet.read_table( + io.BytesIO(fetch_bytes(url)), + columns=["transcription", "intent_class"], + ) + records: list[dict] = [] + for index, source in enumerate(table.to_pylist()): + raw_intent = source["intent_class"] + intent = ( + MINDS14_INTENT_NAMES[raw_intent] + if isinstance(raw_intent, int) + else str(raw_intent) + ) + mapped_label = ( + "informationQuery" + if intent in MINDS14_INFORMATION_QUERY_INTENTS + else "assistantCommand" + if intent in MINDS14_ASSISTANT_COMMAND_INTENTS + else None + ) + known = known_labels_for_mapping(mapped_label, "finance") + record_value = make_record( + record_id=f"open-minds14-zh-CN-{index}", + text=source["transcription"], + language="zh-Hans", + family=f"open_minds14_{intent}", + source_dataset="PolyAI MInDS-14 zh-CN", + source_license="CC-BY-4.0", + source_url=url, + source_revision=MINDS14_REVISION, + source_split="train", + known_labels=known, + labeling_method="official intent and banking-domain mapping", + assistant_command=mapped_label == "assistantCommand", + information_query=mapped_label == "informationQuery", + domain="finance", + ) + if record_value: + records.append(record_value) + return limited_by_family( + records, + per_family=scaled_limit(80), + total=scaled_limit(600), + seed=seed, + salt="minds14-zh-CN", + ) + + +def bitod_domain(active_intent: str) -> str | None: + normalized = active_intent.casefold() + if normalized.startswith("restaurants_") or normalized.startswith("餐馆"): + return "dining" + if normalized.startswith( + ("hotels_", "attractions_", "hkmtr_", "宾馆", "景点", "香港地铁") + ): + return "travel" + if normalized.startswith("weathers_") or normalized.startswith("天气"): + return "weather" + return None + + +def bitod_mapping(active_intent: str) -> tuple[str | None, str | None]: + normalized = active_intent.casefold() + domain = bitod_domain(active_intent) + is_search = ( + normalized.endswith("_search") + or normalized.endswith("查询") + or normalized == "香港地铁" + ) + is_booking = normalized.endswith("_booking") or normalized.endswith("预订") + label = ( + "informationQuery" + if is_search + else "task" + if is_booking + else None + ) + return label, domain + + +def bitod_records(seed: int) -> list[dict]: + records: list[dict] = [] + for file_name, language in ( + ("en_train.json", "en"), + ("zh_train.json", "zh-Hans"), + ): + url = ( + "https://raw.githubusercontent.com/HLTCHKUST/BiToD/" + f"{BITOD_REVISION}/data/{file_name}" + ) + payload = json.loads(fetch_bytes(url)) + for dialogue_id, dialogue in payload.items(): + for turn_index, turn in enumerate(dialogue["Events"]): + if turn.get("Agent") != "User": + continue + active_intent = str(turn.get("active_intent") or "") + mapped_label, domain = bitod_mapping(active_intent) + known = known_labels_for_mapping(mapped_label, domain) + if not known: + continue + record_value = make_record( + record_id=f"open-bitod-{language}-{dialogue_id}-{turn_index}", + text=turn.get("Text") or "", + language=language, + family=f"open_bitod_{active_intent or 'unknown'}", + source_dataset="HLTCHKUST/BiToD", + source_license="Apache-2.0", + source_url=url, + source_revision=BITOD_REVISION, + source_split="train", + known_labels=known, + labeling_method="official active-intent mapping", + task=mapped_label == "task", + information_query=mapped_label == "informationQuery", + domain=domain, + ) + if record_value: + records.append(record_value) + return limited_by_family( + records, + per_family=scaled_limit(120), + total=scaled_limit(1_500), + seed=seed, + salt="bitod", + ) + + +def restaurant8k_records(seed: int) -> list[dict]: + url = ( + "https://raw.githubusercontent.com/PolyAI-LDN/task-specific-datasets/" + f"{RESTAURANT8K_REVISION}/span_extraction/restaurant8k/train_0.json" + ) + records: list[dict] = [] + for index, source in enumerate(json.loads(fetch_bytes(url))): + record_value = make_record( + record_id=f"open-restaurant8k-{index}", + text=source.get("userInput", {}).get("text") or "", + language="en", + family="open_restaurant8k", + source_dataset="PolyAI RESTAURANTS-8K", + source_license="CC-BY-4.0", + source_url=url, + source_revision=RESTAURANT8K_REVISION, + source_split="train_0", + known_labels={"domain"}, + labeling_method="official dataset domain only; slots are not intents", + domain="dining", + ) + if record_value: + records.append(record_value) + return stable_sample( + records, + scaled_limit(1_000), + seed, + "restaurant8k", + ) + + +def formosa_nlu_records(seed: int) -> list[dict]: + url = ( + "https://huggingface.co/datasets/steven0226/" + "formosa-nlu-synth-v1/resolve/" + f"{FORMOSA_NLU_REVISION}/data/train.jsonl" + ) + records: list[dict] = [] + for source in ( + json.loads(line) + for line in fetch_bytes(url).decode("utf-8").splitlines() + if line.strip() + ): + intent = source["intent"] + mapped_label, domain = massive_mapping(intent) + known = known_labels_for_mapping(mapped_label, domain) + if not known: + continue + record_value = make_record( + record_id=f"open-formosanlu-{source['id']}", + text=traditional_to_simplified(source["utt"]), + language="zh-Hans", + family=f"open_formosanlu_{intent}", + source_dataset="FormosaNLU Synth v1", + source_license="CC-BY-4.0", + source_url=url, + source_revision=FORMOSA_NLU_REVISION, + source_split="train", + known_labels=known, + labeling_method=( + "official synthetic MASSIVE-intent mapping; OpenCC t2s conversion" + ), + task=mapped_label == "task", + assistant_command=mapped_label == "assistantCommand", + information_query=mapped_label == "informationQuery", + domain=domain, + sample_weight=0.35, + ) + if record_value: + records.append(record_value) + return limited_by_family( + records, + per_family=scaled_limit(80), + total=scaled_limit(2_000), + seed=seed, + salt="formosa-nlu", + ) + + def holdout_fingerprints(directory: Path) -> set[str]: fingerprints: set[str] = set() for path in sorted(directory.glob("*holdout-corpus.jsonl")): @@ -1133,9 +1635,15 @@ def source_summary(records: list[dict]) -> list[dict]: 1 for value in values if label in value["knownLabels"] and ( - value[label] - if label != "sentiment" + value[ + "replyable" + if label == "replyableMessage" + else label + ] + if label not in {"sentiment", "domain"} else value["sentiment"] != "neutral" + if label == "sentiment" + else value["domain"] is not None ) ) for label in ALL_LABELS @@ -1174,33 +1682,71 @@ def validate_records(records: list[dict], holdouts: set[str]) -> None: texts.add(text_key) -def main() -> None: - arguments = parse_arguments() - rng = random.Random(arguments.seed) - source_builders = ( - ("MASSIVE", massive_records), - ("CrossWOZ", crosswoz_records), - ("GoEmotions", go_emotions_records), - ("MultiDoGO", multidogo_records), - ("Taskmaster-1", taskmaster_records), - ("CLINC150", clinc_records), - ("CFPB", cfpb_records), - ("ASAP", asap_records), +def configured_source_builders() -> tuple[SourceBuilder, ...]: + return ( + SourceBuilder("MASSIVE", massive_records), + SourceBuilder("CrossWOZ", crosswoz_records), + SourceBuilder("GoEmotions", go_emotions_records), + SourceBuilder("MultiDoGO", multidogo_records), + SourceBuilder("Taskmaster-1", taskmaster_records), + SourceBuilder("ASAP", asap_records), + SourceBuilder("SNIPS", snips_records), + SourceBuilder("MInDS-14 zh-CN", minds14_records), + SourceBuilder("BiToD", bitod_records), + SourceBuilder("RESTAURANTS-8K", restaurant8k_records), + SourceBuilder("FormosaNLU Synth v1", formosa_nlu_records), + ) + + +def build_available_sources( + builders: tuple[SourceBuilder, ...], + seed: int, + allow_unavailable_sources: bool, +) -> tuple[list[list[dict]], list[dict[str, str]]]: + source_failures = ( + HTTPError, + URLError, + TimeoutError, + ConnectionError, + OSError, + KeyError, + ValueError, + json.JSONDecodeError, + tarfile.ReadError, + zipfile.BadZipFile, + SourceUnavailable, ) sources: list[list[dict]] = [] unavailable_sources: list[dict[str, str]] = [] - for source_name, builder in source_builders: + for builder in builders: try: - sources.append(builder(arguments.seed)) - except (HTTPError, URLError, TimeoutError, ConnectionError, OSError) as error: - if not arguments.allow_unavailable_sources: + sources.append(builder.build(seed)) + except source_failures as error: + if not (builder.optional or allow_unavailable_sources): raise unavailable_sources.append( { - "dataset": source_name, + "dataset": builder.name, "reason": f"{type(error).__name__}: {error}", } ) + return sources, unavailable_sources + + +def main() -> None: + global SAMPLE_SCALE + + arguments = parse_arguments() + if arguments.sample_scale < 1: + raise ValueError("--sample-scale must be at least 1") + SAMPLE_SCALE = arguments.sample_scale + rng = random.Random(arguments.seed) + source_builders = configured_source_builders() + sources, unavailable_sources = build_available_sources( + source_builders, + arguments.seed, + arguments.allow_unavailable_sources, + ) candidates = [record_value for source in sources for record_value in source] rng.shuffle(candidates) @@ -1300,6 +1846,27 @@ def main() -> None: "dataset": "Verified Enron Intent / GitHub issue holdout source", "reason": "No clean official train split independent from the frozen holdout.", }, + { + "dataset": "CFPB", + "reason": ( + "No official train split; the source is already represented in " + "frozen evaluation data and contains privacy-sensitive narratives." + ), + }, + { + "dataset": "CLINC150", + "reason": ( + "Kept isolated from product training because its assistant intents " + "are not aligned with the product-policy taxonomy." + ), + }, + { + "dataset": "openclaw-zh-greetings / WeChat-AutoSendBless", + "reason": ( + "Repository licenses do not establish a sufficiently clear, " + "versioned rights chain for the underlying message templates." + ), + }, ], "unavailableSources": unavailable_sources, "baseCorpusRecords": len(base_records), diff --git a/Scripts/clipboard_semantics/generate_v6_boundary_corpus.py b/Scripts/clipboard_semantics/generate_v6_boundary_corpus.py new file mode 100644 index 0000000..7602db8 --- /dev/null +++ b/Scripts/clipboard_semantics/generate_v6_boundary_corpus.py @@ -0,0 +1,722 @@ +#!/usr/bin/env python3 +"""Generate a deterministic bilingual supplement for taxonomy-v6 boundaries.""" + +from __future__ import annotations + +import argparse +import hashlib +import itertools +import json +import unicodedata +from collections import Counter +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable, Iterator, Sequence + + +OUTPUT_DIRECTORY = Path("ModelTraining/ClipboardSemantics") +DEFAULT_OUTPUT_PATH = OUTPUT_DIRECTORY / "v6-boundary-training-supplement.jsonl" +DEFAULT_SUMMARY_PATH = OUTPUT_DIRECTORY / "v6-boundary-training-supplement-summary.json" +DEFAULT_POSITIVE_PER_INTENT_LANGUAGE = 1_000 +DEFAULT_NEGATIVE_PER_INTENT_LANGUAGE = 200 +SOURCE_DATASET = "OSGKeyboard taxonomy-v6 deterministic boundary templates" +SOURCE_LICENSE = "OSGKeyboard project license" +SOURCE_REVISION = "v6-boundary-templates-1" +SOURCE_SPLIT = "train" +SAMPLE_WEIGHT = 0.35 + +NEW_INTENTS = ( + "assistantCommand", + "informationQuery", + "systemNotification", +) +HARD_NEGATIVE_INTENTS = ("task", "question", "replyableMessage") +BOUNDARY_INTENTS = NEW_INTENTS + HARD_NEGATIVE_INTENTS +INTENT_LABELS = ( + "task", + "question", + "invitation", + "complaint", + "scheduleNegotiation", + "confirmationDecision", + "followUpReminder", + "blessing", + "replyableMessage", + "assistantCommand", + "informationQuery", + "systemNotification", +) +KNOWN_LABELS = sorted((*BOUNDARY_INTENTS, "domain")) +DOMAINS = ( + "finance", + "travel", + "calendar", + "communication", + "media", + "smartHome", + "shopping", + "dining", + "health", + "weather", + "accountService", + "generalKnowledge", +) +LANGUAGES = ("en", "zh-Hans") +CATEGORY_OFFSETS = { + "assistantCommand": 0, + "informationQuery": 4, + "systemNotification": 8, + "task": 0, + "question": 4, + "replyableMessage": 8, +} + + +@dataclass(frozen=True) +class DomainSlots: + """Finite, project-authored slots for one product-policy domain.""" + + subjects_en: tuple[str, ...] + subjects_zh: tuple[str, ...] + facts_en: tuple[str, ...] + facts_zh: tuple[str, ...] + services_en: tuple[str, ...] + services_zh: tuple[str, ...] + providers_en: tuple[str, ...] + providers_zh: tuple[str, ...] + events_en: tuple[str, ...] + events_zh: tuple[str, ...] + + +DOMAIN_SLOTS: dict[str, DomainSlots] = { + "finance": DomainSlots( + ("savings account", "credit card", "monthly budget", "insurance claim"), + ("储蓄账户", "信用卡", "月度预算", "保险理赔"), + ("current balance", "exchange rate", "payment status", "claim progress"), + ("当前余额", "汇率", "付款状态", "理赔进度"), + ("transfer funds", "replace the card", "submit the claim", "buy the fund"), + ("转账", "补办卡片", "提交理赔", "购买基金"), + ("the bank", "the card issuer", "the insurer", "the broker"), + ("银行", "发卡行", "保险公司", "券商"), + ("was approved", "was declined", "is being reviewed", "needs verification"), + ("已获批准", "已被拒绝", "正在审核", "需要验证"), + ), + "travel": DomainSlots( + ("morning flight", "hotel booking", "train ticket", "airport transfer"), + ("早班航班", "酒店预订", "火车票", "机场接送"), + ("departure time", "platform number", "booking status", "delay estimate"), + ("出发时间", "站台编号", "预订状态", "延误时长"), + ("book the flight", "reserve the hotel", "change the ticket", "order a taxi"), + ("预订航班", "预订酒店", "改签车票", "预约出租车"), + ("the airline", "the hotel", "the railway", "the taxi company"), + ("航空公司", "酒店", "铁路客服", "出租车公司"), + ("was confirmed", "was delayed", "changed gates", "was cancelled"), + ("已确认", "已延误", "已变更登机口", "已取消"), + ), + "calendar": DomainSlots( + ("team meeting", "dentist appointment", "morning alarm", "project reminder"), + ("团队会议", "牙医预约", "早晨闹钟", "项目提醒"), + ("start time", "meeting location", "next occurrence", "attendee list"), + ("开始时间", "会议地点", "下次时间", "参与者名单"), + ("reserve a meeting room", "reschedule the appointment", "invite the team", "book the venue"), + ("预订会议室", "改约时间", "邀请团队", "预订场地"), + ("the office", "the clinic", "the event host", "the venue"), + ("办公室", "诊所", "活动主办方", "场地方"), + ("was added", "was moved", "has a conflict", "starts soon"), + ("已添加", "已改期", "存在冲突", "即将开始"), + ), + "communication": DomainSlots( + ("work inbox", "family group", "video call", "contact list"), + ("工作收件箱", "家庭群聊", "视频通话", "联系人列表"), + ("unread count", "call duration", "delivery status", "contact details"), + ("未读数量", "通话时长", "送达状态", "联系信息"), + ("send the parcel", "print the invitation", "deliver the letter", "arrange an interpreter"), + ("寄送包裹", "印刷邀请函", "投递信件", "安排翻译"), + ("the courier", "the print shop", "the post office", "the agency"), + ("快递公司", "印刷店", "邮局", "服务机构"), + ("finished syncing", "lost connection", "was delivered", "needs permission"), + ("已同步完成", "连接已断开", "已送达", "需要权限"), + ), + "media": DomainSlots( + ("jazz playlist", "evening podcast", "photo album", "news channel"), + ("爵士歌单", "晚间播客", "照片相册", "新闻频道"), + ("episode length", "release date", "track title", "download progress"), + ("单集时长", "发布日期", "曲目名称", "下载进度"), + ("buy the album", "rent the film", "print the photos", "book the studio"), + ("购买专辑", "租赁影片", "冲印照片", "预订录音棚"), + ("the music store", "the cinema service", "the photo lab", "the studio"), + ("音乐商店", "影视服务商", "照片冲印店", "录音棚"), + ("finished downloading", "is unavailable", "resumed playing", "was removed"), + ("已下载完成", "暂不可用", "已继续播放", "已被移除"), + ), + "smartHome": DomainSlots( + ("living-room lights", "front-door lock", "bedroom thermostat", "robot vacuum"), + ("客厅灯", "前门门锁", "卧室温控器", "扫地机器人"), + ("power level", "lock status", "room temperature", "cleaning progress"), + ("电量", "门锁状态", "室温", "清扫进度"), + ("repair the lock", "install the thermostat", "service the vacuum", "replace the sensor"), + ("维修门锁", "安装温控器", "保养扫地机", "更换传感器"), + ("the locksmith", "the installer", "the repair shop", "the electrician"), + ("锁匠", "安装人员", "维修点", "电工"), + ("went offline", "is back online", "detected motion", "finished cleaning"), + ("已离线", "已恢复在线", "检测到移动", "已完成清扫"), + ), + "shopping": DomainSlots( + ("grocery list", "shoe order", "gift basket", "store coupon"), + ("购物清单", "鞋子订单", "礼品篮", "商店优惠券"), + ("current price", "stock level", "delivery date", "discount amount"), + ("当前价格", "库存数量", "送达日期", "折扣金额"), + ("place the order", "exchange the shoes", "wrap the gift", "schedule delivery"), + ("下单", "换鞋", "包装礼物", "预约配送"), + ("the retailer", "the shoe store", "the gift shop", "the courier"), + ("零售商", "鞋店", "礼品店", "快递公司"), + ("was shipped", "is out of stock", "was refunded", "is ready for pickup"), + ("已发货", "已售罄", "已退款", "可到店取货"), + ), + "dining": DomainSlots( + ("dinner booking", "lunch menu", "takeout order", "coffee subscription"), + ("晚餐预订", "午餐菜单", "外卖订单", "咖啡订购"), + ("table availability", "waiting time", "order status", "menu price"), + ("空桌情况", "等位时间", "订单状态", "菜单价格"), + ("reserve a table", "change the order", "deliver the meal", "cater the event"), + ("预订餐桌", "修改订单", "配送餐食", "承办餐饮"), + ("the restaurant", "the takeaway", "the café", "the caterer"), + ("餐厅", "外卖商家", "咖啡店", "餐饮公司"), + ("was accepted", "is being prepared", "is ready", "was cancelled"), + ("已接单", "正在制作", "已备好", "已取消"), + ), + "health": DomainSlots( + ("step record", "sleep report", "prescription", "vaccination record"), + ("步数记录", "睡眠报告", "处方", "疫苗接种记录"), + ("daily total", "renewal date", "dosage note", "appointment status"), + ("当日总数", "续方日期", "剂量说明", "预约状态"), + ("book an examination", "refill the prescription", "deliver the medicine", "arrange home care"), + ("预约检查", "续开处方", "配送药品", "安排居家护理"), + ("the clinic", "the pharmacy", "the hospital", "the care provider"), + ("诊所", "药房", "医院", "护理机构"), + ("was updated", "needs review", "is ready to collect", "was received"), + ("已更新", "需要复核", "可领取", "已收到"), + ), + "weather": DomainSlots( + ("rain forecast", "air-quality report", "storm tracker", "temperature chart"), + ("降雨预报", "空气质量报告", "风暴追踪", "气温图表"), + ("rain chance", "air-quality index", "storm path", "high temperature"), + ("降雨概率", "空气质量指数", "风暴路径", "最高温度"), + ("inspect the roof", "clear the snow", "deliver sandbags", "repair the drain"), + ("检查屋顶", "清理积雪", "运送沙袋", "维修排水管"), + ("the roofer", "the snow service", "the emergency supplier", "the plumber"), + ("屋顶维修方", "除雪服务商", "应急物资商", "水管工"), + ("was updated", "issued a warning", "cleared the alert", "changed direction"), + ("已更新", "已发布预警", "已解除警报", "已改变方向"), + ), + "accountService": DomainSlots( + ("cloud account", "software license", "support ticket", "security setting"), + ("云端账户", "软件许可证", "支持工单", "安全设置"), + ("renewal date", "ticket status", "storage usage", "sign-in history"), + ("续订日期", "工单状态", "存储用量", "登录历史"), + ("upgrade the plan", "recover the account", "renew the license", "schedule support"), + ("升级套餐", "恢复账户", "续订许可证", "预约支持"), + ("the provider", "the support desk", "the software vendor", "the service team"), + ("服务商", "支持团队", "软件供应商", "客服团队"), + ("was renewed", "was suspended", "needs verification", "was restored"), + ("已续订", "已暂停", "需要验证", "已恢复"), + ), + "generalKnowledge": DomainSlots( + ("history article", "science glossary", "language guide", "reference note"), + ("历史条目", "科学词典", "语言指南", "参考笔记"), + ("publication date", "short definition", "source citation", "latest revision"), + ("发布日期", "简短定义", "来源引用", "最新修订"), + ("translate the manuscript", "verify the archive", "print the encyclopedia", "catalog the collection"), + ("翻译手稿", "核验档案", "印刷百科全书", "编目藏品"), + ("the translator", "the archive", "the publisher", "the library"), + ("翻译机构", "档案馆", "出版社", "图书馆"), + ("was revised", "is temporarily unavailable", "added a citation", "finished indexing"), + ("已修订", "暂不可用", "已添加引用", "已完成索引"), + ), +} + +DISPLAY_VALUES = ( + ("compact mode", "紧凑模式"), + ("the top position", "顶部"), + ("a blue highlight", "蓝色高亮"), + ("the favorites section", "收藏区"), +) +APP_SCOPES = ( + ("the dashboard", "仪表盘"), + ("the quick panel", "快捷面板"), + ("the saved view", "已存视图"), + ("the app widget", "应用小组件"), +) +PEOPLE = ( + ("Alex", "小林"), + ("Morgan", "小周"), + ("Taylor", "小陈"), + ("Jordan", "小何"), +) + +TEMPLATES: dict[str, dict[str, tuple[str, ...]]] = { + "en": { + "assistantCommand": ( + "Set {subject} to {value} in {scope}.", + "Pin {subject} at {value} on {scope}.", + "Show {subject} with {value} in {scope}.", + "Move {subject} to {value} on {scope}.", + ), + "informationQuery": ( + "Look up the {fact} for {subject} from {scope}.", + "What is the {fact} for {subject} in {scope}?", + "Show me the latest {fact} for {subject} from {scope}.", + "Find the current {fact} for {subject} in {scope}.", + ), + "systemNotification": ( + "System notice: {subject} {event}; details are in {scope}.", + "Service update: {subject} {event}. Open {scope} for details.", + "Automatic alert: {subject} {event} in {scope}.", + "Status update from {scope}: {subject} {event}.", + ), + "task": ( + "Please ask {provider} to {service} for {subject}.", + "Arrange for {provider} to {service} regarding {subject}.", + "I need {provider} to {service} for {subject}.", + "Have {provider} {service} for {subject}, with confirmation.", + ), + "question": ( + "{person}, do you think {subject} belongs in {scope}?", + "{person}, would {subject} work better with {value}?", + "In your opinion, is {subject} suitable for {scope}, {person}?", + "{person}, which presentation of {subject} would you prefer in {scope}?", + ), + "replyableMessage": ( + "{person}, I shared {subject} with you through {scope}; let me know when you see it.", + "{person}, I left the notes about {subject} in {scope} and would value your reaction.", + "{person}, the draft for {subject} is in {scope}; please reply when you have reviewed it.", + "{person}, I updated {subject} in {scope}; tell me whether it works for you.", + ), + }, + "zh-Hans": { + "assistantCommand": ( + "把{subject}在{scope}中设为{value}。", + "将{subject}以{value}固定到{scope}。", + "在{scope}中用{value}显示{subject}。", + "把{subject}移到{scope}的{value}。", + ), + "informationQuery": ( + "查询{scope}里{subject}的{fact}。", + "{scope}中{subject}的{fact}是什么?", + "显示{scope}里{subject}最新的{fact}。", + "查找{scope}中{subject}当前的{fact}。", + ), + "systemNotification": ( + "系统通知:{subject}{event},详情请查看{scope}。", + "服务更新:{subject}{event},可在{scope}查看详情。", + "自动提醒:{scope}中的{subject}{event}。", + "来自{scope}的状态更新:{subject}{event}。", + ), + "task": ( + "请联系{provider}为{subject}{service}。", + "安排{provider}处理{subject}并{service}。", + "我需要{provider}针对{subject}{service}。", + "请让{provider}为{subject}{service},并确认结果。", + ), + "question": ( + "{person},你觉得{subject}适合放在{scope}吗?", + "{person},你认为把{subject}设为{value}会更好吗?", + "{person},依你看{subject}放进{scope}合适吗?", + "{person},你更喜欢{subject}在{scope}里怎样展示?", + ), + "replyableMessage": ( + "{person},我已经通过{scope}把{subject}分享给你,看到后告诉我一声。", + "{person},我把{subject}的说明放在{scope}了,想听听你的看法。", + "{person},关于{subject}的草稿在{scope}里,看完请回复我。", + "{person},我更新了{scope}里的{subject},请告诉我是否合适。", + ), + }, +} + + +def normalize_text(value: str) -> str: + """Apply the corpus's stable NFKC and whitespace normalization.""" + + return " ".join(unicodedata.normalize("NFKC", value).split()).strip() + + +def fingerprint(value: str) -> str: + """Return a conservative normalized key for exact-text exclusion.""" + + return normalize_text(value).casefold() + + +def stable_key(*values: object) -> bytes: + """Create an ordering key independent of Python hash randomization.""" + + return hashlib.sha256("|".join(map(str, values)).encode("utf-8")).digest() + + +def paired_values(values: Sequence[tuple[str, str]], language: str) -> tuple[str, ...]: + index = 0 if language == "en" else 1 + return tuple(value[index] for value in values) + + +def slots_for( + slots: DomainSlots, + language: str, + category: str, +) -> tuple[tuple[str, ...], ...]: + """Return only finite slots that are semantically valid for a category.""" + + suffix = "en" if language == "en" else "zh" + subjects = getattr(slots, f"subjects_{suffix}") + providers = getattr(slots, f"providers_{suffix}") + values = paired_values(DISPLAY_VALUES, language) + scopes = paired_values(APP_SCOPES, language) + people = paired_values(PEOPLE, language) + if category == "assistantCommand": + return subjects, values, scopes + if category == "informationQuery": + return subjects, getattr(slots, f"facts_{suffix}"), scopes + if category == "systemNotification": + return subjects, getattr(slots, f"events_{suffix}"), scopes + if category == "task": + return subjects, getattr(slots, f"services_{suffix}"), providers + if category == "question": + return subjects, values, scopes, people + return subjects, scopes, people + + +def candidate_texts( + language: str, + category: str, + domain: str, +) -> Iterator[tuple[str, str]]: + """Yield every deterministic template/slot combination for one cell.""" + + templates = TEMPLATES[language][category] + slot_groups = slots_for(DOMAIN_SLOTS[domain], language, category) + argument_names = { + "assistantCommand": ("subject", "value", "scope"), + "informationQuery": ("subject", "fact", "scope"), + "systemNotification": ("subject", "event", "scope"), + "task": ("subject", "service", "provider"), + "question": ("subject", "value", "scope", "person"), + "replyableMessage": ("subject", "scope", "person"), + }[category] + candidates: list[tuple[str, str]] = [] + for template_index, template in enumerate(templates, start=1): + family = f"v6_{category}_{language}_template_{template_index}" + for values in itertools.product(*slot_groups): + text = normalize_text(template.format(**dict(zip(argument_names, values)))) + candidates.append((text, family)) + yield from sorted( + candidates, + key=lambda value: stable_key(language, category, domain, *value), + ) + + +def quota_by_domain(total: int, offset: int) -> dict[str, int]: + """Distribute a category total across all domains without randomness.""" + + base, remainder = divmod(total, len(DOMAINS)) + quotas = {domain: base for domain in DOMAINS} + for index in range(remainder): + quotas[DOMAINS[(offset + index) % len(DOMAINS)]] += 1 + return quotas + + +def make_record( + *, + index: int, + text: str, + language: str, + category: str, + domain: str, + template_family: str, +) -> dict: + """Build the complete current training schema for one known boundary.""" + + label_values = {label: False for label in INTENT_LABELS} + label_values[category] = True + # An interpersonal question naturally invites a reply; both fields are + # template-determined while all three routing intents remain false. + if category == "question": + label_values["replyableMessage"] = True + record_id = ( + f"v6-boundary-{language}-{category}-{domain}-" + f"{index:04d}-{hashlib.sha256(text.encode('utf-8')).hexdigest()[:12]}" + ) + return { + "id": record_id, + "text": text, + "language": language, + "split": "train", + "family": template_family, + "task": label_values["task"], + "question": label_values["question"], + "invitation": label_values["invitation"], + "complaint": label_values["complaint"], + "scheduleNegotiation": label_values["scheduleNegotiation"], + "confirmationDecision": label_values["confirmationDecision"], + "followUpReminder": label_values["followUpReminder"], + "blessing": label_values["blessing"], + "sentiment": "neutral", + "replyable": label_values["replyableMessage"], + "assistantCommand": label_values["assistantCommand"], + "informationQuery": label_values["informationQuery"], + "systemNotification": label_values["systemNotification"], + "domain": domain, + "sampleWeight": SAMPLE_WEIGHT, + "knownLabels": KNOWN_LABELS, + "labelingMethod": "deterministic taxonomy-v6 template and finite slots", + "sourceDataset": SOURCE_DATASET, + "sourceLicense": SOURCE_LICENSE, + "sourceURL": "Scripts/clipboard_semantics/generate_v6_boundary_corpus.py", + "sourceRevision": SOURCE_REVISION, + "sourceSplit": SOURCE_SPLIT, + "synthetic": True, + "templateFamily": template_family, + } + + +def discover_holdout_paths(directory: Path) -> list[Path]: + """Find every frozen holdout name covered by the v6 training policy.""" + + paths = set(directory.rglob("*holdout-corpus.jsonl")) + blind = directory / "product-policy-blind-holdout-v1.jsonl" + if blind.is_file(): + paths.add(blind) + return sorted(paths) + + +def load_holdout_fingerprints(paths: Iterable[Path]) -> set[str]: + """Load normalized text keys without depending on any holdout schema extras.""" + + fingerprints: set[str] = set() + for path in sorted(set(paths)): + if not path.is_file(): + continue + for line in path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + value = json.loads(line) + text = value.get("text") + if isinstance(text, str) and normalize_text(text): + fingerprints.add(fingerprint(text)) + return fingerprints + + +def validate_records(records: Sequence[dict], holdouts: set[str]) -> None: + """Fail closed on duplicate, leakage, schema, or annotation mistakes.""" + + if len({record["id"] for record in records}) != len(records): + raise ValueError("Duplicate generated IDs") + text_keys = [fingerprint(record["text"]) for record in records] + if len(set(text_keys)) != len(records): + raise ValueError("Duplicate generated texts") + leaked = set(text_keys) & holdouts + if leaked: + raise ValueError(f"Holdout overlap remained after filtering: {len(leaked)}") + for record in records: + if record["text"] != normalize_text(record["text"]): + raise ValueError(f"Non-NFKC record: {record['id']}") + if record["knownLabels"] != KNOWN_LABELS: + raise ValueError(f"Unexpected known labels: {record['id']}") + if record["domain"] not in DOMAINS: + raise ValueError(f"Unsupported domain: {record['id']}") + if record["sampleWeight"] != SAMPLE_WEIGHT: + raise ValueError(f"Unexpected sample weight: {record['id']}") + + +def generate_records( + *, + positive_per_intent_language: int = DEFAULT_POSITIVE_PER_INTENT_LANGUAGE, + negative_per_intent_language: int = DEFAULT_NEGATIVE_PER_INTENT_LANGUAGE, + holdout_fingerprints: set[str] | None = None, +) -> tuple[list[dict], Counter[str], int]: + """Generate balanced records and return target-intent counts plus exclusions.""" + + if positive_per_intent_language < 1 or negative_per_intent_language < 1: + raise ValueError("Per-intent counts must be positive") + holdouts = holdout_fingerprints or set() + records: list[dict] = [] + target_counts: Counter[str] = Counter() + seen: set[str] = set() + excluded_holdout = 0 + for language in LANGUAGES: + for category in BOUNDARY_INTENTS: + category_total = ( + positive_per_intent_language + if category in NEW_INTENTS + else negative_per_intent_language + ) + quotas = quota_by_domain(category_total, CATEGORY_OFFSETS[category]) + for domain in DOMAINS: + selected = 0 + for text, family in candidate_texts(language, category, domain): + text_key = fingerprint(text) + if text_key in holdouts: + excluded_holdout += 1 + continue + if text_key in seen: + continue + record = make_record( + index=selected + 1, + text=text, + language=language, + category=category, + domain=domain, + template_family=family, + ) + records.append(record) + seen.add(text_key) + target_counts[f"{language}|{category}"] += 1 + selected += 1 + if selected == quotas[domain]: + break + if selected != quotas[domain]: + raise ValueError( + f"Insufficient unique candidates for {language}/{category}/" + f"{domain}: {selected} < {quotas[domain]}" + ) + records.sort(key=lambda record: stable_key(record["id"])) + validate_records(records, holdouts) + return records, target_counts, excluded_holdout + + +def serialize_records(records: Sequence[dict]) -> bytes: + """Serialize JSONL with stable keys and a final newline.""" + + return ( + "\n".join( + json.dumps(record, ensure_ascii=False, sort_keys=True) + for record in records + ) + + "\n" + ).encode("utf-8") + + +def nested_counts(counter: Counter[tuple[str, ...]]) -> dict: + """Turn tuple-key counts into a stable nested JSON object.""" + + root: dict = {} + for keys, count in sorted(counter.items()): + node = root + for key in keys[:-1]: + node = node.setdefault(key, {}) + node[keys[-1]] = count + return root + + +def build_summary( + records: Sequence[dict], + target_counts: Counter[str], + excluded_holdout: int, + payload: bytes, +) -> dict: + """Summarize all requested dimensions and the exact output artifact.""" + + by_language = Counter((record["language"],) for record in records) + by_boundary_target_language = Counter( + { + tuple(key.split("|", 1)): count + for key, count in target_counts.items() + } + ) + by_intent_language = Counter() + for record in records: + for intent in BOUNDARY_INTENTS: + field = "replyable" if intent == "replyableMessage" else intent + if record[field]: + by_intent_language[(record["language"], intent)] += 1 + by_intent = Counter() + for (_, intent), count in by_intent_language.items(): + by_intent[(intent,)] += count + by_domain = Counter((record["domain"],) for record in records) + by_family = Counter((record["templateFamily"],) for record in records) + return { + "schemaVersion": 1, + "sourceRevision": SOURCE_REVISION, + "recordCount": len(records), + "counts": { + "byLanguage": nested_counts(by_language), + "byIntent": nested_counts(by_intent), + "byIntentAndLanguage": nested_counts(by_intent_language), + "byBoundaryTargetAndLanguage": nested_counts( + by_boundary_target_language + ), + "byDomain": nested_counts(by_domain), + "byTemplateFamily": nested_counts(by_family), + }, + "corpusSHA256": hashlib.sha256(payload).hexdigest(), + "excludedHoldoutOverlap": excluded_holdout, + } + + +def write_corpus( + output_path: Path, + summary_path: Path, + *, + positive_per_intent_language: int = DEFAULT_POSITIVE_PER_INTENT_LANGUAGE, + negative_per_intent_language: int = DEFAULT_NEGATIVE_PER_INTENT_LANGUAGE, + holdout_paths: Iterable[Path] = (), +) -> dict: + """Generate and write the corpus and summary files.""" + + holdouts = load_holdout_fingerprints(holdout_paths) + records, target_counts, excluded = generate_records( + positive_per_intent_language=positive_per_intent_language, + negative_per_intent_language=negative_per_intent_language, + holdout_fingerprints=holdouts, + ) + payload = serialize_records(records) + summary = build_summary(records, target_counts, excluded, payload) + output_path.parent.mkdir(parents=True, exist_ok=True) + summary_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_bytes(payload) + summary_path.write_text( + json.dumps(summary, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return summary + + +def parse_arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT_PATH) + parser.add_argument("--summary-output", type=Path, default=DEFAULT_SUMMARY_PATH) + parser.add_argument( + "--holdout-directory", + type=Path, + default=OUTPUT_DIRECTORY, + ) + parser.add_argument( + "--positive-per-intent-language", + type=int, + default=DEFAULT_POSITIVE_PER_INTENT_LANGUAGE, + ) + parser.add_argument( + "--negative-per-intent-language", + type=int, + default=DEFAULT_NEGATIVE_PER_INTENT_LANGUAGE, + ) + return parser.parse_args() + + +def main() -> None: + arguments = parse_arguments() + holdout_paths = discover_holdout_paths(arguments.holdout_directory) + summary = write_corpus( + arguments.output, + arguments.summary_output, + positive_per_intent_language=arguments.positive_per_intent_language, + negative_per_intent_language=arguments.negative_per_intent_language, + holdout_paths=holdout_paths, + ) + print( + "V6_BOUNDARY_CORPUS " + f"records={summary['recordCount']} " + f"excludedHoldout={summary['excludedHoldoutOverlap']} " + f"sha256={summary['corpusSHA256']}" + ) + + +if __name__ == "__main__": + main() diff --git a/Scripts/clipboard_semantics/merge_consensus_labels_v2.py b/Scripts/clipboard_semantics/merge_consensus_labels_v2.py new file mode 100644 index 0000000..f830469 --- /dev/null +++ b/Scripts/clipboard_semantics/merge_consensus_labels_v2.py @@ -0,0 +1,607 @@ +#!/usr/bin/env python3 +"""Merge blind three-state labels from three primary and two review models.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +from collections import Counter +from pathlib import Path + +INTENT_LABELS = ( + "task", + "question", + "invitation", + "complaint", + "scheduleNegotiation", + "confirmationDecision", + "followUpReminder", + "blessing", + "replyableMessage", + "assistantCommand", + "informationQuery", + "systemNotification", +) +DOMAINS = { + "finance", + "travel", + "calendar", + "communication", + "media", + "smartHome", + "shopping", + "dining", + "health", + "weather", + "accountService", + "generalKnowledge", +} +DOMAIN_STATES = {*DOMAINS, "unknown"} +LABEL_STATES = {"true", "false", "unknown"} +SENTIMENT_STATES = {"positive", "neutral", "negative", "unknown"} +PROMPT_VERSION = "clipboard-consensus-v6" + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def read_json_lines(path: Path) -> list[dict]: + return [ + json.loads(line) + for line in path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + + +def write_json_lines(path: Path, records: list[dict]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as handle: + for record in records: + handle.write( + json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n" + ) + + +def parse_labeler(value: str) -> tuple[str, Path]: + name, separator, raw_path = value.partition("=") + if not separator or not name or not raw_path: + raise argparse.ArgumentTypeError("Expected LABELER=PATH") + return name, Path(raw_path) + + +def validate_record(record: dict, expected_ids: set[str]) -> dict: + identifier = record.get("id") + if identifier not in expected_ids: + raise ValueError(f"Unexpected labeler record id: {identifier}") + labels = record.get("labels") + if not isinstance(labels, dict) or set(labels) != set(INTENT_LABELS): + raise ValueError(f"Every intent label is required for {identifier}") + if any(value not in LABEL_STATES for value in labels.values()): + raise ValueError(f"Invalid three-state label for {identifier}") + sentiment = record.get("sentiment") + if sentiment not in SENTIMENT_STATES: + raise ValueError(f"Invalid sentiment for {identifier}: {sentiment}") + domain = record.get("domain") + if domain not in DOMAIN_STATES: + raise ValueError(f"Invalid domain for {identifier}: {domain}") + if not isinstance(record.get("ambiguous"), bool): + raise TypeError(f"Missing ambiguous flag for {identifier}") + if not isinstance(record.get("quotedOrMeta"), bool): + raise TypeError(f"Missing quotedOrMeta flag for {identifier}") + confidence = record.get("confidence") + if not isinstance(confidence, (int, float)) or not 0 <= confidence <= 1: + raise ValueError(f"Invalid confidence for {identifier}: {confidence}") + return { + "id": identifier, + "labels": {label: labels[label] for label in INTENT_LABELS}, + "sentiment": sentiment, + "domain": domain, + "ambiguous": record["ambiguous"], + "quotedOrMeta": record["quotedOrMeta"], + "confidence": round(float(confidence), 4), + } + + +def load_labelers( + values: list[tuple[str, Path]], + expected_ids: set[str], + require_exact_ids: bool, +) -> list[tuple[str, dict[str, dict]]]: + labelers = [] + for name, path in values: + records = [ + validate_record(record, expected_ids) + for record in read_json_lines(path) + ] + by_id = {record["id"]: record for record in records} + if len(by_id) != len(records): + raise ValueError(f"Labeler {name} contains duplicate ids") + if require_exact_ids and set(by_id) != expected_ids: + missing = expected_ids - set(by_id) + extra = set(by_id) - expected_ids + raise ValueError( + f"Labeler {name} id mismatch: missing={len(missing)} " + f"extra={len(extra)}" + ) + labelers.append((name, by_id)) + return labelers + + +def field_values(record: dict) -> dict[str, str]: + return { + **record["labels"], + "sentiment": record["sentiment"], + "domain": record["domain"], + } + + +def primary_review_ids( + primary: list[tuple[str, dict[str, dict]]], + queue_ids: set[str], +) -> set[str]: + review_ids = set() + for identifier in queue_ids: + records = [values[identifier] for _, values in primary] + fields = [field_values(record) for record in records] + unanimous = all( + len({field[name] for field in fields}) == 1 + for name in (*INTENT_LABELS, "sentiment", "domain") + ) + flags_clear = not any( + record["ambiguous"] or record["quotedOrMeta"] for record in records + ) + if not unanimous or not flags_clear: + review_ids.add(identifier) + return review_ids + + +def prepare_review(arguments: argparse.Namespace) -> dict: + queue = read_json_lines(arguments.queue) + queue_by_id = {record["id"]: record for record in queue} + if len(queue_by_id) != len(queue): + raise ValueError("Queue contains duplicate ids") + if len(arguments.primary) != 3: + raise ValueError("Exactly three primary labelers are required") + primary = load_labelers(arguments.primary, set(queue_by_id), True) + review_ids = primary_review_ids(primary, set(queue_by_id)) + review_queue = [ + { + "id": record["id"], + "text": record["text"], + "language": record["language"], + } + for record in queue + if record["id"] in review_ids + ] + write_json_lines(arguments.output, review_queue) + report = { + "schemaVersion": 2, + "promptVersion": PROMPT_VERSION, + "queueCount": len(queue), + "queueSHA256": sha256_file(arguments.queue), + "primaryUnanimousCount": len(queue) - len(review_queue), + "reviewCount": len(review_queue), + "primaryLabelers": [name for name, _ in primary], + "primaryOutputSHA256": { + name: sha256_file(path) for name, path in arguments.primary + }, + } + arguments.report.write_text( + json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return report + + +def split_queue(arguments: argparse.Namespace) -> dict: + queue = read_json_lines(arguments.queue) + if arguments.chunk_size <= 0: + raise ValueError("chunk-size must be positive") + arguments.output_directory.mkdir(parents=True, exist_ok=True) + output_paths = [] + for start in range(0, len(queue), arguments.chunk_size): + index = len(output_paths) + 1 + path = arguments.output_directory / f"chunk-{index:03d}.jsonl" + write_json_lines(path, queue[start : start + arguments.chunk_size]) + output_paths.append(path) + report = { + "schemaVersion": 2, + "queueCount": len(queue), + "queueSHA256": sha256_file(arguments.queue), + "chunkSize": arguments.chunk_size, + "chunkCount": len(output_paths), + "chunks": [ + { + "path": str(path), + "records": len(read_json_lines(path)), + "sha256": sha256_file(path), + } + for path in output_paths + ], + } + arguments.report.write_text( + json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return report + + +def combine_labeler(arguments: argparse.Namespace) -> dict: + queue = read_json_lines(arguments.queue) + queue_ids = [record["id"] for record in queue] + expected_ids = set(queue_ids) + combined = [] + for path in arguments.input: + combined.extend(read_json_lines(path)) + validated = [ + validate_record(record, expected_ids) for record in combined + ] + by_id = {record["id"]: record for record in validated} + if len(by_id) != len(validated): + raise ValueError("Combined labeler outputs contain duplicate ids") + if set(by_id) != expected_ids: + raise ValueError( + "Combined labeler output ids do not match the source queue" + ) + ordered = [by_id[identifier] for identifier in queue_ids] + write_json_lines(arguments.output, ordered) + report = { + "schemaVersion": 2, + "queueCount": len(queue), + "queueSHA256": sha256_file(arguments.queue), + "inputCount": len(arguments.input), + "outputCount": len(ordered), + "outputSHA256": sha256_file(arguments.output), + } + arguments.report.write_text( + json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return report + + +def stable_split(identifier: str) -> str: + bucket = int.from_bytes(hashlib.sha256(identifier.encode()).digest()[:8], "big") + bucket %= 100 + if bucket < 80: + return "silverTrain" + if bucket < 90: + return "silverCalibration" + return "silverAcceptance" + + +def field_consensus( + records: list[dict], + field: str, + required_votes: int, +) -> tuple[str, dict[str, int]]: + values = [ + field_values(record)[field] + for record in records + if field_values(record)[field] != "unknown" + ] + votes = Counter(values) + if not votes: + return "unknown", {} + value, count = votes.most_common(1)[0] + if count < required_votes: + return "unknown", dict(sorted(votes.items())) + return value, dict(sorted(votes.items())) + + +def fleiss_kappa_for_field( + labelers: list[tuple[str, dict[str, dict]]], + identifiers: list[str], + field: str, +) -> float: + categories = ( + sorted(SENTIMENT_STATES) + if field == "sentiment" + else sorted(DOMAIN_STATES) + if field == "domain" + else sorted(LABEL_STATES) + ) + category_totals = Counter() + item_agreements = [] + rater_count = len(labelers) + for identifier in identifiers: + votes = Counter( + field_values(records[identifier])[field] + for _, records in labelers + ) + category_totals.update(votes) + item_agreements.append( + sum(count * (count - 1) for count in votes.values()) + / (rater_count * (rater_count - 1)) + ) + if not item_agreements: + return 0 + observed = sum(item_agreements) / len(item_agreements) + total = sum(category_totals.values()) + expected = sum( + (category_totals[category] / total) ** 2 for category in categories + ) + if math.isclose(expected, 1): + return 1 + return round((observed - expected) / (1 - expected), 4) + + +def training_record( + queue_record: dict, + states: dict[str, str], + tier: str, + votes: dict, + labelers: list[tuple[str, dict[str, dict]]], +) -> dict: + split = stable_split(queue_record["id"]) + known_labels = [ + label + for label in (*INTENT_LABELS, "sentiment", "domain") + if states[label] != "unknown" + ] + return { + "id": f"consensus-v2-{queue_record['id']}", + "sourceRecordID": queue_record["id"], + "text": queue_record["text"], + "language": queue_record["language"], + "family": "consensus_v2", + "split": split, + **{ + ("replyable" if label == "replyableMessage" else label): ( + states[label] == "true" + ) + for label in INTENT_LABELS + }, + "sentiment": ( + states["sentiment"] + if states["sentiment"] != "unknown" + else "neutral" + ), + "domain": ( + states["domain"] if states["domain"] != "unknown" else None + ), + "knownLabels": known_labels, + "labelQualityTier": tier, + "sampleWeight": 1.0 if tier == "A" else 0.65, + "promptVersion": PROMPT_VERSION, + "modelVotes": votes, + "labelers": [name for name, _ in labelers], + } + + +def merge(arguments: argparse.Namespace) -> dict: + queue = read_json_lines(arguments.queue) + queue_by_id = {record["id"]: record for record in queue} + queue_ids = set(queue_by_id) + if len(queue_by_id) != len(queue): + raise ValueError("Queue contains duplicate ids") + if len(arguments.primary) != 3 or len(arguments.reviewer) != 2: + raise ValueError("Three primary and two review labelers are required") + primary = load_labelers(arguments.primary, queue_ids, True) + review_ids = primary_review_ids(primary, queue_ids) + reviewers = load_labelers(arguments.reviewer, review_ids, True) + tier_a = [] + tier_b = [] + human_review = [] + all_labelers = primary + reviewers + field_names = (*INTENT_LABELS, "sentiment", "domain") + for identifier in sorted(queue_ids): + queue_record = queue_by_id[identifier] + primary_records = [records[identifier] for _, records in primary] + if identifier not in review_ids: + states = { + field: field_values(primary_records[0])[field] + for field in field_names + } + votes = { + field: {states[field]: 3} + for field in field_names + } + tier_a.append( + training_record( + queue_record, + states, + "A", + votes, + primary, + ) + ) + continue + combined_records = primary_records + [ + records[identifier] for _, records in reviewers + ] + states = {} + votes = {} + for field in field_names: + states[field], votes[field] = field_consensus( + combined_records, + field, + 4, + ) + ambiguous_votes = sum( + record["ambiguous"] for record in combined_records + ) + quoted_votes = sum( + record["quotedOrMeta"] for record in combined_records + ) + unresolved = [ + field + for field, value in states.items() + if value == "unknown" and votes[field] + ] + positive_intents = any(states[label] == "true" for label in INTENT_LABELS) + if ambiguous_votes >= 2: + unresolved.append("ambiguous") + if quoted_votes >= 2 and positive_intents: + unresolved.append("quotedOrMeta") + if unresolved: + human_review.append( + { + "id": identifier, + "text": queue_record["text"], + "language": queue_record["language"], + "unresolvedFields": sorted(set(unresolved)), + "modelVotes": votes, + "ambiguousVotes": ambiguous_votes, + "quotedOrMetaVotes": quoted_votes, + "resolution": None, + "reviewer": None, + "reason": None, + } + ) + continue + tier_b.append( + training_record( + queue_record, + states, + "B", + votes, + all_labelers, + ) + ) + accepted = tier_a + tier_b + write_json_lines(arguments.tier_a, tier_a) + write_json_lines(arguments.tier_b, tier_b) + write_json_lines(arguments.accepted, accepted) + write_json_lines(arguments.human_review, human_review) + identifiers = sorted(queue_ids) + kappa_by_field = { + field: fleiss_kappa_for_field(primary, identifiers, field) + for field in field_names + } + languages = sorted({record["language"] for record in queue}) + kappa_by_language = { + language: { + field: fleiss_kappa_for_field( + primary, + sorted( + identifier + for identifier, record in queue_by_id.items() + if record["language"] == language + ), + field, + ) + for field in field_names + } + for language in languages + } + kappa_gate_passed = all( + value >= 0.8 + for values in kappa_by_language.values() + for value in values.values() + ) + report = { + "schemaVersion": 2, + "promptVersion": PROMPT_VERSION, + "queueCount": len(queue), + "queueSHA256": sha256_file(arguments.queue), + "tierACount": len(tier_a), + "tierBCount": len(tier_b), + "humanReviewCount": len(human_review), + "acceptanceRate": round(len(accepted) / max(len(queue), 1), 4), + "primaryLabelers": [name for name, _ in primary], + "reviewLabelers": [name for name, _ in reviewers], + "labelerOutputSHA256": { + name: sha256_file(path) + for name, path in (*arguments.primary, *arguments.reviewer) + }, + "primaryFleissKappaByField": kappa_by_field, + "primaryFleissKappaByLanguageAndField": kappa_by_language, + "scaleUpKappaThreshold": 0.8, + "eligibleForScaleUp": kappa_gate_passed, + "acceptedIntentPositiveCounts": { + label: sum( + record["replyable" if label == "replyableMessage" else label] + for record in accepted + ) + for label in INTENT_LABELS + }, + "acceptedLanguageCounts": dict( + sorted(Counter(record["language"] for record in accepted).items()) + ), + "splitCounts": dict( + sorted(Counter(record["split"] for record in accepted).items()) + ), + } + arguments.report.write_text( + json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return report + + +def parser() -> argparse.ArgumentParser: + root = argparse.ArgumentParser() + commands = root.add_subparsers(dest="command", required=True) + review = commands.add_parser("prepare-review") + review.add_argument("--queue", type=Path, required=True) + review.add_argument( + "--primary", + action="append", + type=parse_labeler, + required=True, + ) + review.add_argument("--output", type=Path, required=True) + review.add_argument("--report", type=Path, required=True) + review.set_defaults(handler=prepare_review) + + split = commands.add_parser("split-queue") + split.add_argument("--queue", type=Path, required=True) + split.add_argument("--output-directory", type=Path, required=True) + split.add_argument("--chunk-size", type=int, default=100) + split.add_argument("--report", type=Path, required=True) + split.set_defaults(handler=split_queue) + + combine = commands.add_parser("combine-labeler") + combine.add_argument("--queue", type=Path, required=True) + combine.add_argument( + "--input", + action="append", + type=Path, + required=True, + ) + combine.add_argument("--output", type=Path, required=True) + combine.add_argument("--report", type=Path, required=True) + combine.set_defaults(handler=combine_labeler) + + merge_parser = commands.add_parser("merge") + merge_parser.add_argument("--queue", type=Path, required=True) + merge_parser.add_argument( + "--primary", + action="append", + type=parse_labeler, + required=True, + ) + merge_parser.add_argument( + "--reviewer", + action="append", + type=parse_labeler, + required=True, + ) + merge_parser.add_argument("--tier-a", type=Path, required=True) + merge_parser.add_argument("--tier-b", type=Path, required=True) + merge_parser.add_argument("--accepted", type=Path, required=True) + merge_parser.add_argument("--human-review", type=Path, required=True) + merge_parser.add_argument("--report", type=Path, required=True) + merge_parser.set_defaults(handler=merge) + return root + + +def main() -> None: + arguments = parser().parse_args() + report = arguments.handler(arguments) + print( + f"CONSENSUS_V2_{arguments.command.upper().replace('-', '_')} " + + " ".join(f"{key}={value}" for key, value in report.items() if key.endswith("Count")) + ) + + +if __name__ == "__main__": + main() diff --git a/Scripts/clipboard_semantics/prepare_apple_nl_hardening_corpus.py b/Scripts/clipboard_semantics/prepare_apple_nl_hardening_corpus.py new file mode 100644 index 0000000..683216a --- /dev/null +++ b/Scripts/clipboard_semantics/prepare_apple_nl_hardening_corpus.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +"""Prepare a leakage-free Apple NL corpus with product-scale evaluation splits.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import unicodedata +from collections import Counter +from pathlib import Path + + +EVALUATION_SPLITS = {"validation", "test", "golden"} + + +def read_json_lines(path: Path) -> list[dict]: + return [ + json.loads(line) + for line in path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + + +def fingerprint(text: str) -> str: + normalized = unicodedata.normalize("NFKC", text) + return " ".join(normalized.casefold().split()) + + +def prepare( + training_records: list[dict], + product_records: list[dict], +) -> tuple[list[dict], dict]: + if any(record.get("split") != "train" for record in training_records): + raise ValueError("Training source must contain only split=train records") + + evaluation_records = [ + record for record in product_records if record.get("split") in EVALUATION_SPLITS + ] + if not evaluation_records: + raise ValueError("Product corpus has no evaluation records") + + evaluation_fingerprints = { + fingerprint(record["text"]) for record in evaluation_records + } + evaluation_ids = {record["id"] for record in evaluation_records} + filtered_training = [ + record + for record in training_records + if record["id"] not in evaluation_ids + and fingerprint(record["text"]) not in evaluation_fingerprints + ] + records = filtered_training + evaluation_records + + identifiers = [record["id"] for record in records] + if len(identifiers) != len(set(identifiers)): + raise ValueError("Duplicate record ids remain after filtering") + + training_fingerprints = { + fingerprint(record["text"]) for record in filtered_training + } + if training_fingerprints & evaluation_fingerprints: + raise ValueError("Training and evaluation text overlap remains after filtering") + + report = { + "schemaVersion": 1, + "originalTrainingCount": len(training_records), + "filteredTrainingCount": len(filtered_training), + "excludedTrainingOverlapCount": len(training_records) + - len(filtered_training), + "evaluationCount": len(evaluation_records), + "recordCount": len(records), + "splitCounts": dict( + sorted(Counter(record["split"] for record in records).items()) + ), + "languageCounts": dict( + sorted(Counter(record["language"] for record in records).items()) + ), + "evaluationOverlapCount": 0, + } + return records, report + + +def write_outputs( + records: list[dict], + report: dict, + output_path: Path, + report_path: Path, +) -> dict: + output_path.parent.mkdir(parents=True, exist_ok=True) + payload = "".join( + json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n" + for record in records + ) + output_path.write_text(payload, encoding="utf-8") + final_report = { + **report, + "corpusSHA256": hashlib.sha256(payload.encode()).hexdigest(), + } + report_path.write_text( + json.dumps(final_report, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return final_report + + +def parser() -> argparse.ArgumentParser: + root = argparse.ArgumentParser(description=__doc__) + root.add_argument("--train", type=Path, required=True) + root.add_argument("--product-corpus", type=Path, required=True) + root.add_argument("--output", type=Path, required=True) + root.add_argument("--report", type=Path, required=True) + return root + + +def main() -> None: + arguments = parser().parse_args() + records, report = prepare( + read_json_lines(arguments.train), + read_json_lines(arguments.product_corpus), + ) + final_report = write_outputs( + records, + report, + arguments.output, + arguments.report, + ) + print( + "APPLE_NL_HARDENING_CORPUS " + f"records={final_report['recordCount']} " + f"excludedOverlap={final_report['excludedTrainingOverlapCount']} " + f"sha256={final_report['corpusSHA256']}" + ) + + +if __name__ == "__main__": + main() diff --git a/Scripts/clipboard_semantics/prepare_blessing_benchmark.py b/Scripts/clipboard_semantics/prepare_blessing_benchmark.py new file mode 100644 index 0000000..fde4b4e --- /dev/null +++ b/Scripts/clipboard_semantics/prepare_blessing_benchmark.py @@ -0,0 +1,306 @@ +#!/usr/bin/env python3 +"""Prepare a blind, double-annotation queue for the blessing benchmark.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import unicodedata +from collections import Counter +from pathlib import Path + + +SEED = 20260828 +OUTPUT_DIRECTORY = Path("ModelTraining/ClipboardSemantics/BlessingBenchmark") +SOURCE_PATH = Path( + "ModelTraining/ClipboardSemantics/comprehensive-online-holdout-corpus.jsonl" +) +PII_PATTERN = re.compile( + r"(?:[\w.+-]+@[\w.-]+\.\w+)|(?:\+?\d[\d ()-]{8,}\d)|" + r"(?:\b\d{3}-\d{2}-\d{4}\b)", + re.IGNORECASE, +) +EXPLICIT_PATTERN = re.compile( + r"(?:祝|愿你|愿您|愿他|愿她|愿大家|恭喜|祝贺|生日快乐|" + r"新年快乐|一路平安|一路顺风|康复|前程|平安|安康|如意|好梦|" + r"希望.{0,40}(?:快乐|幸福|平安|顺利|康复|成功|健康)|" + r"wish|hope you|may you|may your|congrat|happy birthday|" + r"happy new year|good luck|best wishes|get well|safe travel|" + r"sweet dream|peace and happiness|future success)", + re.IGNORECASE, +) +BOUNDARY_PATTERN = re.compile( + r"(?:祝福语|祝福模板|祝福文案|谢谢.{0,30}祝福|感谢.{0,30}祝福|" + r"收到.{0,30}祝福|庆祝|庆功|引用.{0,20}(?:祝|愿|恭喜)|" + r"怎么.{0,20}(?:祝|生日快乐)|如何.{0,20}(?:祝|生日快乐)|" + r"template|thanks?.{0,64}(?:wish|wishes|congratulations)|" + r"celebrat|quotes?.{0,32}(?:wish|congratulat)|" + r"how to write.{0,32}(?:wish|greeting))", + re.IGNORECASE, +) +PLAIN_GREETING_PATTERN = re.compile( + r"^(?:你好|您好|早上好|中午好|下午好|晚上好|晚安|好久不见|" + r"hello|good morning|good afternoon|good evening|long time no see)" + r"[！!。,.，~～]*$", + re.IGNORECASE, +) + + +def parse_arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--source", type=Path, default=SOURCE_PATH) + parser.add_argument("--output-directory", type=Path, default=OUTPUT_DIRECTORY) + parser.add_argument("--seed", type=int, default=SEED) + parser.add_argument("--chinese-records", type=int, default=3_000) + parser.add_argument("--english-records", type=int, default=1_500) + parser.add_argument( + "--training-corpus", + action="append", + default=[], + type=Path, + help="Additional JSONL whose text must not overlap the review queue.", + ) + return parser.parse_args() + + +def normalized_text(value: str) -> str: + value = unicodedata.normalize("NFKC", value.replace("\u0000", " ")) + return " ".join(value.split()).strip() + + +def fingerprint(value: str) -> str: + return normalized_text(value).casefold() + + +def file_sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def load_records(path: Path) -> list[dict]: + return [ + json.loads(line) + for line in path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + + +def protected_fingerprints(paths: list[Path]) -> set[str]: + values: set[str] = set() + for path in paths: + for record_value in load_records(path): + values.add(fingerprint(record_value["text"])) + return values + + +def selection_stratum(record_value: dict) -> str: + text = normalized_text(record_value["text"]) + if BOUNDARY_PATTERN.search(text) or PLAIN_GREETING_PATTERN.fullmatch(text): + return "boundary_candidate" + if EXPLICIT_PATTERN.search(text): + return "explicit_candidate" + if record_value.get("blessing"): + return "weak_positive_candidate" + if record_value.get("sentiment") == "positive": + return "positive_language_boundary" + return "natural_negative" + + +def stable_priority(record_value: dict, seed: int, salt: str) -> bytes: + return hashlib.sha256( + f"{seed}|{salt}|{record_value['id']}".encode() + ).digest() + + +def select_language( + records: list[dict], + *, + language: str, + target: int, + seed: int, + protected: set[str], +) -> list[dict]: + candidates: list[dict] = [] + seen: set[str] = set() + for record_value in records: + if record_value.get("language") != language: + continue + text = normalized_text(record_value.get("text") or "") + text_key = fingerprint(text) + if ( + not 2 <= len(text) <= 500 + or PII_PATTERN.search(text) + or text_key in protected + or text_key in seen + ): + continue + seen.add(text_key) + candidate = dict(record_value) + candidate["_normalizedText"] = text + candidate["_stratum"] = selection_stratum(record_value) + candidates.append(candidate) + + fractions = { + "explicit_candidate": 0.30, + "boundary_candidate": 0.25, + "weak_positive_candidate": 0.10, + "positive_language_boundary": 0.15, + "natural_negative": 0.20, + } + selected: list[dict] = [] + selected_ids: set[str] = set() + remaining = target + for index, (stratum, fraction) in enumerate(fractions.items()): + desired = target - len(selected) if index == len(fractions) - 1 else round( + target * fraction + ) + values = sorted( + ( + value + for value in candidates + if value["_stratum"] == stratum + ), + key=lambda value: stable_priority(value, seed, stratum), + ) + for value in values[:desired]: + selected.append(value) + selected_ids.add(value["id"]) + remaining = target - len(selected) + + if remaining: + fillers = sorted( + (value for value in candidates if value["id"] not in selected_ids), + key=lambda value: stable_priority(value, seed, "fill"), + ) + selected.extend(fillers[:remaining]) + if len(selected) != target: + raise RuntimeError( + f"Only selected {len(selected)}/{target} review records for {language}" + ) + return selected + + +def write_jsonl(path: Path, records: list[dict]) -> None: + path.write_text( + "\n".join( + json.dumps(record_value, ensure_ascii=False, sort_keys=True) + for record_value in records + ) + + "\n", + encoding="utf-8", + ) + + +def main() -> None: + arguments = parse_arguments() + if arguments.chinese_records < 100 or arguments.english_records < 100: + raise ValueError("Each language requires at least 100 review records") + + source_records = load_records(arguments.source) + protected = protected_fingerprints(arguments.training_corpus) + selected = select_language( + source_records, + language="zh-Hans", + target=arguments.chinese_records, + seed=arguments.seed, + protected=protected, + ) + select_language( + source_records, + language="en", + target=arguments.english_records, + seed=arguments.seed, + protected=protected, + ) + + output_directory = arguments.output_directory + output_directory.mkdir(parents=True, exist_ok=True) + queue: list[dict] = [] + provenance: list[dict] = [] + annotation_template: list[dict] = [] + for index, source in enumerate( + sorted(selected, key=lambda value: stable_priority(value, arguments.seed, "queue")), + start=1, + ): + review_id = f"blessing-review-{index:05d}" + queue.append( + { + "id": review_id, + "text": source["_normalizedText"], + "language": source["language"], + "annotationStatus": "unreviewed", + } + ) + provenance.append( + { + "id": review_id, + "sourceRecordID": source["id"], + "sourceDataset": source.get("sourceDataset"), + "sourceLicense": source.get("sourceLicense"), + "sourceURL": source.get("sourceURL"), + "selectionStratum": source["_stratum"], + "previousWeakLabel": bool(source.get("blessing")), + } + ) + annotation_template.append( + { + "id": review_id, + "label": None, + "boundaryCategory": None, + "confidence": None, + "notes": "", + } + ) + + queue_path = output_directory / "review-queue.jsonl" + provenance_path = output_directory / "sealed-provenance.jsonl" + annotator_a_path = output_directory / "annotator-a.jsonl" + annotator_b_path = output_directory / "annotator-b.jsonl" + write_jsonl(queue_path, queue) + write_jsonl(provenance_path, provenance) + write_jsonl(annotator_a_path, annotation_template) + write_jsonl(annotator_b_path, annotation_template) + + manifest = { + "schemaVersion": 1, + "seed": arguments.seed, + "status": "awaiting-double-human-annotation", + "humanReviewComplete": False, + "policy": ( + "Evaluation-only queue derived from the frozen comprehensive holdout. " + "Never merge these records into training." + ), + "records": len(queue), + "languages": dict(Counter(value["language"] for value in queue)), + "selectionStrata": dict( + Counter(value["selectionStratum"] for value in provenance) + ), + "sourceDatasets": dict( + Counter(value["sourceDataset"] for value in provenance) + ), + "validation": { + "duplicateNormalizedTexts": ( + len(queue) + - len({fingerprint(value["text"]) for value in queue}) + ), + "configuredTrainingOverlap": sum( + fingerprint(value["text"]) in protected for value in queue + ), + "containsDetectedPII": any( + PII_PATTERN.search(value["text"]) for value in queue + ), + }, + "artifacts": { + "reviewQueueSHA256": file_sha256(queue_path), + "sealedProvenanceSHA256": file_sha256(provenance_path), + }, + } + (output_directory / "manifest.json").write_text( + json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + print(json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/Scripts/clipboard_semantics/requirements-research.txt b/Scripts/clipboard_semantics/requirements-research.txt index 501a908..d0fe7b7 100644 --- a/Scripts/clipboard_semantics/requirements-research.txt +++ b/Scripts/clipboard_semantics/requirements-research.txt @@ -1,3 +1,6 @@ +ijson==3.5.1 numpy==2.4.4 +opencc-python-reimplemented==0.1.7 +pyarrow==25.0.1 scikit-learn==1.9.0 scipy==1.18.1 diff --git a/Scripts/clipboard_semantics/run_iterative_retraining.py b/Scripts/clipboard_semantics/run_iterative_retraining.py index 7925cf0..0ce1050 100755 --- a/Scripts/clipboard_semantics/run_iterative_retraining.py +++ b/Scripts/clipboard_semantics/run_iterative_retraining.py @@ -37,7 +37,11 @@ INTENTS = ( "followUpReminder", "blessing", "replyableMessage", + "assistantCommand", + "informationQuery", + "systemNotification", ) +LEGACY_IMPLICITLY_KNOWN_INTENTS = frozenset(INTENTS[:9]) OUTPUT_DIRECTORY = Path("ModelTraining/ClipboardSemantics/IterativeResearch") BASE_CORPUS = Path( "ModelTraining/ClipboardSemantics/clipboard_semantic_corpus.jsonl" @@ -282,7 +286,7 @@ def record_label(record: dict[str, Any], intent: str) -> bool: def is_known(record: dict[str, Any], intent: str) -> bool: known = record.get("knownLabels") if known is None: - return True + return intent in LEGACY_IMPLICITLY_KNOWN_INTENTS return intent in known or ( intent == "replyableMessage" and "replyable" in known ) @@ -449,6 +453,7 @@ def sample_weight( weight = 1.0 if record.get("knownLabels") is not None: weight *= configuration.external_weight + weight *= float(record.get("sampleWeight", 1.0)) if record.get("_augmentation"): weight *= 0.60 family = str(record.get("family", "")).casefold() @@ -498,13 +503,20 @@ def calibrated_thresholds( languages = np.array([record["language"] for record in records]) result: dict[str, dict[str, Any]] = {} for intent in INTENTS: + known_mask = np.array( + [is_known(record, intent) for record in records], + dtype=bool, + ) expected = np.array( [record_label(record, intent) for record in records], dtype=bool ) - selection = select_threshold(expected, probabilities[intent]) + selection = select_threshold( + expected[known_mask], + probabilities[intent][known_mask], + ) by_language = {} for language in sorted(set(languages)): - mask = languages == language + mask = (languages == language) & known_mask positives = int(np.sum(expected[mask])) negatives = int(np.sum(~expected[mask])) if positives < 20 or negatives < 20: @@ -574,11 +586,15 @@ def metrics_for_records( predictions = runtime_predictions(records, probabilities, thresholds) per_intent = {} for intent in INTENTS: + known_mask = np.array( + [is_known(record, intent) for record in records], + dtype=bool, + ) expected = np.array( [record_label(record, intent) for record in records], dtype=bool ) counts = BinaryCounts() - counts.update(expected, predictions[intent]) + counts.update(expected[known_mask], predictions[intent][known_mask]) per_intent[intent] = counts.metrics() return aggregate_metrics(per_intent) @@ -819,14 +835,21 @@ def batched_evaluation( probabilities = prediction_probabilities(matrix, models) predictions = runtime_predictions(batch, probabilities, thresholds) for intent in INTENTS: + known_mask = np.array( + [is_known(record, intent) for record in batch], + dtype=bool, + ) expected = np.array( [record_label(record, intent) for record in batch], dtype=bool ) - counts[intent].update(expected, predictions[intent]) + counts[intent].update( + expected[known_mask], + predictions[intent][known_mask], + ) for language in {record["language"] for record in batch}: mask = np.array( [record["language"] == language for record in batch] - ) + ) & known_mask by_language[language][intent].update( expected[mask], predictions[intent][mask] ) @@ -843,7 +866,7 @@ def batched_evaluation( == source for record in batch ] - ) + ) & known_mask by_source[source][intent].update( expected[mask], predictions[intent][mask] ) diff --git a/Scripts/clipboard_semantics/tests/test_adjudicate_consensus_conflicts.py b/Scripts/clipboard_semantics/tests/test_adjudicate_consensus_conflicts.py new file mode 100644 index 0000000..0cd7009 --- /dev/null +++ b/Scripts/clipboard_semantics/tests/test_adjudicate_consensus_conflicts.py @@ -0,0 +1,268 @@ +import json +import sys +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +import adjudicate_consensus_conflicts as adjudication + + +class AdjudicationTests(unittest.TestCase): + def test_prepare_adds_all_product_policy_fields(self): + with tempfile.TemporaryDirectory() as raw_directory: + directory = Path(raw_directory) + conflicts = directory / "conflicts.jsonl" + self._write( + conflicts, + [ + { + "id": "one", + "text": "Could you send the report?", + "language": "en", + "unresolvedFields": ["sentiment"], + } + ], + ) + queue = directory / "queue.jsonl" + report = adjudication.prepare( + SimpleNamespace( + conflicts=conflicts, + queue=queue, + chunk_directory=directory / "chunks", + chunk_size=10, + report=directory / "report.json", + include_product_policy_fields=True, + ) + ) + + fields = self._read(queue)[0]["unresolvedFields"] + self.assertTrue(set(adjudication.PRODUCT_POLICY_FIELDS) <= set(fields)) + self.assertIn("sentiment", fields) + self.assertTrue(report["includesProductPolicyFields"]) + + def test_requires_evidence_from_input_text(self): + queue = { + "id": "one", + "text": "Could you send the report?", + "unresolvedFields": ["task"], + } + record = self._adjudication("one", task="true") + record["evidence"]["task"] = "not present" + + with self.assertRaisesRegex(ValueError, "exact text quote"): + adjudication.validate_adjudication(record, queue) + + def test_accepts_matching_high_confidence_adjudication(self): + report, accepted, excluded, remaining = self._merge( + first=self._adjudication("one", task="true"), + second=self._adjudication("one", task="true"), + ) + + self.assertEqual(1, report["acceptedTierCCount"]) + self.assertEqual(0, report["remainingHumanReviewCount"]) + self.assertTrue(accepted[0]["task"]) + self.assertEqual("C", accepted[0]["labelQualityTier"]) + self.assertEqual(0.35, accepted[0]["sampleWeight"]) + self.assertEqual([], excluded) + self.assertEqual([], remaining) + + def test_keeps_disagreement_for_human_review(self): + report, accepted, excluded, remaining = self._merge( + first=self._adjudication("one", task="true"), + second=self._adjudication("one", task="false"), + ) + + self.assertEqual(0, report["acceptedTierCCount"]) + self.assertEqual([], accepted) + self.assertEqual([], excluded) + self.assertEqual( + ["adjudicator-disagreement"], + remaining[0]["aiAdjudication"]["rejectedFields"]["task"], + ) + + def test_keeps_low_confidence_for_human_review(self): + first = self._adjudication("one", task="true") + first["confidence"]["task"] = 0.89 + report, _, _, remaining = self._merge( + first=first, + second=self._adjudication("one", task="true"), + ) + + self.assertEqual(1, report["remainingHumanReviewCount"]) + self.assertEqual( + ["low-confidence"], + remaining[0]["aiAdjudication"]["rejectedFields"]["task"], + ) + + def test_excludes_matching_high_confidence_device_command(self): + first = self._adjudication("one", task="true") + second = self._adjudication("one", task="true") + for record in (first, second): + record["recordDisposition"] = "exclude-device-command" + + report, accepted, excluded, remaining = self._merge(first, second) + + self.assertEqual(1, report["excludedDeviceCommandCount"]) + self.assertEqual([], accepted) + self.assertEqual("exclude-device-command", excluded[0]["disposition"]) + self.assertEqual([], remaining) + + def test_keeps_disposition_disagreement_for_human_review(self): + first = self._adjudication("one", task="true") + second = self._adjudication("one", task="true") + second["recordDisposition"] = "exclude-device-command" + + report, accepted, excluded, remaining = self._merge(first, second) + + self.assertEqual(1, report["remainingHumanReviewCount"]) + self.assertEqual([], accepted) + self.assertEqual([], excluded) + self.assertEqual( + ["adjudicator-disagreement"], + remaining[0]["aiAdjudication"]["rejectedFields"][ + "recordDisposition" + ], + ) + + def test_review_sample_is_unique_and_includes_decision_template(self): + with tempfile.TemporaryDirectory() as raw_directory: + directory = Path(raw_directory) + remaining = directory / "remaining.jsonl" + records = [] + for index, field in enumerate(("task", "recordDisposition", "task")): + first = self._adjudication(str(index), task="true") + second = self._adjudication(str(index), task="false") + records.append( + { + "id": str(index), + "text": "Could you send the report?", + "language": "en" if index % 2 else "zh-Hans", + "aiAdjudication": { + "rejectedFields": { + field: ["adjudicator-disagreement"] + }, + "adjudicatorA": first, + "adjudicatorB": second, + }, + } + ) + self._write(remaining, records) + sample = directory / "sample.jsonl" + report = adjudication.review_sample( + SimpleNamespace( + remaining=remaining, + sample=sample, + sample_size=3, + report=directory / "report.json", + ) + ) + output = self._read(sample) + + self.assertEqual(3, report["sampleCount"]) + self.assertEqual(3, len({record["id"] for record in output})) + self.assertIsNone(output[0]["humanDecision"].popitem()[1]) + disposition = next( + record + for record in output + if "recordDisposition" in record["fieldReviews"] + ) + self.assertEqual( + "keep", + disposition["fieldReviews"]["recordDisposition"][ + "adjudicatorA" + ]["value"], + ) + + def _merge(self, first, second): + with tempfile.TemporaryDirectory() as raw_directory: + directory = Path(raw_directory) + conflicts = directory / "conflicts.jsonl" + queue = directory / "queue.jsonl" + first_path = directory / "first.jsonl" + second_path = directory / "second.jsonl" + self._write( + conflicts, + [ + { + "id": "one", + "text": "Could you send the report?", + "language": "en", + "unresolvedFields": ["task"], + "modelVotes": { + **{ + label: {"false": 4} + for label in adjudication.INTENT_LABELS + }, + "sentiment": {"neutral": 4}, + }, + } + ], + ) + self._write( + queue, + [ + { + "id": "one", + "text": "Could you send the report?", + "language": "en", + "unresolvedFields": ["task"], + } + ], + ) + self._write(first_path, [first]) + self._write(second_path, [second]) + accepted_path = directory / "accepted.jsonl" + excluded_path = directory / "excluded.jsonl" + remaining_path = directory / "remaining.jsonl" + report = adjudication.merge( + SimpleNamespace( + conflicts=conflicts, + queue=queue, + adjudicator_a=[first_path], + adjudicator_b=[second_path], + adjudicator_a_name="first", + adjudicator_b_name="second", + minimum_confidence=0.9, + accepted=accepted_path, + excluded=excluded_path, + remaining=remaining_path, + report=directory / "report.json", + ) + ) + return ( + report, + self._read(accepted_path), + self._read(excluded_path), + self._read(remaining_path), + ) + + def _adjudication(self, identifier, task): + return { + "id": identifier, + "recordDisposition": "keep", + "dispositionConfidence": 0.95, + "dispositionEvidence": "send the report", + "resolutions": {"task": task}, + "confidence": {"task": 0.95}, + "evidence": {"task": "send the report"}, + } + + def _write(self, path, records): + path.write_text( + "\n".join(json.dumps(record) for record in records) + "\n", + encoding="utf-8", + ) + + def _read(self, path): + return [ + json.loads(line) + for line in path.read_text(encoding="utf-8").splitlines() + if line + ] + + +if __name__ == "__main__": + unittest.main() diff --git a/Scripts/clipboard_semantics/tests/test_assemble_v6_model_corpus.py b/Scripts/clipboard_semantics/tests/test_assemble_v6_model_corpus.py new file mode 100644 index 0000000..01df4e6 --- /dev/null +++ b/Scripts/clipboard_semantics/tests/test_assemble_v6_model_corpus.py @@ -0,0 +1,66 @@ +import json +import sys +import tempfile +import unittest +from pathlib import Path + + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +import assemble_v6_model_corpus as corpus + + +class AssembleV6ModelCorpusTests(unittest.TestCase): + def write_jsonl(self, path: Path, records: list[dict]) -> None: + path.write_text( + "".join(json.dumps(record) + "\n" for record in records), + encoding="utf-8", + ) + + def test_assembles_disjoint_train_and_evaluation_splits(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + train = root / "train.jsonl" + evaluation = root / "evaluation.jsonl" + self.write_jsonl( + train, + [{"id": "train-1", "text": "hello", "split": "train", "language": "en"}], + ) + self.write_jsonl( + evaluation, + [ + { + "id": "test-1", + "text": "world", + "split": "test", + "language": "en", + } + ], + ) + + records, report = corpus.assemble(train, evaluation) + + self.assertEqual(2, len(records)) + self.assertEqual({"test": 1, "train": 1}, report["splitCounts"]) + self.assertEqual(0, report["evaluationOverlapCount"]) + + def test_rejects_nfkc_casefold_overlap(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + train = root / "train.jsonl" + evaluation = root / "evaluation.jsonl" + self.write_jsonl( + train, + [{"id": "train-1", "text": "ＡＢＣ", "split": "train", "language": "en"}], + ) + self.write_jsonl( + evaluation, + [{"id": "test-1", "text": "abc", "split": "test", "language": "en"}], + ) + + with self.assertRaisesRegex(ValueError, "overlap"): + corpus.assemble(train, evaluation) + + +if __name__ == "__main__": + unittest.main() diff --git a/Scripts/clipboard_semantics/tests/test_blessing_corpus_tools.py b/Scripts/clipboard_semantics/tests/test_blessing_corpus_tools.py new file mode 100644 index 0000000..de6cf93 --- /dev/null +++ b/Scripts/clipboard_semantics/tests/test_blessing_corpus_tools.py @@ -0,0 +1,170 @@ +from __future__ import annotations + +import importlib.util +import sys +import unittest +from pathlib import Path + + +SCRIPT_DIRECTORY = Path(__file__).resolve().parent.parent + + +def load_module(name: str, file_name: str): + spec = importlib.util.spec_from_file_location(name, SCRIPT_DIRECTORY / file_name) + if spec is None or spec.loader is None: + raise RuntimeError(f"Unable to load {file_name}") + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +generator = load_module( + "generate_blessing_training_corpus", + "generate_blessing_training_corpus.py", +) +extractor = load_module( + "extract_lccc_blessing_candidates", + "extract_lccc_blessing_candidates.py", +) +benchmark_preparer = load_module( + "prepare_blessing_benchmark", + "prepare_blessing_benchmark.py", +) +benchmark_finalizer = load_module( + "finalize_blessing_benchmark", + "finalize_blessing_benchmark.py", +) + + +class BlessingCorpusGeneratorTests(unittest.TestCase): + def test_targets_are_balanced_per_language(self) -> None: + targets = generator.allocate_targets(generator.ZH_FAMILIES, 10_000) + positive = sum( + targets[family.name] + for family in generator.ZH_FAMILIES + if family.blessing + ) + negative = sum( + targets[family.name] + for family in generator.ZH_FAMILIES + if not family.blessing + ) + + self.assertEqual(positive, 5_000) + self.assertEqual(negative, 5_000) + + def test_generation_is_unique_and_partial_label_only(self) -> None: + records = generator.generate_language( + language="zh-Hans", + common_slots=generator.ZH_COMMON, + families=generator.ZH_FAMILIES, + target=1_000, + seed=generator.SEED, + reserved=set(), + ) + + self.assertEqual(len(records), 1_000) + self.assertEqual( + len({generator.fingerprint(record["text"]) for record in records}), + 1_000, + ) + self.assertTrue( + all(record["knownLabels"] == ["blessing"] for record in records) + ) + self.assertEqual(sum(record["blessing"] for record in records), 500) + + +class LCCCBlessingCandidateTests(unittest.TestCase): + def test_direct_wishes_are_positive(self) -> None: + self.assertEqual( + extractor.classify("祝你生日快乐，愿新的一岁平安顺利"), + ("positive", "direct_wish"), + ) + self.assertEqual( + extractor.classify("恭喜你顺利毕业"), + ("positive", "congratulation"), + ) + + def test_boundaries_are_negative(self) -> None: + self.assertEqual( + extractor.classify("帮我写一段生日祝福语"), + ("negative", "meta_request"), + ) + self.assertEqual( + extractor.classify("谢谢大家发来的生日祝福"), + ("negative", "received_thanks"), + ) + self.assertEqual( + extractor.classify("我们晚上一起庆祝项目上线"), + ("negative", "celebration_mention"), + ) + self.assertEqual( + extractor.classify("晚上好"), + ("negative", "plain_greeting"), + ) + + def test_question_is_not_promoted_to_direct_wish(self) -> None: + self.assertEqual( + extractor.classify("可以对我说一句生日快乐吗？"), + ("negative", "meta_request"), + ) + + +class BlessingBenchmarkTests(unittest.TestCase): + def test_selection_prioritizes_boundary_before_explicit_marker(self) -> None: + record = { + "text": "文档引用了“祝你生日快乐”作为写作示例。", + "blessing": False, + "sentiment": "neutral", + } + + self.assertEqual( + benchmark_preparer.selection_stratum(record), + "boundary_candidate", + ) + + def test_selection_includes_implicit_positive_language(self) -> None: + record = { + "text": "I hope you continue to know peace and happiness.", + "blessing": False, + "sentiment": "positive", + } + + self.assertEqual( + benchmark_preparer.selection_stratum(record), + "explicit_candidate", + ) + + def test_benchmark_split_is_deterministic(self) -> None: + first = benchmark_finalizer.benchmark_split("blessing-review-00042") + second = benchmark_finalizer.benchmark_split("blessing-review-00042") + + self.assertEqual(first, second) + self.assertIn(first, {"calibration", "test"}) + + def test_binary_kappa_reports_partial_agreement(self) -> None: + annotation_a = { + "1": {"label": True}, + "2": {"label": True}, + "3": {"label": False}, + "4": {"label": False}, + } + annotation_b = { + "1": {"label": True}, + "2": {"label": False}, + "3": {"label": False}, + "4": {"label": False}, + } + + agreement, kappa = benchmark_finalizer.binary_cohen_kappa( + annotation_a, + annotation_b, + ) + + self.assertEqual(agreement, 0.75) + self.assertEqual(kappa, 0.5) + + +if __name__ == "__main__": + unittest.main() diff --git a/Scripts/clipboard_semantics/tests/test_build_corpus_registry.py b/Scripts/clipboard_semantics/tests/test_build_corpus_registry.py new file mode 100644 index 0000000..4d32fbb --- /dev/null +++ b/Scripts/clipboard_semantics/tests/test_build_corpus_registry.py @@ -0,0 +1,216 @@ +import json +import sys +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +import build_corpus_registry as registry + + +class CorpusRegistryTests(unittest.TestCase): + def test_evaluation_overlap_blocks_training(self): + report, records, train, _, _ = self._build( + train_records=[ + self._record("train-1", "Send report 123", "train", task=True) + ], + evaluation_records=[ + self._record("eval-1", "Send report 123", "test", task=True) + ], + ) + + self.assertEqual(1, report["canonicalRecordCount"]) + self.assertEqual("evaluation-only", records[0]["allowedUse"]) + self.assertEqual([], train) + self.assertEqual(1, report["exactDuplicateCount"]) + + def test_unknown_labels_remain_unknown(self): + report, records, train, pilot, _ = self._build( + train_records=[ + { + **self._record( + "open-1", + "Could you send the report?", + "train", + task=True, + ), + "knownLabels": ["task"], + } + ], + all_intents_known=False, + ) + + self.assertEqual(1, report["trainCandidateCount"]) + self.assertEqual("true", records[0]["labels"]["task"]) + self.assertEqual("unknown", records[0]["labels"]["question"]) + self.assertEqual("unknown", records[0]["labels"]["assistantCommand"]) + self.assertEqual(1, len(train)) + self.assertEqual(1, len(pilot)) + + def test_domain_only_record_and_weight_reach_training_output(self): + report, records, train, _, _ = self._build( + train_records=[ + { + **self._record("domain-1", "Table for five", "train", task=False), + "domain": "dining", + "knownLabels": ["domain"], + "sampleWeight": 0.5, + } + ], + all_intents_known=False, + source_weight=0.7, + ) + + self.assertEqual(1, report["trainCandidateCount"]) + self.assertEqual("dining", records[0]["domain"]) + self.assertEqual(["domain"], train[0]["knownLabels"]) + self.assertEqual(0.35, train[0]["sampleWeight"]) + + def test_conflicting_source_labels_enter_human_review(self): + _, records, train, _, review = self._build( + train_records=[ + self._record("one", "Please send it.", "train", task=True), + self._record("two", "Please send it.", "train", task=False), + ], + ) + + self.assertEqual(["task"], records[0]["labelConflicts"]) + self.assertEqual([], train) + self.assertEqual(["task"], review[0]["conflicts"]) + + def test_rejects_unsafe_training_license(self): + report, records, train, pilot, _ = self._build( + train_records=[ + self._record("unsafe", "Research dialogue", "train", task=True) + ], + train_license="research-only", + ) + + self.assertEqual(0, report["canonicalRecordCount"]) + self.assertEqual({"unsafe-license": 1}, report["excludedRecordCounts"]) + self.assertEqual([], records) + self.assertEqual([], train) + self.assertEqual([], pilot) + + def test_pilot_uses_unique_clusters(self): + records = [ + self._record( + f"record-{index}", + f"Could you send report {index} before Friday?", + "train", + task=True, + ) + for index in range(4) + ] + _, _, _, pilot, _ = self._build( + train_records=records, + pilot_count=10, + ) + + self.assertEqual(1, len(pilot)) + + def _build( + self, + train_records, + evaluation_records=None, + all_intents_known=True, + train_license="MIT", + pilot_count=10, + source_weight=1.0, + ): + evaluation_records = evaluation_records or [] + with tempfile.TemporaryDirectory() as raw_directory: + directory = Path(raw_directory) + train_path = directory / "train.jsonl" + evaluation_path = directory / "evaluation.jsonl" + self._write_json_lines(train_path, train_records) + self._write_json_lines(evaluation_path, evaluation_records) + manifest_path = directory / "sources.json" + manifest_path.write_text( + json.dumps( + { + "schemaVersion": 1, + "sources": [ + { + "id": "train", + "path": str(train_path), + "license": train_license, + "defaultUse": "train", + "sourceType": "fixture", + "allIntentLabelsKnown": all_intents_known, + "sentimentKnown": False, + "weight": source_weight, + "required": True, + }, + { + "id": "evaluation", + "path": str(evaluation_path), + "license": "evaluation-only", + "defaultUse": "evaluation-only", + "sourceType": "fixture", + "allIntentLabelsKnown": True, + "sentimentKnown": False, + "required": True, + }, + ], + "excludedSources": [], + } + ), + encoding="utf-8", + ) + paths = { + name: directory / f"{name}.jsonl" + for name in ( + "registry", + "train_candidates", + "pilot", + "human_review", + ) + } + report_path = directory / "report.json" + report = registry.build( + SimpleNamespace( + repository_root=directory, + source_manifest=manifest_path, + report=report_path, + pilot_count=pilot_count, + **paths, + ) + ) + return ( + report, + self._read_json_lines(paths["registry"]), + self._read_json_lines(paths["train_candidates"]), + self._read_json_lines(paths["pilot"]), + self._read_json_lines(paths["human_review"]), + ) + + def _record(self, identifier, text, split, task): + return { + "id": identifier, + "text": text, + "language": "en", + "family": "fixture", + "split": split, + "task": task, + } + + def _write_json_lines(self, path, records): + path.write_text( + "\n".join(json.dumps(record) for record in records) + + ("\n" if records else ""), + encoding="utf-8", + ) + + def _read_json_lines(self, path): + return [ + json.loads(line) + for line in path.read_text(encoding="utf-8").splitlines() + if line + ] + + +if __name__ == "__main__": + unittest.main() diff --git a/Scripts/clipboard_semantics/tests/test_evaluate_product_policy_anchors.py b/Scripts/clipboard_semantics/tests/test_evaluate_product_policy_anchors.py new file mode 100644 index 0000000..e94ce2e --- /dev/null +++ b/Scripts/clipboard_semantics/tests/test_evaluate_product_policy_anchors.py @@ -0,0 +1,144 @@ +import json +import sys +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +import evaluate_product_policy_anchors as policy + + +class ProductPolicyAnchorTests(unittest.TestCase): + def test_prepare_and_evaluate_anchor_accuracy(self): + with tempfile.TemporaryDirectory() as raw_directory: + directory = Path(raw_directory) + anchors = directory / "anchors.json" + anchors.write_text( + json.dumps( + [ + { + "id": "one", + "text": "open my inbox", + "language": "en", + "expected": { + "recordDisposition": "exclude-device-command" + }, + }, + { + "id": "two", + "text": "Could you send the report?", + "language": "en", + "expected": { + "recordDisposition": "keep", + "replyableMessage": "true", + "task": "true", + "question": "true", + "ambiguous": "false", + }, + }, + ] + ), + encoding="utf-8", + ) + queue = directory / "queue.jsonl" + policy.prepare(SimpleNamespace(anchors=anchors, queue=queue)) + labeler = directory / "labeler.jsonl" + records = [ + self._record( + "one", + "open my inbox", + disposition="exclude-device-command", + replyableMessage="false", + task="true", + question="false", + ambiguous="false", + ), + self._record( + "two", + "Could you send the report?", + disposition="keep", + replyableMessage="true", + task="true", + question="true", + ambiguous="false", + ), + ] + labeler.write_text( + "\n".join(json.dumps(record) for record in records) + "\n", + encoding="utf-8", + ) + report = policy.evaluate( + SimpleNamespace( + anchors=anchors, + queue=queue, + labeler=[("test", labeler)], + minimum_accuracy=0.95, + report=directory / "report.json", + ) + ) + + self.assertTrue(report["eligibleForCorpusReadjudication"]) + self.assertEqual( + 1.0, + report["labelers"]["test"]["decisionAccuracy"], + ) + + def test_target_gate_ignores_non_target_mismatch(self): + anchors = [ + { + "id": "one", + "text": "take your time", + "language": "en", + "expected": { + "recordDisposition": "keep", + "replyableMessage": "true", + "task": "false", + "question": "false", + "ambiguous": "false", + }, + } + ] + adjudication = self._record( + "one", + "take your time", + disposition="keep", + replyableMessage="true", + task="false", + question="false", + ambiguous="true", + ) + + result = policy.evaluate_labeler( + anchors, + {"one": adjudication}, + {"replyableMessage", "task", "question"}, + ) + + self.assertEqual(1.0, result["gateDecisionAccuracy"]) + self.assertLess(result["decisionAccuracy"], 1.0) + + def _record(self, identifier, text, disposition, **values): + resolutions = { + field: ( + "unknown" + if field == "domain" + else "false" + ) + for field in policy.ANCHOR_FIELDS + } + resolutions.update(values) + return { + "id": identifier, + "recordDisposition": disposition, + "dispositionConfidence": 0.99, + "dispositionEvidence": text, + "resolutions": resolutions, + "confidence": {field: 0.99 for field in resolutions}, + "evidence": {field: text for field in resolutions}, + } + + +if __name__ == "__main__": + unittest.main() diff --git a/Scripts/clipboard_semantics/tests/test_finalize_v6_blind_holdout.py b/Scripts/clipboard_semantics/tests/test_finalize_v6_blind_holdout.py new file mode 100644 index 0000000..1e64dfc --- /dev/null +++ b/Scripts/clipboard_semantics/tests/test_finalize_v6_blind_holdout.py @@ -0,0 +1,181 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +import finalize_v6_blind_holdout as finalizer + + +class FinalizeV6BlindHoldoutTests(unittest.TestCase): + def test_review_accepts_each_high_confidence_field_independently(self): + primary = self._labelers( + "primary", + [ + self._states(task="true", question="true", domain="finance"), + self._states(task="true", question="true", domain="travel"), + self._states(task="true", question="false", domain="calendar"), + ], + ) + reviewers = self._labelers( + "reviewer", + [ + self._states(task="true", question="true", domain="finance"), + self._states(task="false", question="false", domain="travel"), + ], + ) + + states = finalizer.resolve_model_states( + "record-1", + primary, + reviewers, + in_review_queue=True, + ) + output = finalizer.output_record( + {"id": "record-1", "text": "Text", "language": "en"}, + states, + "test", + ) + + self.assertEqual("true", states["task"]) + self.assertEqual("unknown", states["question"]) + self.assertEqual("unknown", states["domain"]) + self.assertTrue(output["task"]) + self.assertIn("task", output["knownLabels"]) + self.assertNotIn("question", output["knownLabels"]) + self.assertNotIn("domain", output["knownLabels"]) + self.assertIsNone(output["domain"]) + + def test_human_fields_override_models_but_exclusion_infers_nothing(self): + model_states = self._states( + task="false", + question="false", + replyableMessage="false", + ) + overrides = finalizer.apply_human_overrides( + model_states, + { + "recordDisposition": "keep", + "task": "true", + "question": "unknown", + "replyableMessage": "true", + "ambiguous": "false", + }, + ) + + self.assertEqual( + ["task", "question", "replyableMessage", "ambiguous"], + overrides, + ) + self.assertEqual("true", model_states["task"]) + self.assertEqual("unknown", model_states["question"]) + self.assertEqual("true", model_states["replyableMessage"]) + + excluded_states = self._states(task="true") + excluded_overrides = finalizer.apply_human_overrides( + excluded_states, + {"recordDisposition": "exclude-device-command"}, + ) + self.assertEqual([], excluded_overrides) + self.assertEqual("true", excluded_states["task"]) + + def test_split_assigns_twenty_of_each_kind_per_language(self): + records = [ + { + "id": f"{language}-{index:03d}", + "text": f"{language} {index}", + "language": language, + } + for language in ("en", "zh-Hans") + for index in range(60) + ] + + assignments = finalizer.assign_splits(records, records_per_split=20) + + for language in ("en", "zh-Hans"): + counts = { + split: sum( + assignments[f"{language}-{index:03d}"] == split + for index in range(60) + ) + for split in finalizer.SPLITS + } + self.assertEqual( + {"validation": 20, "test": 20, "golden": 20}, + counts, + ) + self.assertEqual("validation", assignments["en-000"]) + self.assertEqual("test", assignments["en-020"]) + self.assertEqual("golden", assignments["en-040"]) + + def test_unknown_fields_are_not_known_or_positive(self): + states = self._states( + assistantCommand="unknown", + sentiment="unknown", + domain="unknown", + ) + states["ambiguous"] = "unknown" + + output = finalizer.output_record( + {"id": "record-1", "text": "unchanged", "language": "en"}, + states, + "golden", + ) + + self.assertFalse(output["assistantCommand"]) + self.assertEqual("neutral", output["sentiment"]) + self.assertIsNone(output["domain"]) + self.assertIsNone(output["ambiguous"]) + self.assertNotIn("assistantCommand", output["knownLabels"]) + self.assertNotIn("sentiment", output["knownLabels"]) + self.assertNotIn("domain", output["knownLabels"]) + self.assertNotEqual("train", output["split"]) + self.assertEqual("unchanged", output["text"]) + + def test_unreviewed_field_requires_unanimous_non_unknown_vote(self): + primary = self._labelers( + "primary", + [ + self._states(blessing="unknown"), + self._states(blessing="unknown"), + self._states(blessing="unknown"), + ], + ) + + states = finalizer.resolve_model_states( + "record-1", + primary, + [], + in_review_queue=False, + ) + + self.assertEqual("unknown", states["blessing"]) + self.assertEqual("false", states["task"]) + + def test_duplicate_ids_are_rejected(self): + with self.assertRaisesRegex(ValueError, "duplicate id"): + finalizer.unique_records( + [{"id": "same"}, {"id": "same"}], + "test input", + ) + + def _states(self, **overrides): + states = {field: "false" for field in finalizer.INTENT_LABELS} + states["sentiment"] = "neutral" + states["domain"] = "unknown" + states["ambiguous"] = "false" + states.update(overrides) + return states + + def _labelers(self, prefix, states): + return [ + (f"{prefix}-{index}", {"record-1": state}) + for index, state in enumerate(states) + ] + + +if __name__ == "__main__": + unittest.main() diff --git a/Scripts/clipboard_semantics/tests/test_generate_open_training_corpus.py b/Scripts/clipboard_semantics/tests/test_generate_open_training_corpus.py new file mode 100644 index 0000000..7577cc2 --- /dev/null +++ b/Scripts/clipboard_semantics/tests/test_generate_open_training_corpus.py @@ -0,0 +1,175 @@ +import json +import sys +import unittest +from pathlib import Path +from unittest import mock + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +import generate_open_training_corpus as corpus + + +class OpenTrainingCorpusTests(unittest.TestCase): + def test_schema_contains_new_intents_and_single_domain(self): + record = corpus.make_record( + record_id="one", + text="Play some jazz", + language="en", + family="fixture", + source_dataset="fixture", + source_license="CC0-1.0", + source_url="https://example.test/train", + source_revision="deadbeef", + source_split="train", + known_labels=corpus.known_labels_for_mapping( + "assistantCommand", + "media", + ), + labeling_method="fixture", + assistant_command=True, + domain="media", + ) + + self.assertEqual( + {"assistantCommand", "informationQuery", "systemNotification"} + <= set(corpus.INTENT_LABELS), + True, + ) + self.assertEqual("media", record["domain"]) + self.assertEqual( + [ + "assistantCommand", + "domain", + "informationQuery", + "systemNotification", + ], + record["knownLabels"], + ) + self.assertFalse(record["task"]) + self.assertFalse(record["informationQuery"]) + + def test_massive_mapping_is_conservative(self): + self.assertEqual( + ("assistantCommand", "calendar"), + corpus.massive_mapping("alarm_set"), + ) + self.assertEqual( + ("informationQuery", "generalKnowledge"), + corpus.massive_mapping("qa_factoid"), + ) + self.assertEqual( + ("task", "travel"), + corpus.massive_mapping("transport_taxi"), + ) + self.assertEqual((None, None), corpus.massive_mapping("general_greet")) + + def test_bitod_mapping_supports_official_chinese_intents(self): + self.assertEqual( + ("informationQuery", "dining"), + corpus.bitod_mapping("餐馆查询"), + ) + self.assertEqual(("task", "travel"), corpus.bitod_mapping("宾馆预订")) + self.assertEqual( + ("informationQuery", "travel"), + corpus.bitod_mapping("香港地铁"), + ) + self.assertEqual( + ("informationQuery", "weather"), + corpus.bitod_mapping("天气查询"), + ) + + def test_new_builders_are_pinned_optional_and_isolated(self): + builders = corpus.configured_source_builders() + names = {builder.name for builder in builders} + + self.assertTrue( + { + "SNIPS", + "MInDS-14 zh-CN", + "BiToD", + "RESTAURANTS-8K", + "FormosaNLU Synth v1", + } + <= names + ) + self.assertNotIn("CFPB", names) + self.assertNotIn("CLINC150", names) + self.assertNotIn("openclaw-zh-greetings", names) + self.assertNotIn("WeChat-AutoSendBless", names) + self.assertTrue(all(builder.optional for builder in builders)) + for revision in ( + corpus.SNIPS_REVISION, + corpus.MINDS14_REVISION, + corpus.BITOD_REVISION, + corpus.RESTAURANT8K_REVISION, + corpus.FORMOSA_NLU_REVISION, + ): + self.assertRegex(revision, r"^[0-9a-f]{40}$") + + def test_optional_builder_failure_is_reported_unavailable(self): + def unavailable(_seed): + raise corpus.SourceUnavailable("offline") + + sources, failures = corpus.build_available_sources( + (corpus.SourceBuilder("fixture", unavailable),), + seed=7, + allow_unavailable_sources=False, + ) + + self.assertEqual([], sources) + self.assertEqual("fixture", failures[0]["dataset"]) + self.assertIn("offline", failures[0]["reason"]) + + def test_snips_uses_only_official_train_intents(self): + def payload(url): + intent = next( + value for value in corpus.SNIPS_MAPPING if f"/{value}/" in url + ) + return json.dumps( + { + intent: [ + { + "data": [ + {"text": "example "}, + {"text": intent}, + ] + } + ] + } + ).encode() + + with mock.patch.object(corpus, "fetch_bytes", side_effect=payload): + records = corpus.snips_records(seed=3) + + self.assertEqual(6, len(records)) + self.assertTrue(all(record["sourceSplit"] == "train" for record in records)) + self.assertTrue( + all(record["sourceRevision"] == corpus.SNIPS_REVISION for record in records) + ) + play = next(record for record in records if "PlayMusic" in record["text"]) + self.assertTrue(play["assistantCommand"]) + self.assertEqual("media", play["domain"]) + + def test_formosa_synthetic_records_are_simplified_and_keep_low_weight(self): + payload = ( + json.dumps( + { + "id": "syn-1", + "utt": "播放爵士樂", + "intent": "play_music", + } + ) + + "\n" + ).encode() + with mock.patch.object(corpus, "fetch_bytes", return_value=payload): + records = corpus.formosa_nlu_records(seed=3) + + self.assertEqual(1, len(records)) + self.assertEqual("播放爵士乐", records[0]["text"]) + self.assertEqual("zh-Hans", records[0]["language"]) + self.assertEqual(0.35, records[0]["sampleWeight"]) + self.assertEqual("train", records[0]["sourceSplit"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/Scripts/clipboard_semantics/tests/test_generate_v6_boundary_corpus.py b/Scripts/clipboard_semantics/tests/test_generate_v6_boundary_corpus.py new file mode 100644 index 0000000..1daa927 --- /dev/null +++ b/Scripts/clipboard_semantics/tests/test_generate_v6_boundary_corpus.py @@ -0,0 +1,199 @@ +import hashlib +import json +import sys +import tempfile +import unicodedata +import unittest +from collections import Counter +from pathlib import Path + + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +import generate_v6_boundary_corpus as corpus + + +REQUIRED_FIELDS = { + "id", + "text", + "language", + "split", + "family", + "task", + "question", + "invitation", + "complaint", + "scheduleNegotiation", + "confirmationDecision", + "followUpReminder", + "blessing", + "sentiment", + "replyable", + "assistantCommand", + "informationQuery", + "systemNotification", + "domain", + "sampleWeight", + "knownLabels", + "labelingMethod", + "sourceDataset", + "sourceLicense", + "sourceURL", + "sourceRevision", + "sourceSplit", + "synthetic", + "templateFamily", +} + + +class V6BoundaryCorpusTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.records, cls.target_counts, cls.excluded = corpus.generate_records() + cls.payload = corpus.serialize_records(cls.records) + + def test_default_generation_is_reproducible(self): + records, target_counts, excluded = corpus.generate_records() + payload = corpus.serialize_records(records) + + self.assertEqual(self.payload, payload) + self.assertEqual(self.target_counts, target_counts) + self.assertEqual(self.excluded, excluded) + self.assertEqual( + hashlib.sha256(self.payload).hexdigest(), + hashlib.sha256(payload).hexdigest(), + ) + + def test_default_scale_language_intent_and_domain_balance(self): + self.assertGreaterEqual(len(self.records), 6_000) + self.assertEqual( + {"en": 3_600, "zh-Hans": 3_600}, + Counter(record["language"] for record in self.records), + ) + for language in corpus.LANGUAGES: + for intent in corpus.NEW_INTENTS: + self.assertEqual( + 1_000, + self.target_counts[f"{language}|{intent}"], + ) + for intent in corpus.HARD_NEGATIVE_INTENTS: + self.assertEqual( + 200, + self.target_counts[f"{language}|{intent}"], + ) + self.assertEqual( + {domain: 600 for domain in corpus.DOMAINS}, + Counter(record["domain"] for record in self.records), + ) + + def test_records_have_full_schema_low_weight_and_known_boundaries(self): + for record in self.records: + self.assertEqual(REQUIRED_FIELDS, set(record)) + self.assertEqual(0.35, record["sampleWeight"]) + self.assertEqual("train", record["split"]) + self.assertEqual("train", record["sourceSplit"]) + self.assertEqual(corpus.SOURCE_DATASET, record["sourceDataset"]) + self.assertEqual(corpus.SOURCE_LICENSE, record["sourceLicense"]) + self.assertEqual(corpus.SOURCE_REVISION, record["sourceRevision"]) + self.assertTrue(record["synthetic"]) + self.assertEqual(record["family"], record["templateFamily"]) + self.assertEqual(corpus.KNOWN_LABELS, record["knownLabels"]) + self.assertEqual(record["text"], unicodedata.normalize("NFKC", record["text"])) + + routing_count = sum(record[intent] for intent in corpus.NEW_INTENTS) + if any(record[intent] for intent in corpus.NEW_INTENTS): + self.assertEqual(1, routing_count) + self.assertFalse(record["task"]) + self.assertFalse(record["question"]) + self.assertFalse(record["replyable"]) + else: + self.assertEqual(0, routing_count) + self.assertTrue( + record["task"] or record["question"] or record["replyable"] + ) + + def test_generated_ids_and_normalized_texts_are_unique(self): + ids = [record["id"] for record in self.records] + texts = [corpus.fingerprint(record["text"]) for record in self.records] + + self.assertEqual(len(ids), len(set(ids))) + self.assertEqual(len(texts), len(set(texts))) + + def test_holdout_discovery_and_nfkc_overlap_exclusion(self): + system_record = next( + record + for record in self.records + if record["language"] == "zh-Hans" + and record["systemNotification"] + and ":" in record["text"] + ) + blind_record = next( + record + for record in self.records + if record["language"] == "en" and record["informationQuery"] + ) + with tempfile.TemporaryDirectory() as temporary_directory: + directory = Path(temporary_directory) + wildcard_holdout = directory / "frozen-holdout-corpus.jsonl" + blind_holdout = directory / "product-policy-blind-holdout-v1.jsonl" + wildcard_holdout.write_text( + json.dumps( + {"text": system_record["text"].replace(":", "：")}, + ensure_ascii=False, + ) + + "\n", + encoding="utf-8", + ) + blind_holdout.write_text( + json.dumps({"text": blind_record["text"]}) + "\n", + encoding="utf-8", + ) + + paths = corpus.discover_holdout_paths(directory) + holdouts = corpus.load_holdout_fingerprints(paths) + records, _, excluded = corpus.generate_records( + holdout_fingerprints=holdouts + ) + + generated = {corpus.fingerprint(record["text"]) for record in records} + self.assertEqual({wildcard_holdout, blind_holdout}, set(paths)) + self.assertNotIn(corpus.fingerprint(system_record["text"]), generated) + self.assertNotIn(corpus.fingerprint(blind_record["text"]), generated) + self.assertGreaterEqual(excluded, 2) + self.assertEqual(len(self.records), len(records)) + + def test_summary_counts_and_hash_match_written_jsonl(self): + with tempfile.TemporaryDirectory() as temporary_directory: + directory = Path(temporary_directory) + output = directory / "corpus.jsonl" + summary_path = directory / "summary.json" + + summary = corpus.write_corpus( + output, + summary_path, + positive_per_intent_language=12, + negative_per_intent_language=12, + ) + persisted = json.loads(summary_path.read_text(encoding="utf-8")) + output_sha256 = hashlib.sha256(output.read_bytes()).hexdigest() + + self.assertEqual(summary, persisted) + self.assertEqual(output_sha256, summary["corpusSHA256"]) + self.assertEqual(144, summary["recordCount"]) + self.assertEqual( + {"en": 72, "zh-Hans": 72}, + summary["counts"]["byLanguage"], + ) + self.assertEqual(48, summary["counts"]["byIntent"]["replyableMessage"]) + self.assertEqual( + 12, + summary["counts"]["byBoundaryTargetAndLanguage"]["en"][ + "replyableMessage" + ], + ) + self.assertEqual(set(corpus.DOMAINS), set(summary["counts"]["byDomain"])) + self.assertTrue(summary["counts"]["byTemplateFamily"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/Scripts/clipboard_semantics/tests/test_iterative_retraining.py b/Scripts/clipboard_semantics/tests/test_iterative_retraining.py index 419e547..62ad8ee 100644 --- a/Scripts/clipboard_semantics/tests/test_iterative_retraining.py +++ b/Scripts/clipboard_semantics/tests/test_iterative_retraining.py @@ -91,6 +91,30 @@ class IterativeRetrainingTests(unittest.TestCase): self.assertEqual([False, True], predicted["task"].tolist()) + def test_registry_sample_weight_is_applied(self): + configuration = research.configurations()[0] + record = { + "id": "weighted", + "text": "Play music", + "language": "en", + "knownLabels": ["assistantCommand"], + "assistantCommand": True, + "sampleWeight": 0.35, + } + + self.assertIn("assistantCommand", research.INTENTS) + self.assertAlmostEqual( + configuration.external_weight * 0.35, + research.sample_weight(record, configuration), + ) + + def test_legacy_records_do_not_define_new_labels_as_false(self): + record = {"id": "legacy", "text": "Play music", "language": "en"} + + self.assertTrue(research.is_known(record, "task")) + self.assertFalse(research.is_known(record, "assistantCommand")) + self.assertFalse(research.is_known(record, "systemNotification")) + if __name__ == "__main__": unittest.main() diff --git a/Scripts/clipboard_semantics/tests/test_merge_consensus_labels_v2.py b/Scripts/clipboard_semantics/tests/test_merge_consensus_labels_v2.py new file mode 100644 index 0000000..9ce88c7 --- /dev/null +++ b/Scripts/clipboard_semantics/tests/test_merge_consensus_labels_v2.py @@ -0,0 +1,286 @@ +import json +import sys +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +import merge_consensus_labels_v2 as consensus + + +class ConsensusV2Tests(unittest.TestCase): + def test_primary_unanimous_becomes_tier_a(self): + report, tier_a, tier_b, human = self._merge( + primary_states=[self._states(task="true")] * 3, + review_states=[], + ) + + self.assertEqual(1, report["tierACount"]) + self.assertEqual(1, len(tier_a)) + self.assertTrue(tier_a[0]["task"]) + self.assertEqual(1.0, tier_a[0]["sampleWeight"]) + self.assertEqual([], tier_b) + self.assertEqual([], human) + + def test_four_of_five_becomes_tier_b(self): + report, tier_a, tier_b, human = self._merge( + primary_states=[ + self._states(task="true"), + self._states(task="true"), + self._states(task="false"), + ], + review_states=[ + self._states(task="true"), + self._states(task="true"), + ], + ) + + self.assertEqual(0, report["tierACount"]) + self.assertEqual(1, report["tierBCount"]) + self.assertEqual([], tier_a) + self.assertTrue(tier_b[0]["task"]) + self.assertEqual(0.65, tier_b[0]["sampleWeight"]) + self.assertEqual([], human) + + def test_three_two_vote_requires_human_review(self): + report, _, tier_b, human = self._merge( + primary_states=[ + self._states(task="true"), + self._states(task="true"), + self._states(task="false"), + ], + review_states=[ + self._states(task="true"), + self._states(task="false"), + ], + ) + + self.assertEqual(1, report["humanReviewCount"]) + self.assertEqual([], tier_b) + self.assertIn("task", human[0]["unresolvedFields"]) + + def test_unknown_primary_state_triggers_review(self): + with tempfile.TemporaryDirectory() as raw_directory: + directory = Path(raw_directory) + queue = directory / "queue.jsonl" + self._write( + queue, + [{"id": "record-1", "text": "Hello", "language": "en"}], + ) + primary = [] + for index, state in enumerate( + [ + self._states(question="unknown"), + self._states(question="false"), + self._states(question="false"), + ] + ): + path = directory / f"primary-{index}.jsonl" + self._write(path, [self._label("record-1", state)]) + primary.append((f"primary-{index}", path)) + output = directory / "review.jsonl" + report_path = directory / "report.json" + report = consensus.prepare_review( + SimpleNamespace( + queue=queue, + primary=primary, + output=output, + report=report_path, + ) + ) + + self.assertEqual(1, report["reviewCount"]) + self.assertEqual("record-1", self._read(output)[0]["id"]) + + def test_quoted_positive_is_sent_to_human(self): + report, _, _, human = self._merge( + primary_states=[ + self._states(blessing="true"), + self._states(blessing="true"), + self._states(blessing="false"), + ], + review_states=[ + self._states(blessing="true"), + self._states(blessing="true"), + ], + review_quoted=[True, True], + ) + + self.assertEqual(1, report["humanReviewCount"]) + self.assertIn("quotedOrMeta", human[0]["unresolvedFields"]) + + def test_unanimous_unknown_stays_out_of_known_labels(self): + _, tier_a, _, _ = self._merge( + primary_states=[self._states(assistantCommand="unknown")] * 3, + review_states=[], + ) + + self.assertEqual(1, len(tier_a)) + self.assertNotIn("assistantCommand", tier_a[0]["knownLabels"]) + self.assertFalse(tier_a[0]["assistantCommand"]) + self.assertIsNone(tier_a[0]["domain"]) + + def test_split_and_combine_preserve_queue_order(self): + with tempfile.TemporaryDirectory() as raw_directory: + directory = Path(raw_directory) + queue_path = directory / "queue.jsonl" + queue = [ + {"id": f"record-{index}", "text": str(index), "language": "en"} + for index in range(5) + ] + self._write(queue_path, queue) + chunks = directory / "chunks" + split_report = consensus.split_queue( + SimpleNamespace( + queue=queue_path, + output_directory=chunks, + chunk_size=2, + report=directory / "split-report.json", + ) + ) + self.assertEqual(3, split_report["chunkCount"]) + outputs = [] + for path in sorted(chunks.glob("*.jsonl")): + output_path = directory / f"labeled-{path.name}" + self._write( + output_path, + [ + self._label(record["id"], self._states()) + for record in self._read(path) + ], + ) + outputs.append(output_path) + combined_path = directory / "combined.jsonl" + report = consensus.combine_labeler( + SimpleNamespace( + queue=queue_path, + input=outputs, + output=combined_path, + report=directory / "combine-report.json", + ) + ) + + self.assertEqual(5, report["outputCount"]) + self.assertEqual( + [record["id"] for record in queue], + [record["id"] for record in self._read(combined_path)], + ) + + def _merge( + self, + primary_states, + review_states, + review_quoted=None, + ): + review_quoted = review_quoted or [False] * len(review_states) + with tempfile.TemporaryDirectory() as raw_directory: + directory = Path(raw_directory) + queue_path = directory / "queue.jsonl" + self._write( + queue_path, + [ + { + "id": "record-1", + "text": "Could you send the report?", + "language": "en", + } + ], + ) + primary = [] + for index, state in enumerate(primary_states): + path = directory / f"primary-{index}.jsonl" + self._write(path, [self._label("record-1", state)]) + primary.append((f"primary-{index}", path)) + review = [] + review_required = ( + len( + consensus.primary_review_ids( + consensus.load_labelers( + primary, + {"record-1"}, + True, + ), + {"record-1"}, + ) + ) + == 1 + ) + if review_required: + for index, state in enumerate(review_states): + path = directory / f"review-{index}.jsonl" + self._write( + path, + [ + self._label( + "record-1", + state, + quoted=review_quoted[index], + ) + ], + ) + review.append((f"review-{index}", path)) + else: + for index in range(2): + path = directory / f"review-{index}.jsonl" + self._write(path, []) + review.append((f"review-{index}", path)) + outputs = { + name: directory / f"{name}.jsonl" + for name in ("tier_a", "tier_b", "accepted", "human_review") + } + report_path = directory / "report.json" + report = consensus.merge( + SimpleNamespace( + queue=queue_path, + primary=primary, + reviewer=review, + report=report_path, + **outputs, + ) + ) + return ( + report, + self._read(outputs["tier_a"]), + self._read(outputs["tier_b"]), + self._read(outputs["human_review"]), + ) + + def _states(self, **overrides): + values = {label: "false" for label in consensus.INTENT_LABELS} + values.update(overrides) + values["sentiment"] = overrides.get("sentiment", "neutral") + values["domain"] = overrides.get("domain", "unknown") + return values + + def _label(self, identifier, states, quoted=False): + return { + "id": identifier, + "labels": { + label: states[label] for label in consensus.INTENT_LABELS + }, + "sentiment": states["sentiment"], + "domain": states["domain"], + "ambiguous": False, + "quotedOrMeta": quoted, + "confidence": 0.95, + } + + def _write(self, path, records): + path.write_text( + "\n".join(json.dumps(record) for record in records) + + ("\n" if records else ""), + encoding="utf-8", + ) + + def _read(self, path): + return [ + json.loads(line) + for line in path.read_text(encoding="utf-8").splitlines() + if line + ] + + +if __name__ == "__main__": + unittest.main() diff --git a/Scripts/clipboard_semantics/tests/test_prepare_apple_nl_hardening_corpus.py b/Scripts/clipboard_semantics/tests/test_prepare_apple_nl_hardening_corpus.py new file mode 100644 index 0000000..50b01ab --- /dev/null +++ b/Scripts/clipboard_semantics/tests/test_prepare_apple_nl_hardening_corpus.py @@ -0,0 +1,63 @@ +import importlib.util +import unittest +from pathlib import Path + + +SCRIPT_PATH = ( + Path(__file__).resolve().parents[1] / "prepare_apple_nl_hardening_corpus.py" +) +SPEC = importlib.util.spec_from_file_location("prepare_apple_nl_hardening_corpus", SCRIPT_PATH) +MODULE = importlib.util.module_from_spec(SPEC) +assert SPEC.loader is not None +SPEC.loader.exec_module(MODULE) + + +class PrepareAppleNLHardeningCorpusTests(unittest.TestCase): + def test_prepare_removes_id_and_normalized_text_overlap(self) -> None: + training = [ + {"id": "train-1", "text": "Keep me", "language": "en", "split": "train"}, + {"id": "shared-id", "text": "Different", "language": "en", "split": "train"}, + {"id": "train-3", "text": " SAME TEXT ", "language": "en", "split": "train"}, + ] + product = [ + { + "id": "shared-id", + "text": "Evaluation by id", + "language": "en", + "split": "validation", + }, + { + "id": "evaluation-2", + "text": "same text", + "language": "en", + "split": "test", + }, + { + "id": "ignored-train", + "text": "Not evaluation", + "language": "en", + "split": "train", + }, + ] + + records, report = MODULE.prepare(training, product) + + self.assertEqual([record["id"] for record in records], [ + "train-1", + "shared-id", + "evaluation-2", + ]) + self.assertEqual(report["filteredTrainingCount"], 1) + self.assertEqual(report["excludedTrainingOverlapCount"], 2) + self.assertEqual(report["evaluationOverlapCount"], 0) + + def test_prepare_requires_evaluation_records(self) -> None: + with self.assertRaisesRegex(ValueError, "no evaluation records"): + MODULE.prepare( + [{"id": "train", "text": "x", "language": "en", "split": "train"}], + [{"id": "product", "text": "y", "language": "en", "split": "train"}], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/Scripts/clipboard_semantics/train_models.swift b/Scripts/clipboard_semantics/train_models.swift index b718781..b8aab06 100644 --- a/Scripts/clipboard_semantics/train_models.swift +++ b/Scripts/clipboard_semantics/train_models.swift @@ -11,6 +11,7 @@ private struct CorpusRecord: Codable { let family: String let knownLabels: Set? let sourceDataset: String? + let sampleWeight: Double? let task: Bool let question: Bool let invitation: Bool @@ -21,6 +22,10 @@ private struct CorpusRecord: Codable { let blessing: Bool let sentiment: String let replyable: Bool + let assistantCommand: Bool? + let informationQuery: Bool? + let systemNotification: Bool? + let domain: String? } private struct BinaryMetrics: Codable { @@ -141,6 +146,10 @@ private enum ClassifierID: String, CaseIterable { case followUpReminder case blessing case replyableMessage + case assistantCommand + case informationQuery + case systemNotification + case domain case sentiment var resourceName: String { @@ -154,6 +163,10 @@ private enum ClassifierID: String, CaseIterable { case .followUpReminder: "FollowUpReminderIntentClassifier" case .blessing: "BlessingIntentClassifier" case .replyableMessage: "ConversationalReplyIntentClassifier" + case .assistantCommand: "AssistantCommandIntentClassifier" + case .informationQuery: "InformationQueryIntentClassifier" + case .systemNotification: "SystemNotificationIntentClassifier" + case .domain: "ClipboardDomainClassifier" case .sentiment: "SentimentClassifier" } } @@ -169,6 +182,14 @@ private enum ClassifierID: String, CaseIterable { case .followUpReminder: ["notFollowUpReminder", "followUpReminder"] case .blessing: ["notBlessing", "blessing"] case .replyableMessage: ["notReplyableMessage", "replyableMessage"] + case .assistantCommand: ["notAssistantCommand", "assistantCommand"] + case .informationQuery: ["notInformationQuery", "informationQuery"] + case .systemNotification: ["notSystemNotification", "systemNotification"] + case .domain: + [ + "finance", "travel", "calendar", "communication", "media", "smartHome", + "shopping", "dining", "health", "weather", "accountService", "generalKnowledge" + ] case .sentiment: ["negative", "neutral", "positive"] } } @@ -184,7 +205,10 @@ private enum ClassifierID: String, CaseIterable { case .followUpReminder: "followUpReminder" case .blessing: "blessing" case .replyableMessage: "replyableMessage" - case .sentiment: nil + case .assistantCommand: "assistantCommand" + case .informationQuery: "informationQuery" + case .systemNotification: "systemNotification" + case .domain, .sentiment: nil } } @@ -297,7 +321,8 @@ private enum ClassifierID: String, CaseIterable { "task_question", "task_statement" ] - case .question, .replyableMessage, .sentiment: + case .question, .replyableMessage, .assistantCommand, .informationQuery, + .systemNotification, .domain, .sentiment: return [] } } @@ -314,7 +339,8 @@ private enum ClassifierID: String, CaseIterable { 0.75 case .confirmationDecision: 0.90 - case .question, .replyableMessage, .sentiment: + case .question, .replyableMessage, .assistantCommand, .informationQuery, + .systemNotification, .domain, .sentiment: 0 } } @@ -334,12 +360,31 @@ private enum ClassifierID: String, CaseIterable { case .blessing: record.blessing ? "blessing" : "notBlessing" case .replyableMessage: record.replyable ? "replyableMessage" : "notReplyableMessage" + case .assistantCommand: + record.assistantCommand == true ? "assistantCommand" : "notAssistantCommand" + case .informationQuery: + record.informationQuery == true ? "informationQuery" : "notInformationQuery" + case .systemNotification: + record.systemNotification == true ? "systemNotification" : "notSystemNotification" + case .domain: + record.domain ?? { preconditionFailure("Known domain record is missing domain") }() case .sentiment: record.sentiment } } func hasKnownLabel(in record: CorpusRecord) -> Bool { - record.knownLabels?.contains(rawValue) ?? true + if let knownLabels = record.knownLabels { + return knownLabels.contains(rawValue) + || (self == .replyableMessage && knownLabels.contains("replyable")) + } + // Legacy product corpora predate knownLabels and only fully annotate + // the original nine intents plus sentiment. New fields must stay unknown. + switch self { + case .assistantCommand, .informationQuery, .systemNotification, .domain: + return false + default: + return true + } } } @@ -397,6 +442,7 @@ private let reportURL = resolvedURL( flag: "--report", defaultPath: "ModelTraining/ClipboardSemantics/evaluation-report.json" ) +private let requestedLanguage = commandLineValue(after: "--language") private func loadCorpus() throws -> [CorpusRecord] { let content = try String(contentsOf: corpusURL, encoding: .utf8) @@ -419,9 +465,13 @@ private func sourceBalancedPrefix( classifier: ClassifierID, label: String ) -> [CorpusRecord] { - guard records.count > limit else { return records } + // MLTextClassifier's dictionary API has no per-example weight parameter. + // Quantized weight buckets plus deterministic smooth weighted round-robin + // preserve registry weights without introducing nondeterministic duplication. var grouped = Dictionary(grouping: records) { - $0.sourceDataset ?? "generated" + let weight = min(max($0.sampleWeight ?? 1.0, 0.01), 1.0) + let bucket = (weight * 100).rounded() / 100 + return "\($0.sourceDataset ?? "generated")|weight=\(bucket)" } for source in grouped.keys.sorted() { var generator = SeededGenerator( @@ -431,24 +481,49 @@ private func sourceBalancedPrefix( ) ) grouped[source]?.shuffle(using: &generator) + if let values = grouped[source], let first = values.first { + let weight = min(max(first.sampleWeight ?? 1.0, 0.01), 1.0) + let weightedCount = max(1, Int((Double(values.count) * weight).rounded())) + grouped[source] = Array(values.prefix(weightedCount)) + } } let sources = grouped.keys.sorted() + let weightedTotal = grouped.values.reduce(0) { $0 + $1.count } + if weightedTotal <= limit { + return sources.flatMap { grouped[$0] ?? [] } + } var offsets = Dictionary(uniqueKeysWithValues: sources.map { ($0, 0) }) + let sourceWeights = Dictionary(uniqueKeysWithValues: sources.map { source in + (source, grouped[source]?.first?.sampleWeight ?? 1.0) + }) + var schedulingScores = Dictionary(uniqueKeysWithValues: sources.map { ($0, 0.0) }) var selected: [CorpusRecord] = [] while selected.count < limit { - var addedRecord = false - for source in sources where selected.count < limit { - let offset = offsets[source] ?? 0 - guard let values = grouped[source], values.indices.contains(offset) else { - continue - } - selected.append(values[offset]) - offsets[source] = offset + 1 - addedRecord = true + let available = sources.filter { + let offset = offsets[$0] ?? 0 + return grouped[$0]?.indices.contains(offset) == true } - if !addedRecord { + if available.isEmpty { break } + let totalWeight = available.reduce(0.0) { + $0 + max(sourceWeights[$1] ?? 1.0, 0.01) + } + for source in available { + schedulingScores[source, default: 0] += max( + sourceWeights[source] ?? 1.0, + 0.01 + ) + } + let source = available.max { + let left = schedulingScores[$0, default: 0] + let right = schedulingScores[$1, default: 0] + return left == right ? $0 > $1 : left < right + }! + let offset = offsets[source] ?? 0 + selected.append(grouped[source]![offset]) + offsets[source] = offset + 1 + schedulingScores[source, default: 0] -= totalWeight } return selected } @@ -462,18 +537,28 @@ private func curatedTrainingRecords( } let generatedRecords = knownRecords.filter { $0.sourceDataset == nil } let openRecords = knownRecords.filter { $0.sourceDataset != nil } - guard !openRecords.isEmpty else { return generatedRecords } + let weightedGeneratedRecords = sourceBalancedPrefix( + generatedRecords, + limit: generatedRecords.count, + classifier: classifier, + label: "generated" + ) + guard !openRecords.isEmpty else { return weightedGeneratedRecords } - let generatedByLabel = Dictionary(grouping: generatedRecords) { + let generatedByLabel = Dictionary(grouping: weightedGeneratedRecords) { classifier.label(for: $0) } let openByLabel = Dictionary(grouping: openRecords) { classifier.label(for: $0) } + let openOnlyBalancedCount = classifier.labels + .compactMap { openByLabel[$0]?.count } + .min() ?? 0 let multiplier = switch classifier { case .blessing: 2.0 - case .task, .question, .complaint, .confirmationDecision, .sentiment: + case .task, .question, .complaint, .confirmationDecision, .assistantCommand, + .informationQuery, .systemNotification, .domain, .sentiment: 1.0 case .invitation, .scheduleNegotiation, .followUpReminder, .replyableMessage: 0.5 @@ -481,7 +566,8 @@ private func curatedTrainingRecords( let selectedOpenRecords = classifier.labels.flatMap { label in let generatedCount = generatedByLabel[label]?.count ?? 0 - let limit = max(1, Int((Double(generatedCount) * multiplier).rounded())) + let anchorCount = generatedCount > 0 ? generatedCount : openOnlyBalancedCount + let limit = max(1, Int((Double(anchorCount) * multiplier).rounded())) return sourceBalancedPrefix( openByLabel[label] ?? [], limit: limit, @@ -489,7 +575,7 @@ private func curatedTrainingRecords( label: label ) } - return generatedRecords + selectedOpenRecords + return weightedGeneratedRecords + selectedOpenRecords } private func balancedTexts( @@ -1192,15 +1278,36 @@ private func writeJSON(_ value: T, to url: URL) throws { } private func main() throws { - let records = try loadCorpus() + let loadedRecords = try loadCorpus() + let records = requestedLanguage.map { language in + loadedRecords.filter { $0.language == language } + } ?? loadedRecords + precondition(!records.isEmpty, "No corpus records match the requested language") let trainingRecords = records.filter { $0.split == "train" } let validationRecords = records.filter { $0.split == "validation" } let testRecords = records.filter { $0.split == "test" } let goldenRecords = records.filter { $0.split == "golden" } let algorithms = selectedAlgorithms() - let classifiers = selectedClassifiers() + let requestedClassifiers = selectedClassifiers() + let classifiers = requestedClassifiers.filter { classifier in + let trainingLabels = Set( + trainingRecords + .filter { classifier.hasKnownLabel(in: $0) } + .map { classifier.label(for: $0) } + ) + return Set(classifier.labels).isSubset(of: trainingLabels) + && [validationRecords, testRecords, goldenRecords].allSatisfy { + !$0.filter { classifier.hasKnownLabel(in: $0) }.isEmpty + } + } + for classifier in requestedClassifiers where !classifiers.contains(classifier) { + print( + "TRAIN_SKIPPED classifier=\(classifier.rawValue) " + + "reason=insufficient-known-label-coverage" + ) + } precondition(!algorithms.isEmpty, "No supported algorithms requested") - precondition(!classifiers.isEmpty, "No supported classifiers requested") + precondition(!classifiers.isEmpty, "No classifiers have sufficient known-label coverage") try fileManager.createDirectory( at: resourceDirectory, @@ -1212,6 +1319,15 @@ private func main() throws { for classifierID in classifiers { var candidates: [TrainedCandidate] = [] + let knownValidationRecords = validationRecords.filter { + classifierID.hasKnownLabel(in: $0) + } + let knownTestRecords = testRecords.filter { + classifierID.hasKnownLabel(in: $0) + } + let knownGoldenRecords = goldenRecords.filter { + classifierID.hasKnownLabel(in: $0) + } for algorithm in algorithms { do { candidates.append( @@ -1219,9 +1335,9 @@ private func main() throws { classifierID: classifierID, algorithm: algorithm, trainingRecords: trainingRecords, - validationRecords: validationRecords, - testRecords: testRecords, - goldenRecords: goldenRecords + validationRecords: knownValidationRecords, + testRecords: knownTestRecords, + goldenRecords: knownGoldenRecords ) ) } catch { @@ -1288,7 +1404,9 @@ private func main() throws { goldenCount: goldenRecords.count, selectionPolicy: "Open records with unknown labels are excluded per classifier, and source-balanced " - + "caps anchor each label to the reviewed generated corpus size. " + + "caps anchor each label to the reviewed generated corpus size. Because Create ML " + + "does not expose per-example weights, registry sampleWeight values are applied as " + + "deterministic quantized quotas with smooth weighted source scheduling. " + "Validation only: global and per-language binary thresholds require precision " + ">= 0.97, then maximize recall; languages with fewer than 20 examples per class " + "fall back to the global threshold. " @@ -1300,7 +1418,7 @@ private func main() throws { try writeJSON(report, to: reportURL) try writeJSON( ModelManifest( - schemaVersion: 2, + schemaVersion: 4, generatedAt: generatedAt, corpusRecordCount: records.count, classifiers: manifestClassifiers diff --git a/Scripts/clipboard_semantics/train_v6_tiny_transformers.py b/Scripts/clipboard_semantics/train_v6_tiny_transformers.py new file mode 100644 index 0000000..3709cc0 --- /dev/null +++ b/Scripts/clipboard_semantics/train_v6_tiny_transformers.py @@ -0,0 +1,491 @@ +#!/usr/bin/env python3 +"""Fine-tune bilingual Tiny Transformer challengers for taxonomy-v6 intents.""" + +from __future__ import annotations + +import argparse +import json +import random +import resource +import time +from dataclasses import dataclass +from pathlib import Path + +import numpy as np +import torch +from torch.utils.data import DataLoader, Dataset +from transformers import AutoModelForSequenceClassification, AutoTokenizer + + +INTENTS = ( + "task", + "question", + "invitation", + "complaint", + "scheduleNegotiation", + "confirmationDecision", + "followUpReminder", + "blessing", + "replyableMessage", + "assistantCommand", + "informationQuery", + "systemNotification", +) +LANGUAGE_MODELS = { + "en": "en", + "zh-Hans": "zh", +} + + +def parse_arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--corpus", type=Path, required=True) + parser.add_argument("--output-directory", type=Path, required=True) + parser.add_argument("--english-model", type=Path, required=True) + parser.add_argument("--chinese-model", type=Path, required=True) + parser.add_argument("--epochs", type=int, default=3) + parser.add_argument("--batch-size", type=int, default=128) + parser.add_argument("--max-length", type=int, default=96) + parser.add_argument("--learning-rate", type=float, default=3e-4) + parser.add_argument("--seed", type=int, default=20260828) + return parser.parse_args() + + +def set_seed(seed: int) -> None: + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + if torch.backends.mps.is_available(): + torch.mps.manual_seed(seed) + + +def intent_value(record: dict, intent: str) -> bool: + if intent == "replyableMessage": + return bool(record.get(intent, record.get("replyable", False))) + return bool(record.get(intent, False)) + + +def intent_mask(record: dict) -> list[float]: + known_labels = record.get("knownLabels") + if known_labels is None: + return [1.0] * len(INTENTS) + known = set(known_labels) + return [float(intent in known) for intent in INTENTS] + + +def load_records(path: Path) -> dict[str, dict[str, list[dict]]]: + records = { + language: {"train": [], "validation": [], "test": [], "golden": []} + for language in LANGUAGE_MODELS + } + with path.open(encoding="utf-8") as stream: + for line in stream: + record = json.loads(line) + language = record.get("language") + split = record.get("split") + if language in records and split in records[language]: + records[language][split].append(record) + return records + + +class IntentDataset(Dataset): + def __init__( + self, + records: list[dict], + tokenizer: AutoTokenizer, + max_length: int, + ) -> None: + encoded = tokenizer( + [record["text"] for record in records], + max_length=max_length, + padding="max_length", + truncation=True, + return_tensors="pt", + ) + self.inputs = dict(encoded) + self.labels = torch.tensor( + [ + [float(intent_value(record, intent)) for intent in INTENTS] + for record in records + ], + dtype=torch.float32, + ) + self.masks = torch.tensor( + [intent_mask(record) for record in records], + dtype=torch.float32, + ) + self.weights = torch.tensor( + [float(record.get("sampleWeight", 1.0)) for record in records], + dtype=torch.float32, + ) + + def __len__(self) -> int: + return self.labels.shape[0] + + def __getitem__(self, index: int) -> dict[str, torch.Tensor]: + item = {key: value[index] for key, value in self.inputs.items()} + item["labels"] = self.labels[index] + item["masks"] = self.masks[index] + item["weights"] = self.weights[index] + return item + + +@dataclass(frozen=True) +class Metrics: + true_positive: int + true_negative: int + false_positive: int + false_negative: int + precision: float + recall: float + f1: float + + def as_dict(self) -> dict: + return { + "truePositive": self.true_positive, + "trueNegative": self.true_negative, + "falsePositive": self.false_positive, + "falseNegative": self.false_negative, + "precision": round(self.precision, 6), + "recall": round(self.recall, 6), + "f1": round(self.f1, 6), + } + + +def calculate_metrics( + expected: np.ndarray, + predicted: np.ndarray, + mask: np.ndarray, +) -> Metrics: + expected = expected[mask.astype(bool)].astype(bool) + predicted = predicted[mask.astype(bool)].astype(bool) + true_positive = int(np.sum(expected & predicted)) + true_negative = int(np.sum(~expected & ~predicted)) + false_positive = int(np.sum(~expected & predicted)) + false_negative = int(np.sum(expected & ~predicted)) + precision_denominator = true_positive + false_positive + recall_denominator = true_positive + false_negative + precision = ( + true_positive / precision_denominator if precision_denominator else 0.0 + ) + recall = true_positive / recall_denominator if recall_denominator else 0.0 + f1 = ( + 2 * precision * recall / (precision + recall) + if precision + recall + else 0.0 + ) + return Metrics( + true_positive, + true_negative, + false_positive, + false_negative, + precision, + recall, + f1, + ) + + +def predict( + model: AutoModelForSequenceClassification, + tokenizer: AutoTokenizer, + records: list[dict], + device: torch.device, + max_length: int, + batch_size: int, +) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + model.eval() + probabilities: list[np.ndarray] = [] + expected: list[np.ndarray] = [] + masks: list[np.ndarray] = [] + with torch.inference_mode(): + for start in range(0, len(records), batch_size): + batch = records[start : start + batch_size] + encoded = tokenizer( + [record["text"] for record in batch], + max_length=max_length, + padding="max_length", + truncation=True, + return_tensors="pt", + ) + inputs = {key: value.to(device) for key, value in encoded.items()} + probabilities.append( + torch.sigmoid(model(**inputs).logits).cpu().numpy() + ) + expected.append( + np.array( + [ + [float(intent_value(record, intent)) for intent in INTENTS] + for record in batch + ], + dtype=np.float32, + ) + ) + masks.append( + np.array([intent_mask(record) for record in batch], dtype=np.float32) + ) + return ( + np.concatenate(probabilities), + np.concatenate(expected), + np.concatenate(masks), + ) + + +def choose_thresholds( + probabilities: np.ndarray, + expected: np.ndarray, + masks: np.ndarray, +) -> np.ndarray: + thresholds: list[float] = [] + for index in range(len(INTENTS)): + known = masks[:, index].astype(bool) + labels = expected[known, index] + if not np.any(labels == 1) or not np.any(labels == 0): + thresholds.append(0.5) + continue + candidates = [] + for threshold in np.linspace(0.05, 0.95, 91): + metrics = calculate_metrics( + expected[:, index], + probabilities[:, index] >= threshold, + masks[:, index], + ) + candidates.append((metrics.f1, metrics.precision, metrics.recall, threshold)) + thresholds.append(float(max(candidates)[3])) + return np.array(thresholds, dtype=np.float32) + + +def summarize( + probabilities: np.ndarray, + expected: np.ndarray, + masks: np.ndarray, + thresholds: np.ndarray, +) -> dict: + per_intent = {} + supported_metrics = [] + for index, intent in enumerate(INTENTS): + metrics = calculate_metrics( + expected[:, index], + probabilities[:, index] >= thresholds[index], + masks[:, index], + ) + known_count = int(np.sum(masks[:, index])) + positive_count = int(np.sum(expected[:, index] * masks[:, index])) + per_intent[intent] = { + "knownCount": known_count, + "positiveCount": positive_count, + **metrics.as_dict(), + } + if positive_count > 0: + supported_metrics.append(metrics) + return { + "records": int(expected.shape[0]), + "evaluatedIntentCount": len(supported_metrics), + "macroPrecision": round( + float(np.mean([metrics.precision for metrics in supported_metrics])), 6 + ), + "macroRecall": round( + float(np.mean([metrics.recall for metrics in supported_metrics])), 6 + ), + "macroF1": round( + float(np.mean([metrics.f1 for metrics in supported_metrics])), 6 + ), + "perIntent": per_intent, + } + + +def train_language( + language: str, + model_path: Path, + records: dict[str, list[dict]], + arguments: argparse.Namespace, + device: torch.device, +) -> dict: + set_seed(arguments.seed) + tokenizer = AutoTokenizer.from_pretrained(model_path, local_files_only=True) + model = AutoModelForSequenceClassification.from_pretrained( + model_path, + local_files_only=True, + num_labels=len(INTENTS), + problem_type="multi_label_classification", + ignore_mismatched_sizes=True, + ).to(device) + dataset = IntentDataset(records["train"], tokenizer, arguments.max_length) + generator = torch.Generator().manual_seed(arguments.seed) + loader = DataLoader( + dataset, + batch_size=arguments.batch_size, + shuffle=True, + generator=generator, + ) + weighted_positive = (dataset.labels * dataset.masks) * dataset.weights[:, None] + weighted_known = dataset.masks * dataset.weights[:, None] + positive_counts = weighted_positive.sum(dim=0) + negative_counts = weighted_known.sum(dim=0) - positive_counts + positive_weights = torch.clamp( + negative_counts / torch.clamp(positive_counts, min=1), + min=1, + max=20, + ).to(device) + criterion = torch.nn.BCEWithLogitsLoss( + pos_weight=positive_weights, + reduction="none", + ) + optimizer = torch.optim.AdamW( + model.parameters(), + lr=arguments.learning_rate, + weight_decay=0.01, + ) + + epoch_losses = [] + training_started = time.perf_counter() + for epoch in range(arguments.epochs): + model.train() + running_loss = 0.0 + for batch in loader: + labels = batch.pop("labels").to(device) + masks = batch.pop("masks").to(device) + weights = batch.pop("weights").to(device).unsqueeze(1) + inputs = {key: value.to(device) for key, value in batch.items()} + optimizer.zero_grad(set_to_none=True) + losses = criterion(model(**inputs).logits, labels) + weighted_masks = masks * weights + loss = (losses * weighted_masks).sum() / weighted_masks.sum().clamp(min=1) + loss.backward() + torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0) + optimizer.step() + running_loss += float(loss.detach().cpu()) + average_loss = running_loss / max(len(loader), 1) + epoch_losses.append(round(average_loss, 6)) + print( + f"TRAIN language={language} epoch={epoch + 1}/{arguments.epochs} " + f"loss={average_loss:.6f}", + flush=True, + ) + + predictions = {} + for split in ("validation", "test", "golden"): + predictions[split] = predict( + model, + tokenizer, + records[split], + device, + arguments.max_length, + arguments.batch_size, + ) + thresholds = choose_thresholds(*predictions["validation"]) + evaluations = { + split: summarize(*values, thresholds) + for split, values in predictions.items() + } + + sample_text = records["test"][0]["text"] + encoded = tokenizer( + sample_text, + max_length=arguments.max_length, + padding="max_length", + truncation=True, + return_tensors="pt", + ) + inputs = {key: value.to(device) for key, value in encoded.items()} + model.eval() + with torch.inference_mode(): + started = time.perf_counter() + model(**inputs) + if device.type == "mps": + torch.mps.synchronize() + cold_ms = (time.perf_counter() - started) * 1_000 + warm_samples = [] + for _ in range(100): + started = time.perf_counter() + model(**inputs) + if device.type == "mps": + torch.mps.synchronize() + warm_samples.append((time.perf_counter() - started) * 1_000) + + output = arguments.output_directory / LANGUAGE_MODELS[language] + output.mkdir(parents=True, exist_ok=True) + model.save_pretrained(output) + tokenizer.save_pretrained(output) + model_bytes = sum(path.stat().st_size for path in output.iterdir() if path.is_file()) + return { + "language": language, + "baseModel": str(model_path), + "trainRecords": len(records["train"]), + "epochLosses": epoch_losses, + "trainingSeconds": round(time.perf_counter() - training_started, 3), + "thresholds": { + intent: round(float(thresholds[index]), 4) + for index, intent in enumerate(INTENTS) + }, + "evaluations": evaluations, + "runtime": { + "engine": f"PyTorch eager on {device.type}", + "coldMilliseconds": round(cold_ms, 3), + "warmMeanMilliseconds": round(float(np.mean(warm_samples)), 3), + "warmP95Milliseconds": round(float(np.percentile(warm_samples, 95)), 3), + "processMaximumRSSBytes": int( + resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + ), + }, + "savedModelBytes": model_bytes, + } + + +def main() -> None: + arguments = parse_arguments() + set_seed(arguments.seed) + device = torch.device("mps" if torch.backends.mps.is_available() else "cpu") + records = load_records(arguments.corpus) + model_paths = { + "en": arguments.english_model, + "zh-Hans": arguments.chinese_model, + } + language_reports = [] + for language in ("zh-Hans", "en"): + counts = { + split: len(split_records) + for split, split_records in records[language].items() + } + print(f"DATA language={language} counts={counts}", flush=True) + language_reports.append( + train_language( + language, + model_paths[language], + records[language], + arguments, + device, + ) + ) + if device.type == "mps": + torch.mps.empty_cache() + + report = { + "schemaVersion": 1, + "purpose": "Taxonomy-v6 Tiny Transformer research challenger", + "corpus": str(arguments.corpus), + "seed": arguments.seed, + "intents": list(INTENTS), + "configuration": { + "epochs": arguments.epochs, + "batchSize": arguments.batch_size, + "maxLength": arguments.max_length, + "learningRate": arguments.learning_rate, + }, + "languages": language_reports, + "limitations": [ + "Thresholds use only the frozen validation split, which has 20 records per language.", + "PyTorch runtime is not directly comparable with Core ML runtime.", + "The Chinese UER checkpoint does not declare a model-weight license in its model card.", + ], + } + arguments.output_directory.mkdir(parents=True, exist_ok=True) + report_path = arguments.output_directory / "training-evaluation-report.json" + report_path.write_text( + json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + print(f"REPORT {report_path}", flush=True) + + +if __name__ == "__main__": + main()