chore(semantics): add v6 release gate pipeline
- Add reproducible v6 boundary, blessing, and consensus-adjudication corpora, plus the tiny-transformer trainer and v6 release-gate evaluator that gate every candidate on the deployed baselines. - Wire consensus-label merging, product-policy anchor evaluation, and sealed blessing benchmark review with their pytest coverage. - Refresh open-training corpus generation, iterative retraining runner, and random-holdout evaluation so v6 candidates can be benchmarked end-to-end.
This commit is contained in:
Rocky
@@ -0,0 +1,607 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Merge blind three-state labels from three primary and two review models."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
INTENT_LABELS = (
|
||||
"task",
|
||||
"question",
|
||||
"invitation",
|
||||
"complaint",
|
||||
"scheduleNegotiation",
|
||||
"confirmationDecision",
|
||||
"followUpReminder",
|
||||
"blessing",
|
||||
"replyableMessage",
|
||||
"assistantCommand",
|
||||
"informationQuery",
|
||||
"systemNotification",
|
||||
)
|
||||
DOMAINS = {
|
||||
"finance",
|
||||
"travel",
|
||||
"calendar",
|
||||
"communication",
|
||||
"media",
|
||||
"smartHome",
|
||||
"shopping",
|
||||
"dining",
|
||||
"health",
|
||||
"weather",
|
||||
"accountService",
|
||||
"generalKnowledge",
|
||||
}
|
||||
DOMAIN_STATES = {*DOMAINS, "unknown"}
|
||||
LABEL_STATES = {"true", "false", "unknown"}
|
||||
SENTIMENT_STATES = {"positive", "neutral", "negative", "unknown"}
|
||||
PROMPT_VERSION = "clipboard-consensus-v6"
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def read_json_lines(path: Path) -> list[dict]:
|
||||
return [
|
||||
json.loads(line)
|
||||
for line in path.read_text(encoding="utf-8").splitlines()
|
||||
if line.strip()
|
||||
]
|
||||
|
||||
|
||||
def write_json_lines(path: Path, records: list[dict]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", encoding="utf-8") as handle:
|
||||
for record in records:
|
||||
handle.write(
|
||||
json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n"
|
||||
)
|
||||
|
||||
|
||||
def parse_labeler(value: str) -> tuple[str, Path]:
|
||||
name, separator, raw_path = value.partition("=")
|
||||
if not separator or not name or not raw_path:
|
||||
raise argparse.ArgumentTypeError("Expected LABELER=PATH")
|
||||
return name, Path(raw_path)
|
||||
|
||||
|
||||
def validate_record(record: dict, expected_ids: set[str]) -> dict:
|
||||
identifier = record.get("id")
|
||||
if identifier not in expected_ids:
|
||||
raise ValueError(f"Unexpected labeler record id: {identifier}")
|
||||
labels = record.get("labels")
|
||||
if not isinstance(labels, dict) or set(labels) != set(INTENT_LABELS):
|
||||
raise ValueError(f"Every intent label is required for {identifier}")
|
||||
if any(value not in LABEL_STATES for value in labels.values()):
|
||||
raise ValueError(f"Invalid three-state label for {identifier}")
|
||||
sentiment = record.get("sentiment")
|
||||
if sentiment not in SENTIMENT_STATES:
|
||||
raise ValueError(f"Invalid sentiment for {identifier}: {sentiment}")
|
||||
domain = record.get("domain")
|
||||
if domain not in DOMAIN_STATES:
|
||||
raise ValueError(f"Invalid domain for {identifier}: {domain}")
|
||||
if not isinstance(record.get("ambiguous"), bool):
|
||||
raise TypeError(f"Missing ambiguous flag for {identifier}")
|
||||
if not isinstance(record.get("quotedOrMeta"), bool):
|
||||
raise TypeError(f"Missing quotedOrMeta flag for {identifier}")
|
||||
confidence = record.get("confidence")
|
||||
if not isinstance(confidence, (int, float)) or not 0 <= confidence <= 1:
|
||||
raise ValueError(f"Invalid confidence for {identifier}: {confidence}")
|
||||
return {
|
||||
"id": identifier,
|
||||
"labels": {label: labels[label] for label in INTENT_LABELS},
|
||||
"sentiment": sentiment,
|
||||
"domain": domain,
|
||||
"ambiguous": record["ambiguous"],
|
||||
"quotedOrMeta": record["quotedOrMeta"],
|
||||
"confidence": round(float(confidence), 4),
|
||||
}
|
||||
|
||||
|
||||
def load_labelers(
|
||||
values: list[tuple[str, Path]],
|
||||
expected_ids: set[str],
|
||||
require_exact_ids: bool,
|
||||
) -> list[tuple[str, dict[str, dict]]]:
|
||||
labelers = []
|
||||
for name, path in values:
|
||||
records = [
|
||||
validate_record(record, expected_ids)
|
||||
for record in read_json_lines(path)
|
||||
]
|
||||
by_id = {record["id"]: record for record in records}
|
||||
if len(by_id) != len(records):
|
||||
raise ValueError(f"Labeler {name} contains duplicate ids")
|
||||
if require_exact_ids and set(by_id) != expected_ids:
|
||||
missing = expected_ids - set(by_id)
|
||||
extra = set(by_id) - expected_ids
|
||||
raise ValueError(
|
||||
f"Labeler {name} id mismatch: missing={len(missing)} "
|
||||
f"extra={len(extra)}"
|
||||
)
|
||||
labelers.append((name, by_id))
|
||||
return labelers
|
||||
|
||||
|
||||
def field_values(record: dict) -> dict[str, str]:
|
||||
return {
|
||||
**record["labels"],
|
||||
"sentiment": record["sentiment"],
|
||||
"domain": record["domain"],
|
||||
}
|
||||
|
||||
|
||||
def primary_review_ids(
|
||||
primary: list[tuple[str, dict[str, dict]]],
|
||||
queue_ids: set[str],
|
||||
) -> set[str]:
|
||||
review_ids = set()
|
||||
for identifier in queue_ids:
|
||||
records = [values[identifier] for _, values in primary]
|
||||
fields = [field_values(record) for record in records]
|
||||
unanimous = all(
|
||||
len({field[name] for field in fields}) == 1
|
||||
for name in (*INTENT_LABELS, "sentiment", "domain")
|
||||
)
|
||||
flags_clear = not any(
|
||||
record["ambiguous"] or record["quotedOrMeta"] for record in records
|
||||
)
|
||||
if not unanimous or not flags_clear:
|
||||
review_ids.add(identifier)
|
||||
return review_ids
|
||||
|
||||
|
||||
def prepare_review(arguments: argparse.Namespace) -> dict:
|
||||
queue = read_json_lines(arguments.queue)
|
||||
queue_by_id = {record["id"]: record for record in queue}
|
||||
if len(queue_by_id) != len(queue):
|
||||
raise ValueError("Queue contains duplicate ids")
|
||||
if len(arguments.primary) != 3:
|
||||
raise ValueError("Exactly three primary labelers are required")
|
||||
primary = load_labelers(arguments.primary, set(queue_by_id), True)
|
||||
review_ids = primary_review_ids(primary, set(queue_by_id))
|
||||
review_queue = [
|
||||
{
|
||||
"id": record["id"],
|
||||
"text": record["text"],
|
||||
"language": record["language"],
|
||||
}
|
||||
for record in queue
|
||||
if record["id"] in review_ids
|
||||
]
|
||||
write_json_lines(arguments.output, review_queue)
|
||||
report = {
|
||||
"schemaVersion": 2,
|
||||
"promptVersion": PROMPT_VERSION,
|
||||
"queueCount": len(queue),
|
||||
"queueSHA256": sha256_file(arguments.queue),
|
||||
"primaryUnanimousCount": len(queue) - len(review_queue),
|
||||
"reviewCount": len(review_queue),
|
||||
"primaryLabelers": [name for name, _ in primary],
|
||||
"primaryOutputSHA256": {
|
||||
name: sha256_file(path) for name, path in arguments.primary
|
||||
},
|
||||
}
|
||||
arguments.report.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return report
|
||||
|
||||
|
||||
def split_queue(arguments: argparse.Namespace) -> dict:
|
||||
queue = read_json_lines(arguments.queue)
|
||||
if arguments.chunk_size <= 0:
|
||||
raise ValueError("chunk-size must be positive")
|
||||
arguments.output_directory.mkdir(parents=True, exist_ok=True)
|
||||
output_paths = []
|
||||
for start in range(0, len(queue), arguments.chunk_size):
|
||||
index = len(output_paths) + 1
|
||||
path = arguments.output_directory / f"chunk-{index:03d}.jsonl"
|
||||
write_json_lines(path, queue[start : start + arguments.chunk_size])
|
||||
output_paths.append(path)
|
||||
report = {
|
||||
"schemaVersion": 2,
|
||||
"queueCount": len(queue),
|
||||
"queueSHA256": sha256_file(arguments.queue),
|
||||
"chunkSize": arguments.chunk_size,
|
||||
"chunkCount": len(output_paths),
|
||||
"chunks": [
|
||||
{
|
||||
"path": str(path),
|
||||
"records": len(read_json_lines(path)),
|
||||
"sha256": sha256_file(path),
|
||||
}
|
||||
for path in output_paths
|
||||
],
|
||||
}
|
||||
arguments.report.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return report
|
||||
|
||||
|
||||
def combine_labeler(arguments: argparse.Namespace) -> dict:
|
||||
queue = read_json_lines(arguments.queue)
|
||||
queue_ids = [record["id"] for record in queue]
|
||||
expected_ids = set(queue_ids)
|
||||
combined = []
|
||||
for path in arguments.input:
|
||||
combined.extend(read_json_lines(path))
|
||||
validated = [
|
||||
validate_record(record, expected_ids) for record in combined
|
||||
]
|
||||
by_id = {record["id"]: record for record in validated}
|
||||
if len(by_id) != len(validated):
|
||||
raise ValueError("Combined labeler outputs contain duplicate ids")
|
||||
if set(by_id) != expected_ids:
|
||||
raise ValueError(
|
||||
"Combined labeler output ids do not match the source queue"
|
||||
)
|
||||
ordered = [by_id[identifier] for identifier in queue_ids]
|
||||
write_json_lines(arguments.output, ordered)
|
||||
report = {
|
||||
"schemaVersion": 2,
|
||||
"queueCount": len(queue),
|
||||
"queueSHA256": sha256_file(arguments.queue),
|
||||
"inputCount": len(arguments.input),
|
||||
"outputCount": len(ordered),
|
||||
"outputSHA256": sha256_file(arguments.output),
|
||||
}
|
||||
arguments.report.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return report
|
||||
|
||||
|
||||
def stable_split(identifier: str) -> str:
|
||||
bucket = int.from_bytes(hashlib.sha256(identifier.encode()).digest()[:8], "big")
|
||||
bucket %= 100
|
||||
if bucket < 80:
|
||||
return "silverTrain"
|
||||
if bucket < 90:
|
||||
return "silverCalibration"
|
||||
return "silverAcceptance"
|
||||
|
||||
|
||||
def field_consensus(
|
||||
records: list[dict],
|
||||
field: str,
|
||||
required_votes: int,
|
||||
) -> tuple[str, dict[str, int]]:
|
||||
values = [
|
||||
field_values(record)[field]
|
||||
for record in records
|
||||
if field_values(record)[field] != "unknown"
|
||||
]
|
||||
votes = Counter(values)
|
||||
if not votes:
|
||||
return "unknown", {}
|
||||
value, count = votes.most_common(1)[0]
|
||||
if count < required_votes:
|
||||
return "unknown", dict(sorted(votes.items()))
|
||||
return value, dict(sorted(votes.items()))
|
||||
|
||||
|
||||
def fleiss_kappa_for_field(
|
||||
labelers: list[tuple[str, dict[str, dict]]],
|
||||
identifiers: list[str],
|
||||
field: str,
|
||||
) -> float:
|
||||
categories = (
|
||||
sorted(SENTIMENT_STATES)
|
||||
if field == "sentiment"
|
||||
else sorted(DOMAIN_STATES)
|
||||
if field == "domain"
|
||||
else sorted(LABEL_STATES)
|
||||
)
|
||||
category_totals = Counter()
|
||||
item_agreements = []
|
||||
rater_count = len(labelers)
|
||||
for identifier in identifiers:
|
||||
votes = Counter(
|
||||
field_values(records[identifier])[field]
|
||||
for _, records in labelers
|
||||
)
|
||||
category_totals.update(votes)
|
||||
item_agreements.append(
|
||||
sum(count * (count - 1) for count in votes.values())
|
||||
/ (rater_count * (rater_count - 1))
|
||||
)
|
||||
if not item_agreements:
|
||||
return 0
|
||||
observed = sum(item_agreements) / len(item_agreements)
|
||||
total = sum(category_totals.values())
|
||||
expected = sum(
|
||||
(category_totals[category] / total) ** 2 for category in categories
|
||||
)
|
||||
if math.isclose(expected, 1):
|
||||
return 1
|
||||
return round((observed - expected) / (1 - expected), 4)
|
||||
|
||||
|
||||
def training_record(
|
||||
queue_record: dict,
|
||||
states: dict[str, str],
|
||||
tier: str,
|
||||
votes: dict,
|
||||
labelers: list[tuple[str, dict[str, dict]]],
|
||||
) -> dict:
|
||||
split = stable_split(queue_record["id"])
|
||||
known_labels = [
|
||||
label
|
||||
for label in (*INTENT_LABELS, "sentiment", "domain")
|
||||
if states[label] != "unknown"
|
||||
]
|
||||
return {
|
||||
"id": f"consensus-v2-{queue_record['id']}",
|
||||
"sourceRecordID": queue_record["id"],
|
||||
"text": queue_record["text"],
|
||||
"language": queue_record["language"],
|
||||
"family": "consensus_v2",
|
||||
"split": split,
|
||||
**{
|
||||
("replyable" if label == "replyableMessage" else label): (
|
||||
states[label] == "true"
|
||||
)
|
||||
for label in INTENT_LABELS
|
||||
},
|
||||
"sentiment": (
|
||||
states["sentiment"]
|
||||
if states["sentiment"] != "unknown"
|
||||
else "neutral"
|
||||
),
|
||||
"domain": (
|
||||
states["domain"] if states["domain"] != "unknown" else None
|
||||
),
|
||||
"knownLabels": known_labels,
|
||||
"labelQualityTier": tier,
|
||||
"sampleWeight": 1.0 if tier == "A" else 0.65,
|
||||
"promptVersion": PROMPT_VERSION,
|
||||
"modelVotes": votes,
|
||||
"labelers": [name for name, _ in labelers],
|
||||
}
|
||||
|
||||
|
||||
def merge(arguments: argparse.Namespace) -> dict:
|
||||
queue = read_json_lines(arguments.queue)
|
||||
queue_by_id = {record["id"]: record for record in queue}
|
||||
queue_ids = set(queue_by_id)
|
||||
if len(queue_by_id) != len(queue):
|
||||
raise ValueError("Queue contains duplicate ids")
|
||||
if len(arguments.primary) != 3 or len(arguments.reviewer) != 2:
|
||||
raise ValueError("Three primary and two review labelers are required")
|
||||
primary = load_labelers(arguments.primary, queue_ids, True)
|
||||
review_ids = primary_review_ids(primary, queue_ids)
|
||||
reviewers = load_labelers(arguments.reviewer, review_ids, True)
|
||||
tier_a = []
|
||||
tier_b = []
|
||||
human_review = []
|
||||
all_labelers = primary + reviewers
|
||||
field_names = (*INTENT_LABELS, "sentiment", "domain")
|
||||
for identifier in sorted(queue_ids):
|
||||
queue_record = queue_by_id[identifier]
|
||||
primary_records = [records[identifier] for _, records in primary]
|
||||
if identifier not in review_ids:
|
||||
states = {
|
||||
field: field_values(primary_records[0])[field]
|
||||
for field in field_names
|
||||
}
|
||||
votes = {
|
||||
field: {states[field]: 3}
|
||||
for field in field_names
|
||||
}
|
||||
tier_a.append(
|
||||
training_record(
|
||||
queue_record,
|
||||
states,
|
||||
"A",
|
||||
votes,
|
||||
primary,
|
||||
)
|
||||
)
|
||||
continue
|
||||
combined_records = primary_records + [
|
||||
records[identifier] for _, records in reviewers
|
||||
]
|
||||
states = {}
|
||||
votes = {}
|
||||
for field in field_names:
|
||||
states[field], votes[field] = field_consensus(
|
||||
combined_records,
|
||||
field,
|
||||
4,
|
||||
)
|
||||
ambiguous_votes = sum(
|
||||
record["ambiguous"] for record in combined_records
|
||||
)
|
||||
quoted_votes = sum(
|
||||
record["quotedOrMeta"] for record in combined_records
|
||||
)
|
||||
unresolved = [
|
||||
field
|
||||
for field, value in states.items()
|
||||
if value == "unknown" and votes[field]
|
||||
]
|
||||
positive_intents = any(states[label] == "true" for label in INTENT_LABELS)
|
||||
if ambiguous_votes >= 2:
|
||||
unresolved.append("ambiguous")
|
||||
if quoted_votes >= 2 and positive_intents:
|
||||
unresolved.append("quotedOrMeta")
|
||||
if unresolved:
|
||||
human_review.append(
|
||||
{
|
||||
"id": identifier,
|
||||
"text": queue_record["text"],
|
||||
"language": queue_record["language"],
|
||||
"unresolvedFields": sorted(set(unresolved)),
|
||||
"modelVotes": votes,
|
||||
"ambiguousVotes": ambiguous_votes,
|
||||
"quotedOrMetaVotes": quoted_votes,
|
||||
"resolution": None,
|
||||
"reviewer": None,
|
||||
"reason": None,
|
||||
}
|
||||
)
|
||||
continue
|
||||
tier_b.append(
|
||||
training_record(
|
||||
queue_record,
|
||||
states,
|
||||
"B",
|
||||
votes,
|
||||
all_labelers,
|
||||
)
|
||||
)
|
||||
accepted = tier_a + tier_b
|
||||
write_json_lines(arguments.tier_a, tier_a)
|
||||
write_json_lines(arguments.tier_b, tier_b)
|
||||
write_json_lines(arguments.accepted, accepted)
|
||||
write_json_lines(arguments.human_review, human_review)
|
||||
identifiers = sorted(queue_ids)
|
||||
kappa_by_field = {
|
||||
field: fleiss_kappa_for_field(primary, identifiers, field)
|
||||
for field in field_names
|
||||
}
|
||||
languages = sorted({record["language"] for record in queue})
|
||||
kappa_by_language = {
|
||||
language: {
|
||||
field: fleiss_kappa_for_field(
|
||||
primary,
|
||||
sorted(
|
||||
identifier
|
||||
for identifier, record in queue_by_id.items()
|
||||
if record["language"] == language
|
||||
),
|
||||
field,
|
||||
)
|
||||
for field in field_names
|
||||
}
|
||||
for language in languages
|
||||
}
|
||||
kappa_gate_passed = all(
|
||||
value >= 0.8
|
||||
for values in kappa_by_language.values()
|
||||
for value in values.values()
|
||||
)
|
||||
report = {
|
||||
"schemaVersion": 2,
|
||||
"promptVersion": PROMPT_VERSION,
|
||||
"queueCount": len(queue),
|
||||
"queueSHA256": sha256_file(arguments.queue),
|
||||
"tierACount": len(tier_a),
|
||||
"tierBCount": len(tier_b),
|
||||
"humanReviewCount": len(human_review),
|
||||
"acceptanceRate": round(len(accepted) / max(len(queue), 1), 4),
|
||||
"primaryLabelers": [name for name, _ in primary],
|
||||
"reviewLabelers": [name for name, _ in reviewers],
|
||||
"labelerOutputSHA256": {
|
||||
name: sha256_file(path)
|
||||
for name, path in (*arguments.primary, *arguments.reviewer)
|
||||
},
|
||||
"primaryFleissKappaByField": kappa_by_field,
|
||||
"primaryFleissKappaByLanguageAndField": kappa_by_language,
|
||||
"scaleUpKappaThreshold": 0.8,
|
||||
"eligibleForScaleUp": kappa_gate_passed,
|
||||
"acceptedIntentPositiveCounts": {
|
||||
label: sum(
|
||||
record["replyable" if label == "replyableMessage" else label]
|
||||
for record in accepted
|
||||
)
|
||||
for label in INTENT_LABELS
|
||||
},
|
||||
"acceptedLanguageCounts": dict(
|
||||
sorted(Counter(record["language"] for record in accepted).items())
|
||||
),
|
||||
"splitCounts": dict(
|
||||
sorted(Counter(record["split"] for record in accepted).items())
|
||||
),
|
||||
}
|
||||
arguments.report.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return report
|
||||
|
||||
|
||||
def parser() -> argparse.ArgumentParser:
|
||||
root = argparse.ArgumentParser()
|
||||
commands = root.add_subparsers(dest="command", required=True)
|
||||
review = commands.add_parser("prepare-review")
|
||||
review.add_argument("--queue", type=Path, required=True)
|
||||
review.add_argument(
|
||||
"--primary",
|
||||
action="append",
|
||||
type=parse_labeler,
|
||||
required=True,
|
||||
)
|
||||
review.add_argument("--output", type=Path, required=True)
|
||||
review.add_argument("--report", type=Path, required=True)
|
||||
review.set_defaults(handler=prepare_review)
|
||||
|
||||
split = commands.add_parser("split-queue")
|
||||
split.add_argument("--queue", type=Path, required=True)
|
||||
split.add_argument("--output-directory", type=Path, required=True)
|
||||
split.add_argument("--chunk-size", type=int, default=100)
|
||||
split.add_argument("--report", type=Path, required=True)
|
||||
split.set_defaults(handler=split_queue)
|
||||
|
||||
combine = commands.add_parser("combine-labeler")
|
||||
combine.add_argument("--queue", type=Path, required=True)
|
||||
combine.add_argument(
|
||||
"--input",
|
||||
action="append",
|
||||
type=Path,
|
||||
required=True,
|
||||
)
|
||||
combine.add_argument("--output", type=Path, required=True)
|
||||
combine.add_argument("--report", type=Path, required=True)
|
||||
combine.set_defaults(handler=combine_labeler)
|
||||
|
||||
merge_parser = commands.add_parser("merge")
|
||||
merge_parser.add_argument("--queue", type=Path, required=True)
|
||||
merge_parser.add_argument(
|
||||
"--primary",
|
||||
action="append",
|
||||
type=parse_labeler,
|
||||
required=True,
|
||||
)
|
||||
merge_parser.add_argument(
|
||||
"--reviewer",
|
||||
action="append",
|
||||
type=parse_labeler,
|
||||
required=True,
|
||||
)
|
||||
merge_parser.add_argument("--tier-a", type=Path, required=True)
|
||||
merge_parser.add_argument("--tier-b", type=Path, required=True)
|
||||
merge_parser.add_argument("--accepted", type=Path, required=True)
|
||||
merge_parser.add_argument("--human-review", type=Path, required=True)
|
||||
merge_parser.add_argument("--report", type=Path, required=True)
|
||||
merge_parser.set_defaults(handler=merge)
|
||||
return root
|
||||
|
||||
|
||||
def main() -> None:
|
||||
arguments = parser().parse_args()
|
||||
report = arguments.handler(arguments)
|
||||
print(
|
||||
f"CONSENSUS_V2_{arguments.command.upper().replace('-', '_')} "
|
||||
+ " ".join(f"{key}={value}" for key, value in report.items() if key.endswith("Count"))
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user