3b54f996ca
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01RCcxND1mMWu6Lt2c2PxexN
103 lines
4.2 KiB
Python
103 lines
4.2 KiB
Python
"""QA harness — 2 % spot-check sample per extraction batch.
|
||
|
||
Mechanical checks run here; faithfulness (does the extraction match the ad?)
|
||
is judged by Claude reading the generated review file. Deviations from
|
||
QUALITY_BAR.md ("Extraction quality" section) are logged, not auto-fixed.
|
||
|
||
Usage:
|
||
python pipeline/qa_sample.py --extractions data/evidence/<slug>.jsonl \
|
||
--ads data/raw/jobs/<slug>_ads.json [--rate 0.02] [--seed <slug>]
|
||
|
||
Output:
|
||
docs/qa/<batch>-sample.md ad text + extraction side-by-side for review
|
||
docs/qa/<batch>-issues.log mechanical deviations (one line each)
|
||
|
||
Deterministic sampling (seeded by batch name) so re-runs pick the same
|
||
records and reviews stay comparable.
|
||
"""
|
||
import argparse
|
||
import json
|
||
import os
|
||
import random
|
||
import sys
|
||
|
||
sys.path.insert(0, os.path.dirname(__file__))
|
||
from extract_local import ARRAY_FIELDS, SENIORITY
|
||
|
||
BASE = os.path.join(os.path.dirname(__file__), "..")
|
||
QA_DIR = os.path.join(BASE, "docs", "qa")
|
||
|
||
|
||
def mechanical_issues(rec):
|
||
issues = []
|
||
hard = {x.lower() for x in rec.get("hard_skills", [])}
|
||
soft = {x.lower() for x in rec.get("soft_skills", [])}
|
||
tools = {x.lower() for x in rec.get("tools", [])}
|
||
if hard & soft:
|
||
issues.append(f"cross-field dupe hard/soft: {sorted(hard & soft)}")
|
||
if hard & tools:
|
||
issues.append(f"cross-field dupe hard/tools: {sorted(hard & tools)}")
|
||
if rec.get("seniority") not in SENIORITY:
|
||
issues.append(f"invalid seniority {rec.get('seniority')!r}")
|
||
for field, (max_items, max_len) in ARRAY_FIELDS.items():
|
||
vals = rec.get(field, [])
|
||
if len(vals) > max_items:
|
||
issues.append(f"{field}: {len(vals)} items > {max_items}")
|
||
for v in vals:
|
||
if "<EFBFBD>" in v or len(v) > max_len:
|
||
issues.append(f"{field}: bad item {v!r}")
|
||
# lowercase rule: only flag all-caps sentences, products are fine
|
||
if v.isupper() and len(v) > 6:
|
||
issues.append(f"{field}: shouting {v!r}")
|
||
return issues
|
||
|
||
|
||
def main():
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--extractions", required=True)
|
||
ap.add_argument("--ads", required=True)
|
||
ap.add_argument("--rate", type=float, default=0.02)
|
||
ap.add_argument("--seed", default=None, help="defaults to extractions filename")
|
||
args = ap.parse_args()
|
||
|
||
batch = os.path.splitext(os.path.basename(args.extractions))[0]
|
||
rng = random.Random(args.seed or batch)
|
||
|
||
recs = [json.loads(l) for l in open(args.extractions, encoding="utf-8") if l.strip()]
|
||
ads = {a["job_id"]: a for a in json.load(open(args.ads, encoding="utf-8"))}
|
||
n = max(1, round(len(recs) * args.rate))
|
||
sample = rng.sample(recs, min(n, len(recs)))
|
||
|
||
os.makedirs(QA_DIR, exist_ok=True)
|
||
issues_total = 0
|
||
review = [f"# QA sample — {batch}", "",
|
||
f"{len(sample)} of {len(recs)} records ({args.rate:.0%}). "
|
||
"Judge each extraction against QUALITY_BAR.md 'Extraction quality': "
|
||
"faithful, no invented items, plausible seniority.", ""]
|
||
with open(os.path.join(QA_DIR, f"{batch}-issues.log"), "w", encoding="utf-8") as log:
|
||
for rec in sample:
|
||
ad = ads.get(rec["job_id"], {})
|
||
probs = mechanical_issues(rec)
|
||
issues_total += len(probs)
|
||
for p in probs:
|
||
log.write(f"{rec['job_id']}\t{p}\n")
|
||
review += [f"## {ad.get('title', rec['job_id'])} ({ad.get('country','?')})", "",
|
||
"**Ad (first 2000 chars):**", "",
|
||
"> " + (ad.get("description", "AD TEXT MISSING")[:2000]
|
||
).replace("\n", "\n> "), "",
|
||
"**Extraction:**", "", "```json",
|
||
json.dumps({k: v for k, v in rec.items() if k != "job_id"},
|
||
ensure_ascii=False, indent=1),
|
||
"```", ""]
|
||
if probs:
|
||
review += ["**Mechanical issues:** " + "; ".join(probs), ""]
|
||
out = os.path.join(QA_DIR, f"{batch}-sample.md")
|
||
open(out, "w", encoding="utf-8").write("\n".join(review))
|
||
print(f"{len(sample)} sampled, {issues_total} mechanical issues -> {out}")
|
||
if issues_total > len(sample): # more than 1 issue per record on average
|
||
sys.exit(1)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|