86 lines
3.2 KiB
Python
86 lines
3.2 KiB
Python
"""Phase 3b — store per-ad skill extractions in the MSSQL evidence store.
|
|
|
|
The extraction itself is done by Claude Code reading the raw ads batch-wise
|
|
(data/raw/jobs/recruiter_ads.json) and writing data/evidence/extractions.json:
|
|
|
|
[
|
|
{"job_id": "...", "seniority": "senior|mid|junior|n/a",
|
|
"hard_skills": ["candidate sourcing", ...],
|
|
"tools": ["Applicant tracking system (ATS)", "LinkedIn Recruiter", ...],
|
|
"methods": ["structured interviews", ...],
|
|
"responsibilities": ["full-cycle recruiting", ...]}
|
|
]
|
|
|
|
This script normalizes entities (lowercase/trim) and loads two tables:
|
|
- evidence_job(job_id, title, employer, country, seniority)
|
|
- evidence_entity(job_id, entity_type, entity)
|
|
Idempotent: tables are rebuilt on each run.
|
|
"""
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
sys.path.insert(0, os.path.dirname(__file__))
|
|
from db import connect
|
|
|
|
BASE = os.path.join(os.path.dirname(__file__), "..")
|
|
ADS = os.path.join(BASE, "data", "raw", "jobs", "recruiter_ads.json")
|
|
EXTRACTIONS = os.path.join(BASE, "data", "evidence", "extractions.json")
|
|
|
|
|
|
# Produktfamilien zusammenfassen, damit die Aggregation nicht an
|
|
# Namensvarianten zersplittert (Prozente zählen DISTINCT jobs).
|
|
CANONICAL = {
|
|
"linkedin": "LinkedIn / LinkedIn Recruiter",
|
|
"linkedin recruiter": "LinkedIn / LinkedIn Recruiter",
|
|
"ms office": "Microsoft Office",
|
|
"microsoft office suite": "Microsoft Office",
|
|
}
|
|
|
|
|
|
def norm(s: str) -> str:
|
|
v = " ".join(str(s).strip().split())
|
|
return CANONICAL.get(v.lower(), v)
|
|
|
|
|
|
def main():
|
|
if not (os.path.exists(ADS) and os.path.exists(EXTRACTIONS)):
|
|
print("SKIP: ads or extractions missing "
|
|
"(needs JSearch subscription / extraction batch) — TODO in README")
|
|
return
|
|
ads = {a["job_id"]: a for a in json.load(open(ADS, encoding="utf-8"))}
|
|
ext = json.load(open(EXTRACTIONS, encoding="utf-8"))
|
|
|
|
cn = connect()
|
|
cur = cn.cursor()
|
|
cur.execute("IF OBJECT_ID('evidence_entity','U') IS NOT NULL DROP TABLE evidence_entity")
|
|
cur.execute("IF OBJECT_ID('evidence_job','U') IS NOT NULL DROP TABLE evidence_job")
|
|
cur.execute("""CREATE TABLE evidence_job (
|
|
job_id NVARCHAR(100) NOT NULL PRIMARY KEY,
|
|
title NVARCHAR(400), employer NVARCHAR(400),
|
|
country NVARCHAR(5), seniority NVARCHAR(20))""")
|
|
cur.execute("""CREATE TABLE evidence_entity (
|
|
job_id NVARCHAR(100) NOT NULL,
|
|
entity_type NVARCHAR(30) NOT NULL,
|
|
entity NVARCHAR(300) NOT NULL)""")
|
|
cur.execute("CREATE INDEX IX_evidence_entity ON evidence_entity(entity_type, entity)")
|
|
|
|
n_ent = 0
|
|
for e in ext:
|
|
ad = ads.get(e["job_id"], {})
|
|
cur.execute("INSERT INTO evidence_job VALUES (?,?,?,?,?)",
|
|
e["job_id"], ad.get("title"), ad.get("employer"),
|
|
ad.get("country"), e.get("seniority"))
|
|
for etype in ("hard_skills", "tools", "methods", "responsibilities"):
|
|
for entity in e.get(etype, []):
|
|
cur.execute("INSERT INTO evidence_entity VALUES (?,?,?)",
|
|
e["job_id"], etype, norm(entity))
|
|
n_ent += 1
|
|
cn.commit()
|
|
cn.close()
|
|
print(f"evidence: {len(ext)} jobs, {n_ent} entities loaded")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|