diff --git a/overlays/artificial-intelligence-engineer/intro.md b/overlays/artificial-intelligence-engineer/intro.md new file mode 100644 index 00000000..05ebb8db --- /dev/null +++ b/overlays/artificial-intelligence-engineer/intro.md @@ -0,0 +1,7 @@ +Artificial intelligence engineers apply AI methods from engineering, robotics +and computer science to design programs that simulate intelligence — including +reasoning models, cognitive and knowledge-based systems, problem solving and +decision making. They also integrate structured knowledge into computer +systems (ontologies, knowledge bases) to solve complex problems that normally +require a high level of human expertise, and they carry models from prototype +to monitored production systems. diff --git a/overlays/artificial-intelligence-engineer/workflow.md b/overlays/artificial-intelligence-engineer/workflow.md new file mode 100644 index 00000000..933b5a54 --- /dev/null +++ b/overlays/artificial-intelligence-engineer/workflow.md @@ -0,0 +1,16 @@ +1. Frame the problem: target metric, baseline, cost of errors, and whether + AI is warranted at all (see references/intake.md — mandatory questions). +2. Assess and prepare the data: sources, quality, labels, legal basis, + leakage risks; build the evaluation set before the model. +3. Choose the solution tier: rules/heuristic baseline → prompting an + existing model → RAG over private data → fine-tuning → custom training. +4. Build the pipeline: versioned data, features/prompts, training or + orchestration code, reproducible experiments. +5. Evaluate offline against the baseline — overall and per segment; for + generative systems run the golden-example eval suite. +6. Ship safely: shadow mode → canary → ramp-up, with rollback and + guardrails (input validation, output schemas, human fallback). +7. Monitor in production: drift, quality proxies, cost, latency; alert + thresholds with owners. +8. Iterate and govern: retraining triggers, model cards, bias reviews, + compliance (e.g. EU AI Act risk class). diff --git a/pipeline/p2_generate.py b/pipeline/p2_generate.py index ff486fdc..76e201ad 100644 --- a/pipeline/p2_generate.py +++ b/pipeline/p2_generate.py @@ -19,6 +19,70 @@ sys.path.insert(0, os.path.dirname(__file__)) from db import connect SKILLS_DIR = os.path.join(os.path.dirname(__file__), "..", "skills") +OVERLAYS_DIR = os.path.join(os.path.dirname(__file__), "..", "overlays") +REWRITE_CACHE = os.path.join(os.path.dirname(__file__), "..", "data", "rewrites") + + +def clean_intro(slug, raw_text): + """Phase 2d: one grammatically clean English paragraph instead of raw + ESCO text. gemma3-rewritten once, cached in data/rewrites/.intro.txt + (re-runs cost nothing); falls back to the raw text when Ollama is + unreachable. The raw ESCO original stays reachable via the ESCO URI in + the sources footer.""" + if not raw_text: + return raw_text + os.makedirs(REWRITE_CACHE, exist_ok=True) + cache = os.path.join(REWRITE_CACHE, f"{slug}.intro.txt") + if os.path.exists(cache): + cached = open(cache, encoding="utf-8").read().strip() + if cached: + return cached + try: + import requests + from dotenv import load_dotenv + load_dotenv(os.path.join(os.path.dirname(__file__), "..", ".env")) + url = os.environ.get("OLLAMA_URL", "").rstrip("/") + model = os.environ.get("OLLAMA_MODEL", "") + if not url or not model: + return raw_text + prompt = ( + "Rewrite the following occupation description as ONE grammatically " + "correct, clear English paragraph. Fix subject-verb agreement and " + "awkward phrasing. Preserve every fact; add nothing; remove " + "nothing substantive. Output only the paragraph.\n\n" + raw_text[:1500]) + r = requests.post(f"{url}/api/generate", json={ + "model": model, "prompt": prompt, "stream": False, + "options": {"temperature": 0, "num_ctx": 2048, "num_predict": 400}, + }, timeout=120) + r.raise_for_status() + text = (r.json().get("response") or "").strip() + # sanity: rewrite must stay a single plausible paragraph + if 80 <= len(text) <= max(400, int(len(raw_text) * 1.6)) and "\n\n" not in text: + open(cache, "w", encoding="utf-8", newline="\n").write(text + "\n") + return text + except Exception: + pass + return raw_text + + +def apply_overlays(slug, text): + """Reviewed manual curation that must SURVIVE regeneration (phase 2d). + overlays//intro.md -> replaces the intro paragraph + overlays//workflow.md -> replaces the '## Core workflow' body""" + odir = os.path.join(OVERLAYS_DIR, slug) + if not os.path.isdir(odir): + return text + intro_p = os.path.join(odir, "intro.md") + if os.path.isfile(intro_p): + intro = open(intro_p, encoding="utf-8").read().strip() + text = re.sub(r"(?ms)(^---\n.*?\n---\n\n# [^\n]+\n\n).*?(?=\n\n## )", + lambda m: m.group(1) + intro, text, count=1) + wf_p = os.path.join(odir, "workflow.md") + if os.path.isfile(wf_p): + wf = open(wf_p, encoding="utf-8").read().strip() + text = re.sub(r"(?ms)(^## Core workflow\n\n).*?(?=\n\n## )", + lambda m: m.group(1) + wf, text, count=1) + return text TARGET_COUNT = 100 # ISCO-08 prefixes for knowledge work (management, business, IT, HR, finance, @@ -199,7 +263,6 @@ def gen_package(cn, r): [t["task"] for t in tasks][:8] essential = [s for s in skills if s["relation_type"] == "essential"] optional = [s for s in skills if s["relation_type"] != "essential"] - hot = [s for s in software if s["hot_technology"] == "Y"] trigger_examples = "; ".join(core_tasks[:3]) if core_tasks else \ f"typical {label} responsibilities" @@ -210,6 +273,9 @@ def gen_package(cn, r): ) # --- SKILL.md (< 300 lines, progressive disclosure into references/) --- + # Phase 2d: intro = grammatically clean rewrite (cached, gemma3); + # the raw ESCO text stays reachable via the provenance link in the footer. + intro = clean_intro(slug, (r["description"] or r["definition"] or "").strip()) lines = [ "---", f"name: {slug}", @@ -218,7 +284,7 @@ def gen_package(cn, r): "", f"# {label.title()}", "", - (r["description"] or r["definition"] or "").strip(), + intro, "", "## Core workflow", "", @@ -236,6 +302,7 @@ def gen_package(cn, r): ] # enrichment files (phase 3) survive regeneration — link them when present enrichment = [ + ("references/ai-skills.md", "matched external AI agent skills (tiered, per-source attribution)"), ("references/market.md", "FULL market-evidence report from real job ads (rankings, seniority, titles)"), ("references/usecases.md", "ranked work packages the agent can take over"), ("references/intake.md", "mandatory questions before starting any task"), @@ -252,25 +319,25 @@ def gen_package(cn, r): "## Key competences (essential)", "", ] - for s in essential[:15]: + # Phase 2d: FULL essential list — never a silently cut alphabetical prefix. + for s in essential: lines.append(f"- {s['preferred_label']}") - if hot: - lines += ["", "## Hot technologies", ""] - seen = set() - for s in hot: - if s["example"] not in seen: - lines.append(f"- {s['example']}") - seen.add(s["example"]) - if len(seen) >= 10: - break + # "Hot technologies" intentionally NOT emitted here: it is derived from + # post-gate market evidence by p3c (omitted when the corpus is too thin), + # never from the alphabetical O*NET dump. lines += [ "", "---", f"*Sources: ESCO v1.2.1 ({r['concept_uri']}), O*NET 30.3 ({r['onet_id']}). " "See manifest.json for licensing/attribution.*", - "", ] - write(os.path.join(base, "SKILL.md"), "\n".join(lines)) + # Phase 2d: standardized proxy caveat wherever O*NET data is not an exact match + if r.get("onet_id") and (r.get("match_type") or "").lower() not in ("exact", ""): + lines.append(f"*O*NET nearest match: {r['onet_id']} {r.get('onet_title', '')} " + "(proxy — no exact O*NET occupation exists).*") + lines.append("") + write(os.path.join(base, "SKILL.md"), + apply_overlays(slug, "\n".join(lines))) # --- references/profile.md --- prof = [ diff --git a/pipeline/p3c_aggregate.py b/pipeline/p3c_aggregate.py index 92a557c7..2091b150 100644 --- a/pipeline/p3c_aggregate.py +++ b/pipeline/p3c_aggregate.py @@ -189,6 +189,39 @@ def aggregate_for_occupation(cn, slug, skills_dir, max_retries=3): by_file.setdefault(SECTION_FOR[etype], {}).setdefault(etype, []).append( (entity, pct)) + # Phase 2d: "Hot technologies" in SKILL.md — from the post-gate + # market ranking, never the alphabetical O*NET dump. Omitted when + # the corpus is too thin for a ranking. + HOT_MARKER = "" + skill_path = os.path.join(skills_dir, slug, "SKILL.md") + if os.path.exists(skill_path): + stext = open(skill_path, encoding="utf-8").read() + stext = re.sub(rf"\n?{HOT_MARKER}.*?{HOT_MARKER}\n?", "", + stext, flags=re.S) + if total >= 30: + hot_rows = cur.execute(""" + SELECT TOP 10 ee.entity, COUNT(DISTINCT ee.job_id) AS jobs + FROM evidence_entity ee + JOIN evidence_job ej ON ej.job_id=ee.job_id + WHERE ej.occupation_slug=? AND ee.entity_type='tools' + GROUP BY ee.entity HAVING COUNT(DISTINCT ee.job_id) >= ? + ORDER BY COUNT(DISTINCT ee.job_id) DESC, ee.entity + """, slug, REPORT_MIN_JOBS).fetchall() + if hot_rows: + block = [f"\n{HOT_MARKER}", "", + "## Hot technologies", + "", + f"Top tools from {total} gated job ads " + f"(see references/market.md, as of " + f"{date.today().isoformat()}):", ""] + block += [f"- {e} — {round(100.0*n/total)} %" + for e, n in hot_rows] + block += ["", HOT_MARKER] + stext = re.sub(r"(?m)^---\n\*Sources:", + "\n".join(block) + "\n\n---\n*Sources:", + stext, count=1) + open(skill_path, "w", encoding="utf-8", newline="\n").write(stext) + stamp = date.today().isoformat() for fname, groups in by_file.items(): path = os.path.join(skills_dir, slug, "references", fname)