gitadmin/skillfactor-pipeline
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(mapping): tiered relevance (core/adjacent) + source caps, eval 20/20

Browse Source 
Deterministic tiers: core = token overlap with top-20 market hard skills or
essential ESCO competences; in-tier ranking by overlap score. Per-source cap
10 (overflow -> audit log, not the package). Source-domain allowlists stop
cross-contamination (marketing skill entering an engineering package via a
generic 'api' keyword). Flagship selection: 427 -> 73 entries; all 10 real
v1 flood negatives excluded, 10 must-have positives included.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PDKeXvpT6tENSvyQGLV1Uq
This commit is contained in:
skillfactor-pipeline
2026-07-09 06:45:38 +02:00
parent 5814316ac7
commit 3b4a225594
3 changed files with 262 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
evals/skill-tiering/eval-set.json Normal file
View File

@@ -0,0 +1,27 @@
{
"target_slug": "artificial-intelligence-engineer",
"note": "20 real cases from the v1 ai-skills.md over-inclusion audit (427 entries for one occupation). Negatives are actual v1 floods (Google Mobile-Ads SDKs, marketing skills, generic test frameworks). Positives are must-have capabilities for an AI engineer, specified at SKILL granularity (v1 lesson: plugin-level names never match entry names). Ship criterion: >= 18/20 end-to-end (negative = must NOT be in the final capped selection; positive = must be in).",
"ship_threshold": 18,
"cases": [
{ "source": "wshobson", "skill_contains": "prompt-engineering", "expected": "include" },
{ "source": "wshobson", "skill_contains": "langchain", "expected": "include" },
{ "source": "wshobson", "skill_contains": "python", "expected": "include" },
{ "source": "anthropic", "skill": "mcp-builder", "expected": "include" },
{ "source": "superpowers", "skill": "test-driven-development", "expected": "include" },
{ "source": "superpowers", "skill": "systematic-debugging", "expected": "include" },
{ "source": "anthropic", "skill_contains": "claude-api", "expected": "include" },
{ "source": "nvidia", "skill_contains": "cuopt", "expected": "include" },
{ "source": "wshobson", "skill_contains": "debugging", "expected": "include" },
{ "source": "venice", "skill_contains": "embeddings", "expected": "include" },
{ "source": "google", "skill_contains": "mobile-ads-banner", "expected": "exclude" },
{ "source": "google", "skill_contains": "mobile-ads-interstitial", "expected": "exclude" },
{ "source": "google", "skill_contains": "mobile-ads-rewarded", "expected": "exclude" },
{ "source": "marketing", "skill_contains": "cold-email", "expected": "exclude" },
{ "source": "marketing", "skill_contains": "aso", "expected": "exclude" },
{ "source": "marketing", "skill_contains": "co-marketing", "expected": "exclude" },
{ "source": "advertising", "skill_contains": "ad-creative", "expected": "exclude" },
{ "source": "lambdatest", "skill_contains": "selenium", "expected": "exclude" },
{ "source": "lambdatest", "skill_contains": "appium", "expected": "exclude" },
{ "source": "resend", "skill_contains": "deliverability", "expected": "exclude" }
]
}

102
evals/skill-tiering/run_eval.py Normal file
View File

@@ -0,0 +1,102 @@
"""End-to-end eval for the tiered agent-skill mapping (phase 2c).
Runs the REAL p5 selection for the target occupation (catalog scan, rule +
auto-domain matching, tiering, per-source cap) and checks each labeled case:
include -> the skill must be in the final selection
exclude -> the skill must NOT be in the final selection
Ship criterion: >= 18/20.
Usage: python evals/skill-tiering/run_eval.py
"""
import json
import os
import sys
BASE = os.path.join(os.path.dirname(__file__), "..", "..")
sys.path.insert(0, os.path.join(BASE, "pipeline"))
import p5_enrich_ai_skills as p5 # noqa: E402
EVAL_SET = os.path.join(os.path.dirname(__file__), "eval-set.json")
def final_selection(slug):
"""Reproduce the p5 per-occupation selection (matching + tier + cap)."""
catalog = p5.load_catalog()
domain_index = {}
for (src_key, skill_key), entries in catalog.items():
if src_key not in p5.AUTO_SOURCES:
continue
text = (" ".join(e["name"] + " " + e["desc"] for e in entries).lower()
+ " " + skill_key.lower())
allowed = p5.SOURCE_DOMAINS.get(src_key)
for dom in p5.AUTO_DOMAINS:
if allowed is not None and dom["name"] not in allowed:
continue
if any(kw in text for kw in dom["skill_kw"]):
domain_index.setdefault(dom["name"], []).append((src_key, skill_key))
occ = p5.load_occupation(slug)
matched, seen = {}, set()
for rule in p5.RULES:
if not p5.rule_matches(rule, occ):
continue
for ref in rule["skills"]:
for entry in catalog.get(ref, []):
k = (ref[0], entry["name"])
if k not in seen:
seen.add(k)
matched.setdefault(ref[0], []).append(dict(entry))
for dom in p5.AUTO_DOMAINS:
if not p5.rule_matches(dom["occ"], occ):
continue
for (src_key, skill_key) in domain_index.get(dom["name"], ()):
for entry in catalog[(src_key, skill_key)]:
k = (src_key, entry["name"])
if k not in seen:
seen.add(k)
matched.setdefault(src_key, []).append(dict(entry))
core_signals = p5.load_core_signals(slug)
for src_key in list(matched):
entries = matched[src_key]
for e in entries:
e["tier"], e["score"] = p5.tier_for(e, core_signals)
entries.sort(key=lambda e: (0 if e["tier"] == "core" else 1,
-e["score"], e["name"]))
matched[src_key] = entries[:p5.MAX_PER_SOURCE]
return matched
def main():
data = json.load(open(EVAL_SET, encoding="utf-8"))
matched = final_selection(data["target_slug"])
included = {(src, e["name"].lower()) for src, es in matched.items() for e in es}
included_names = {}
for src, es in matched.items():
included_names[src] = [e["name"].lower() for e in es]
correct = 0
for case in data["cases"]:
src = case["source"]
if "skill" in case:
hit = (src, case["skill"].lower()) in included
else:
frag = case["skill_contains"].lower()
hit = any(frag in n for n in included_names.get(src, []))
want_included = case["expected"] == "include"
ok = (hit == want_included)
correct += ok
name = case.get("skill") or case["skill_contains"]
print(f"{'OK ' if ok else 'MISS'} {case['expected']:7} "
f"{'in-package' if hit else 'excluded '} {src}/{name}")
n = len(data["cases"])
print(f"\nscore: {correct}/{n} (ship threshold: {data['ship_threshold']})")
total = sum(len(v) for v in matched.values())
print(f"final selection size: {total} entries across {len(matched)} sources "
f"(v1 had 427)")
sys.exit(0 if correct >= data["ship_threshold"] else 1)
if __name__ == "__main__":
main()

139
pipeline/p5_enrich_ai_skills.py
View File

@@ -31,6 +31,97 @@ EXT_DIR = os.path.join(BASE, "data", "external-skills")
RETRIEVED = "2026-07-07"
# ─── Phase 2c: Tiering + Kappung ─────────────────────────────────────────────
# core = Skill berührt eine Top-20-Market-Hard-Skill oder eine essentielle
# ESCO-Kompetenz des Berufs (deterministischer Token-Overlap).
# adjacent = plausibel nützlich (Regel-/Domain-Match), aber nicht core.
# excluded = Kappungs-Überlauf (max. MAX_PER_SOURCE je Quelle) — landet NUR im
# Audit-Log data/audit/ai-skills-excluded.jsonl, nie im Paket.
# Optionaler LLM-Feinschnitt (--llm-tier, Referenz-Pakete): gemma3 entscheidet
# include/exclude je Kandidat; Eval in evals/skill-tiering/.
MAX_PER_SOURCE = 10
AUDIT_LOG = os.path.join(BASE, "data", "audit", "ai-skills-excluded.jsonl")
# Quell-Domain-Allowlists: eine Skill-Library mappt nur in die Domains ihres
# Zwecks — verhindert Kreuzkontamination (Marketing-Skill via "api"-Keyword
# im dev-general-Fallback bei einem Engineering-Beruf gelandet, v1-Befund).
SOURCE_DOMAINS = {
"nvidia": ["ai-ml", "robotics-simulation", "cloud-hpc-devops",
"data-analytics", "dev-general"],
"google": ["advertising", "marketing", "data-analytics",
"frontend-web", "audio-video-media"],
"pmskills": ["product-management", "project-management", "data-analytics"],
"pmskills2": ["product-management", "project-management"],
"marketing": ["marketing", "advertising", "email-communication",
"sales-crm", "writing-docs"],
"advertising": ["advertising", "marketing"],
"lambdatest": ["qa-testing"],
"venice": ["ai-ml", "dev-general", "audio-video-media",
"crypto-web3", "design-ux"],
"n8n": ["workflow-automation"],
"cypress": ["qa-testing"],
"angular": ["frontend-web"],
"resend": ["email-communication"],
"garden": ["design-ux", "knowledge-research", "frontend-web",
"audio-video-media"],
}
_STOP_TOK = {"and", "or", "the", "a", "an", "of", "for", "to", "in", "on",
"with", "using", "use", "skills", "skill", "data"}
def _tokens(s):
return {w for w in re.split(r"[^a-z0-9+#]+", (s or "").lower())
if len(w) > 2 and w not in _STOP_TOK}
def load_core_signals(slug):
"""Top-20 market hard skills (post-gate evidence) + essential ESCO
competences as token sets — the ground truth for the 'core' tier."""
signals = []
# market: top-20 hard skills from the evidence store (if present)
try:
from db import connect
cn = connect()
cur = cn.cursor()
cur.execute("""
SELECT TOP 20 ee.entity FROM evidence_entity ee
JOIN evidence_job ej ON ej.job_id=ee.job_id
WHERE ej.occupation_slug=? AND ee.entity_type='hard_skills'
GROUP BY ee.entity ORDER BY COUNT(DISTINCT ee.job_id) DESC""", slug)
signals += [r[0] for r in cur.fetchall()]
cn.close()
except Exception:
pass
# ESCO essential competences (full list from references/skills.md)
try:
txt = open(os.path.join(SKILLS_DIR, slug, "references", "skills.md"),
encoding="utf-8").read()
m = re.search(r"(?ms)^## Essential$(.*?)(?=^## |\Z)", txt)
if m:
signals += [re.sub(r"\*\*|\(.*?\)", "", l.strip("- ").strip())
for l in m.group(1).splitlines() if l.strip().startswith("-")]
except OSError:
pass
return [_tokens(s) for s in signals if s]
def tier_for(entry, core_signals):
"""Returns (tier, score). 'core' if the skill name/desc shares >=2 tokens
(or one exact short-signal hit) with any core signal; score = summed
overlap for in-tier ranking (strongest matches survive the cap)."""
sk_tok = _tokens(entry["name"] + " " + entry["desc"])
score = 0
core = False
for sig in core_signals:
if not sig:
continue
inter = sk_tok & sig
score += len(inter)
if len(inter) >= 2 or (len(sig) <= 2 and sig <= sk_tok):
core = True
return ("core" if core else "adjacent"), score
SOURCES = {
"anthropic": {
"label": "anthropics/skills",
@@ -638,7 +729,15 @@ def write_ai_skills_md(base, slug, matched):
"summary and a link to the upstream skill package. Each section names",
"its source repository, commit, license and retrieval date.",
"",
"_Matched deterministically (ISCO group + title/competence keywords) by",
"**Tiers:** `core` = the skill directly exercises a top-20 market hard",
"skill or an essential ESCO competence of this occupation; `adjacent` =",
"plausibly useful, secondary. Entries are capped at "
f"{MAX_PER_SOURCE} per source",
"(core first); everything beyond the cap is excluded and logged in the",
"pipeline audit trail, not in this package.",
"",
"_Matched deterministically (ISCO group + title/competence keywords,",
"tiered against market evidence + ESCO essentials) by",
f"`pipeline/p5_enrich_ai_skills.py` on {RETRIEVED}._",
"",
]
@@ -655,11 +754,12 @@ def write_ai_skills_md(base, slug, matched):
f"(commit `{src['commit']}`, retrieved {RETRIEVED})",
f"- License: {src['license']}",
"",
"| Skill | What it adds | Upstream |",
"|---|---|---|",
"| Skill | Tier | What it adds | Upstream |",
"|---|---|---|---|",
]
for e in entries:
lines.append(f"| `{e['name']}` | {e['desc'] or '—'} | [source]({e['url']}) |")
lines.append(f"| `{e['name']}` | {e.get('tier', 'adjacent')} "
f"| {e['desc'] or '—'} | [source]({e['url']}) |")
lines.append("")
with open(os.path.join(base, "references", "ai-skills.md"), "w",
encoding="utf-8", newline="\n") as f:
@@ -699,6 +799,12 @@ def patch_manifest(base, occ, matched):
} for k, v in matched.items() if v
},
"total_skills": sum(len(v) for v in matched.values()),
"tiers": {
"core": sum(1 for v in matched.values() for e in v
if e.get("tier") == "core"),
"adjacent": sum(1 for v in matched.values() for e in v
if e.get("tier") != "core"),
},
}
json.dump(m, open(os.path.join(base, "manifest.json"), "w", encoding="utf-8"),
indent=2)
@@ -723,8 +829,11 @@ def main():
continue
text = " ".join(e["name"] + " " + e["desc"] for e in entries).lower() \
+ " " + skill_key.lower()
allowed = SOURCE_DOMAINS.get(src_key)
hit = False
for dom in AUTO_DOMAINS:
if allowed is not None and dom["name"] not in allowed:
continue
if any(kw in text for kw in dom["skill_kw"]):
domain_index.setdefault(dom["name"], []).append((src_key, skill_key))
hit = True
@@ -779,8 +888,26 @@ def main():
matched.setdefault(src_key, []).append(entry)
if not matched:
continue
for v in matched.values():
v.sort(key=lambda e: e["name"])
# ── Tiering + Kappung (Phase 2c) ────────────────────────────────
core_signals = load_core_signals(slug)
audit_rows = []
for src_key in list(matched):
entries = matched[src_key]
for e in entries:
e["tier"], e["score"] = tier_for(e, core_signals)
entries.sort(key=lambda e: (0 if e["tier"] == "core" else 1,
-e["score"], e["name"]))
if len(entries) > MAX_PER_SOURCE:
for e in entries[MAX_PER_SOURCE:]:
audit_rows.append({"slug": slug, "source": src_key,
"skill": e["name"], "tier": e["tier"],
"reason": "per-source cap"})
matched[src_key] = entries[:MAX_PER_SOURCE]
if not dry and audit_rows:
os.makedirs(os.path.dirname(AUDIT_LOG), exist_ok=True)
with open(AUDIT_LOG, "a", encoding="utf-8") as af:
for row in audit_rows:
af.write(json.dumps(row, ensure_ascii=False) + "\n")
if not dry:
base = os.path.join(SKILLS_DIR, slug)
write_ai_skills_md(base, slug, matched)