diff --git a/.gitignore b/.gitignore index 05d9a2e9..f433c017 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,9 @@ __pycache__/ # Local scratch tmp/ + +# SE knowledge layer bulk data +knowledge/data/raw/ +knowledge/data/extracted/ +knowledge/data/se.db* + diff --git a/gitea-custom/templates/home.tmpl b/gitea-custom/templates/home.tmpl index 1106da86..3974d2e4 100644 --- a/gitea-custom/templates/home.tmpl +++ b/gitea-custom/templates/home.tmpl @@ -207,6 +207,7 @@ market-evidence crawl started … running for … … Ollama extraction requests + status as of … · computed in … ms · auto-refreshes every 5 min
@@ -447,6 +448,12 @@ setSt("started", s.started_at); setSt("elapsed", s.elapsed_h + " h " + s.elapsed_m + " min"); setSt("ollama", Number(s.ollama_requests || 0).toLocaleString("en-US")); + if (s.se_step) { + var seEl = document.getElementById("sf-se-status"); + if (seEl) seEl.style.display = ""; + setSt("se_step", s.se_step); + setSt("se_detail", s.se_detail ? s.se_detail.slice(0, 90) : ""); + } setSt("updated", s.updated_at); setSt("ms", s.compute_ms); }).catch(function () {}); diff --git a/knowledge/DECISIONS.md b/knowledge/DECISIONS.md new file mode 100644 index 00000000..e3f38a65 --- /dev/null +++ b/knowledge/DECISIONS.md @@ -0,0 +1,56 @@ +# DECISIONS — Stack-Exchange-Wissensschicht + +Autonome Entscheidungen dieser Pipeline, chronologisch. + +## D1 — Dump-Version +`stackexchange_20260331` (archive.org-Suche am 2026-07-09 ergab einen neueren +Mirror als den im Briefing genannten `stackexchange_20251231`). Dateien liegen +im Unterordner `stackexchange_20260331/`; alle 6 Ziel-Archive vorhanden, +zusammen ~929 MB (Meta-Sites bewusst NICHT geladen). + +## D2 — Projektort & Repo +Als Unterprojekt `knowledge/` im bestehenden Pipeline-Repo `C:\dev\skillfactor` +(Branch `quality/reference-standard`), nicht als separates Repo: die +Homepage-Quellen (`gitea-custom/templates/home.tmpl`), stats.json-Erzeugung +und die Provenance-Maschinerie (p3d) leben hier — die SE-Integration greift +direkt in diese Bausteine. + +## D3 — Referenz-Taxonomien +KEIN Neu-Download: ESCO v1.2.1 und O*NET 30.3 liegen bereits vollständig als +Tabellen im lokalen MSSQL (`skillfactor`-DB: esco_occupation, esco_skill, +esco_occ_skill, onet_occupation, onet_task, onet_software, …), geladen von den +offiziellen Quellen in einer früheren Pipeline-Phase. Wiederverwendung ist +idempotent und spart Zeit/Bandbreite; Versionen sind in den Paket-Manifests +dokumentiert. + +## D4 — 7z-Entpackung +`py7zr` (pip) statt System-7-Zip: keine Binary-Abhängigkeit, selektive +Extraktion nur von `Posts.xml` + `Tags.xml` je Site. + +## D5 — LLM-Arbeitsteilung (Kosten/Kontext) +- Tag→Taxonomie-Mapping: Claude klassifiziert die frequenzstärksten Tags + direkt in Batches (Sprachverständnis, Konfidenz + Rationale); der Long-Tail + (niedrige Frequenz nahe der 20er-Schwelle) läuft über das lokale + gemma3:27b mit Claude-designtem Prompt und wird stichprobengeprüft. + Grund: mehrere tausend Tags × Rationale sprengen das Session-Kontextbudget. +- Q&A-Zusammenfassungen (Schritt 4, ~10×150 Einträge): lokales gemma3 + (etabliertes Muster der Evidence-Pipeline: Ollama extrahiert, Claude + designt Prompt + validiert + macht Stichproben-QA). Läuft detached, GPU + wird mit dem laufenden Evidence-Crawl geteilt. + +## D6 — Homepage-Integration +Der Homepage-Code LIEGT im Arbeitsverzeichnis (Live-Template +`Tools\gitea\custom\templates\home.tmpl`, Spiegel in +`gitea-custom/templates/`). Integration erfolgt daher direkt (kein +`homepage/`-Snippet-Ordner nötig): Stack Exchange wird 6. Provenance-Kategorie +(`stackx`) in p3d/p4/stats.json; der Donut-/Legenden-Anteil bemisst sich +datenbasiert aus den Content-Items der kompilierten Q&A-Einträge (gleiche +Zählmethode wie alle anderen Quellen: count_items über die generierten +Dateien). Für die 10 Eval-Berufe wird das kompilierte Wissen zusätzlich als +`references/practitioner-qa.md` in die zugehörigen Skill-Pakete gelegt — +damit ist der Pie-Anteil echt gemessen, nicht angenommen. + +## D7 — Live-Fortschritt (Nutzerwunsch 2026-07-09) +Die Pipeline schreibt `knowledge/data/progress-se.json` nach jedem Schritt; +der bestehende 5-Minuten-Status-Writer merged das in `assets/status.json`, +die Homepage-Statusleiste zeigt den SE-Layer-Stand neben dem Crawl-Zähler. diff --git a/knowledge/README.md b/knowledge/README.md new file mode 100644 index 00000000..fd09213d --- /dev/null +++ b/knowledge/README.md @@ -0,0 +1,48 @@ +# SkillFactor knowledge layer — Stack Exchange practitioner Q&A + +Curated practitioner knowledge from the Stack Exchange data dumps +(CC-BY-SA 4.0), mapped onto ESCO skills / O*NET occupations and compiled +into per-profession knowledge files. + +**Dump version:** `stackexchange_20260331` (archive.org community mirror — +newest at build time, see DECISIONS.md D1). Sites: workplace, pm, law, +money, softwareengineering, datascience (~929 MB compressed). + +## Setup + +``` +# venv of the parent project is reused +..\.venv\Scripts\pip install py7zr +powershell -File scripts\download.ps1 # resumable, skips existing +``` + +## Pipeline (idempotent, in order) + +``` +python pipeline\se_pipeline.py ingest # 7z -> SQLite (streamed iterparse) +python pipeline\se_pipeline.py filter # quality gates -> qa_pairs +python pipeline\se_pipeline.py tags # tag frequencies >= 20 -> CSV +# -> LLM tag mapping produces mappings\tag_to_skill.csv (see DECISIONS D5) +python pipeline\se_pipeline.py compile # professions\\knowledge.md + vocabulary.json +python pipeline\se_pipeline.py verify # consistency checks (exit != 0 on failure) +``` + +Every step writes `data/progress-se.json`; the homepage status strip picks +it up within 5 minutes (see DECISIONS D7). + +## Layout + +``` +data/raw/ downloaded 7z archives (git-ignored) +data/extracted/ Posts/Tags/Users.xml per site (git-ignored) +data/se.db SQLite: questions/answers/users/qa_pairs (git-ignored) +mappings/ tag_to_skill.csv, unmapped.csv (versioned) +professions/ /knowledge.md, vocabulary.json (versioned) +``` + +## License + +Source content: Stack Exchange Network, **CC-BY-SA 4.0** — see +`ATTRIBUTION.md`. Compiled knowledge files are derivative works and carry +per-entry attribution (question link, author, license); published derived +content must remain share-alike. diff --git a/knowledge/pipeline/se_pipeline.py b/knowledge/pipeline/se_pipeline.py new file mode 100644 index 00000000..69546ebe --- /dev/null +++ b/knowledge/pipeline/se_pipeline.py @@ -0,0 +1,277 @@ +"""SkillFactor knowledge layer — Stack Exchange practitioner Q&A. + +Reproducible pipeline over the archive.org data-dump mirror +(stackexchange_20260331, CC-BY-SA 4.0). See knowledge/README.md. + +CLI: + python knowledge/pipeline/se_pipeline.py ingest 7z -> SQLite (streamed) + python knowledge/pipeline/se_pipeline.py filter quality gates + python knowledge/pipeline/se_pipeline.py tags tag frequency export + python knowledge/pipeline/se_pipeline.py compile profession knowledge files + python knowledge/pipeline/se_pipeline.py verify consistency checks + +Idempotent: existing archives are not re-downloaded (scripts/download.ps1), +extraction/ingest skip completed sites, summaries are cached per post id. +Every step updates data/progress-se.json for the homepage live status. +""" +import argparse +import html +import json +import os +import re +import sqlite3 +import sys +from datetime import datetime +from xml.etree import ElementTree as ET + +K = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # knowledge/ +RAW = os.path.join(K, "data", "raw") +EXTRACTED = os.path.join(K, "data", "extracted") +DB = os.path.join(K, "data", "se.db") +PROGRESS = os.path.join(K, "data", "progress-se.json") + +DUMP_VERSION = "stackexchange_20260331" +DUMP_URL = "https://archive.org/download/stackexchange_20260331/" + +SITES = { + "workplace": "workplace.stackexchange.com", + "pm": "pm.stackexchange.com", + "law": "law.stackexchange.com", + "money": "money.stackexchange.com", + "softwareengineering": "softwareengineering.stackexchange.com", + "datascience": "datascience.stackexchange.com", +} + +MIN_Q_SCORE = 5 +MIN_A_SCORE = 5 +MIN_ANSWER_CHARS = 300 +TAG_MIN_FREQ = 20 + + +def progress(step, **info): + data = {} + if os.path.exists(PROGRESS): + try: + data = json.load(open(PROGRESS, encoding="utf-8")) + except ValueError: + data = {} + data["step"] = step + data["updated_at"] = datetime.now().isoformat(timespec="seconds") + data.setdefault("steps", {})[step] = { + "at": data["updated_at"], **info} + json.dump(data, open(PROGRESS, "w", encoding="utf-8"), indent=1) + + +def strip_html(s): + if not s: + return "" + s = re.sub(r"<(pre|code)[^>]*>(.*?)", + lambda m: "\n```\n" + m.group(2) + "\n```\n", s, flags=re.S) + s = re.sub(r"|

|", "\n", s) + s = re.sub(r"<[^>]+>", "", s) + return html.unescape(s).strip() + + +def db(): + cn = sqlite3.connect(DB) + cn.execute("PRAGMA journal_mode=WAL") + return cn + + +def ensure_schema(cn): + cn.executescript(""" + CREATE TABLE IF NOT EXISTS questions( + site TEXT, id INTEGER, title TEXT, body TEXT, tags TEXT, + score INTEGER, accepted_answer_id INTEGER, view_count INTEGER, + closed INTEGER, owner_id INTEGER, PRIMARY KEY(site, id)); + CREATE TABLE IF NOT EXISTS answers( + site TEXT, id INTEGER, parent_id INTEGER, body TEXT, + score INTEGER, owner_id INTEGER, PRIMARY KEY(site, id)); + CREATE TABLE IF NOT EXISTS users( + site TEXT, id INTEGER, display_name TEXT, PRIMARY KEY(site, id)); + CREATE TABLE IF NOT EXISTS ingest_log( + site TEXT PRIMARY KEY, questions INTEGER, answers INTEGER, + users INTEGER, at TEXT); + CREATE INDEX IF NOT EXISTS ix_answers_parent ON answers(site, parent_id); + """) + + +def cmd_ingest(): + import py7zr + cn = db() + ensure_schema(cn) + done = {r[0] for r in cn.execute("SELECT site FROM ingest_log")} + for key, host in SITES.items(): + if key in done: + print(f"{key}: already ingested — skip") + continue + arc = os.path.join(RAW, f"{host}.7z") + if not os.path.exists(arc): + print(f"{key}: archive missing ({arc}) — skip") + continue + xdir = os.path.join(EXTRACTED, key) + os.makedirs(xdir, exist_ok=True) + if not os.path.exists(os.path.join(xdir, "Posts.xml")): + print(f"{key}: extracting Posts/Tags/Users ...") + with py7zr.SevenZipFile(arc) as z: + names = [n for n in z.getnames() + if n in ("Posts.xml", "Tags.xml", "Users.xml")] + z.extract(path=xdir, targets=names) + + nq = na = nu = 0 + qb, ab = [], [] + for _ev, el in ET.iterparse(os.path.join(xdir, "Posts.xml")): + if el.tag != "row": + continue + pt = el.get("PostTypeId") + if pt == "1": + qb.append((key, int(el.get("Id")), el.get("Title", ""), + strip_html(el.get("Body", "")), + el.get("Tags", ""), int(el.get("Score", 0)), + int(el.get("AcceptedAnswerId", 0) or 0), + int(el.get("ViewCount", 0) or 0), + 1 if el.get("ClosedDate") else 0, + int(el.get("OwnerUserId", 0) or 0))) + nq += 1 + elif pt == "2": + ab.append((key, int(el.get("Id")), + int(el.get("ParentId", 0) or 0), + strip_html(el.get("Body", "")), + int(el.get("Score", 0)), + int(el.get("OwnerUserId", 0) or 0))) + na += 1 + el.clear() + if len(qb) >= 2000: + cn.executemany("INSERT OR REPLACE INTO questions VALUES(?,?,?,?,?,?,?,?,?,?)", qb) + qb = [] + if len(ab) >= 2000: + cn.executemany("INSERT OR REPLACE INTO answers VALUES(?,?,?,?,?,?)", ab) + ab = [] + if qb: + cn.executemany("INSERT OR REPLACE INTO questions VALUES(?,?,?,?,?,?,?,?,?,?)", qb) + if ab: + cn.executemany("INSERT OR REPLACE INTO answers VALUES(?,?,?,?,?,?)", ab) + + upath = os.path.join(xdir, "Users.xml") + if os.path.exists(upath): + ub = [] + for _ev, el in ET.iterparse(upath): + if el.tag == "row": + ub.append((key, int(el.get("Id")), + el.get("DisplayName", ""))) + nu += 1 + el.clear() + if len(ub) >= 5000: + cn.executemany("INSERT OR REPLACE INTO users VALUES(?,?,?)", ub) + ub = [] + if ub: + cn.executemany("INSERT OR REPLACE INTO users VALUES(?,?,?)", ub) + + cn.execute("INSERT OR REPLACE INTO ingest_log VALUES(?,?,?,?,?)", + (key, nq, na, nu, datetime.now().isoformat())) + cn.commit() + print(f"{key}: {nq} questions, {na} answers, {nu} users") + progress("ingest", site=key, questions=nq, answers=na) + cn.close() + progress("ingest-done") + + +def cmd_filter(): + cn = db() + cn.executescript(""" + DROP TABLE IF EXISTS qa_pairs; + CREATE TABLE qa_pairs( + site TEXT, q_id INTEGER, title TEXT, q_body TEXT, tags TEXT, + q_score INTEGER, view_count INTEGER, + a_id INTEGER, a_body TEXT, a_score INTEGER, a_accepted INTEGER, + q_owner INTEGER, a_owner INTEGER, + PRIMARY KEY(site, q_id, a_id)); + """) + stats = {} + for key in SITES: + before = cn.execute( + "SELECT COUNT(*) FROM questions WHERE site=?", (key,)).fetchone()[0] + # accepted + top-scored answer per qualifying question + cn.execute(""" + INSERT OR REPLACE INTO qa_pairs + SELECT q.site, q.id, q.title, q.body, q.tags, q.score, q.view_count, + a.id, a.body, a.score, + CASE WHEN a.id = q.accepted_answer_id THEN 1 ELSE 0 END, + q.owner_id, a.owner_id + FROM questions q + JOIN answers a ON a.site = q.site AND a.parent_id = q.id + WHERE q.site = ? + AND q.score >= ? + AND q.closed = 0 + AND LENGTH(a.body) >= ? + AND (a.id = q.accepted_answer_id OR + a.id = (SELECT a2.id FROM answers a2 + WHERE a2.site = q.site AND a2.parent_id = q.id + ORDER BY a2.score DESC, a2.id LIMIT 1)) + AND EXISTS (SELECT 1 FROM answers ax + WHERE ax.site = q.site AND ax.parent_id = q.id + AND (ax.score >= ? OR ax.id = q.accepted_answer_id)) + """, (key, MIN_Q_SCORE, MIN_ANSWER_CHARS, MIN_A_SCORE)) + after_q = cn.execute( + "SELECT COUNT(DISTINCT q_id) FROM qa_pairs WHERE site=?", + (key,)).fetchone()[0] + after_p = cn.execute( + "SELECT COUNT(*) FROM qa_pairs WHERE site=?", (key,)).fetchone()[0] + stats[key] = {"questions_raw": before, "questions_kept": after_q, + "pairs_kept": after_p} + print(f"{key}: {before} raw questions -> {after_q} kept " + f"({after_p} Q&A pairs)") + cn.commit() + cn.close() + json.dump(stats, open(os.path.join(K, "data", "filter-stats.json"), "w", + encoding="utf-8"), indent=1) + progress("filter-done", **{k: v["questions_kept"] for k, v in stats.items()}) + + +def cmd_tags(): + cn = db() + out = os.path.join(K, "data", "tags_freq.csv") + rows = [] + for key in SITES: + freq = {} + for (tags,) in cn.execute( + "SELECT tags FROM qa_pairs WHERE site=?", (key,)): + for t in re.findall(r"[<|]([^><|]+)[>|]", tags or ""): + freq[t] = freq.get(t, 0) + 1 + for t, n in sorted(freq.items(), key=lambda kv: -kv[1]): + if n >= TAG_MIN_FREQ: + rows.append((key, t, n)) + with open(out, "w", encoding="utf-8", newline="\n") as f: + f.write("site,tag,count\n") + for r in rows: + f.write(f"{r[0]},{r[1]},{r[2]}\n") + print(f"{len(rows)} tags with freq >= {TAG_MIN_FREQ} -> {out}") + per_site = {} + for s, _t, _n in rows: + per_site[s] = per_site.get(s, 0) + 1 + print(per_site) + progress("tags-done", tags_total=len(rows), **per_site) + cn.close() + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("cmd", choices=["ingest", "filter", "tags", + "compile", "verify"]) + args = ap.parse_args() + if args.cmd == "ingest": + cmd_ingest() + elif args.cmd == "filter": + cmd_filter() + elif args.cmd == "tags": + cmd_tags() + elif args.cmd == "compile": + from se_compile import cmd_compile + cmd_compile() + elif args.cmd == "verify": + from se_verify import cmd_verify + cmd_verify() + + +if __name__ == "__main__": + main() diff --git a/knowledge/scripts/download.ps1 b/knowledge/scripts/download.ps1 new file mode 100644 index 00000000..953d0390 --- /dev/null +++ b/knowledge/scripts/download.ps1 @@ -0,0 +1,22 @@ +# Lädt die 6 Stack-Exchange-Site-Archive vom archive.org-Mirror (resumable). +# Version: stackexchange_20260331 (neuester Mirror, geprüft 2026-07-09). +$base = "https://archive.org/download/stackexchange_20260331/stackexchange_20260331" +$dest = "C:\dev\skillfactor\knowledge\data\raw" +New-Item -ItemType Directory -Force $dest | Out-Null +$files = @( + "workplace.stackexchange.com.7z", + "pm.stackexchange.com.7z", + "law.stackexchange.com.7z", + "money.stackexchange.com.7z", + "softwareengineering.stackexchange.com.7z", + "datascience.stackexchange.com.7z" +) +$log = "C:\dev\skillfactor\docs\logs\se_download.log" +foreach ($f in $files) { + "$(Get-Date -Format 'HH:mm:ss') start $f" | Out-File $log -Append -Encoding utf8 + # curl.exe: resumable (-C -), folgt Redirects (-L), still (-sS) + & curl.exe -L -sS -C - -o "$dest\$f" "$base/$f" 2>&1 | Out-File $log -Append -Encoding utf8 + $size = [Math]::Round((Get-Item "$dest\$f").Length / 1MB) + "$(Get-Date -Format 'HH:mm:ss') done $f ($size MB)" | Out-File $log -Append -Encoding utf8 +} +"$(Get-Date -Format 'HH:mm:ss') ALL DOWNLOADS COMPLETE" | Out-File $log -Append -Encoding utf8 diff --git a/pipeline/run-status-writer.ps1 b/pipeline/run-status-writer.ps1 index fcbcf1fa..16e9b154 100644 --- a/pipeline/run-status-writer.ps1 +++ b/pipeline/run-status-writer.ps1 @@ -54,6 +54,12 @@ while ($true) { $startDt = [datetime]::ParseExact($started, "yyyy-MM-dd HH:mm", $null) $elapsed = (Get-Date) - $startDt $sw.Stop() + # Stack-Exchange-Wissensschicht: Fortschritt der knowledge-Pipeline + $se = $null + $seFile = "$dev\knowledge\data\progress-se.json" + if (Test-Path $seFile) { + try { $se = Get-Content $seFile -Raw -Encoding utf8 | ConvertFrom-Json } catch {} + } $obj = [ordered]@{ done = $done total = $total @@ -62,6 +68,11 @@ while ($true) { elapsed_h = [Math]::Floor($elapsed.TotalHours) elapsed_m = $elapsed.Minutes ollama_requests = $ollama + se_step = $(if ($se) { $se.step } else { $null }) + se_updated = $(if ($se) { $se.updated_at } else { $null }) + se_detail = $(if ($se -and $se.steps -and $se.step -and $se.steps.($se.step)) { + ($se.steps.($se.step) | ConvertTo-Json -Compress) + } else { $null }) updated_at = (Get-Date -Format "yyyy-MM-dd HH:mm:ss") compute_ms = $sw.ElapsedMilliseconds }