220 lines
8.6 KiB
Python
220 lines
8.6 KiB
Python
"""Phase 4 — publish the skill library into the running Gitea instance.
|
|
|
|
Creates (idempotent, safe to re-run):
|
|
- orgs: skills-core (public), skills-community (public), tenant-acme (private)
|
|
- one repo per package in skills-core, pushed via git (real commits)
|
|
- skills-community/recruitment-consultant (community edition, PR target)
|
|
- skills-core/marketplace with marketplace.json + README (logo, mermaid)
|
|
|
|
The tenant fork/PR demo lives in p4b_tenant_demo.py.
|
|
Gitea runs on MSSQL behind Caddy; pushes are serialized with a small pause
|
|
(single-core SQL Express — be gentle).
|
|
"""
|
|
import json
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
|
|
import requests
|
|
from dotenv import load_dotenv
|
|
|
|
BASE = os.path.join(os.path.dirname(__file__), "..")
|
|
load_dotenv(os.path.join(BASE, ".env"))
|
|
|
|
GITEA_URL = os.environ["GITEA_URL"].rstrip("/")
|
|
TOKEN = os.environ["GITEA_TOKEN"]
|
|
API = f"{GITEA_URL}/api/v1"
|
|
HDR = {"Authorization": f"token {TOKEN}"}
|
|
SKILLS_DIR = os.path.join(BASE, "skills")
|
|
PUSH_PAUSE_S = 1.5
|
|
|
|
|
|
def api(method, path, ok=(200, 201, 204), **kw):
|
|
r = requests.request(method, f"{API}{path}", headers=HDR, timeout=120, **kw)
|
|
if r.status_code not in ok:
|
|
raise RuntimeError(f"{method} {path} -> {r.status_code}: {r.text[:300]}")
|
|
return r
|
|
|
|
|
|
def ensure_org(name, visibility):
|
|
r = requests.get(f"{API}/orgs/{name}", headers=HDR, timeout=60)
|
|
if r.status_code == 200:
|
|
return
|
|
api("POST", "/orgs", json={"username": name, "visibility": visibility,
|
|
"full_name": name, "description": f"skillfactor — {name}"})
|
|
print(f"org created: {name} ({visibility})")
|
|
|
|
|
|
def ensure_repo(org, name, description=""):
|
|
r = requests.get(f"{API}/repos/{org}/{name}", headers=HDR, timeout=60)
|
|
if r.status_code == 200:
|
|
return False
|
|
api("POST", f"/orgs/{org}/repos",
|
|
json={"name": name, "description": description[:255], "private": False,
|
|
"default_branch": "main", "auto_init": False})
|
|
print(f"repo created: {org}/{name}")
|
|
return True
|
|
|
|
|
|
def push_dir(org, repo, src_dir, message):
|
|
"""Push src_dir content as single commit to main (force: package is source of truth)."""
|
|
push_url = GITEA_URL.replace("://", f"://gitadmin:{TOKEN}@") + f"/{org}/{repo}.git"
|
|
tmp = tempfile.mkdtemp(prefix="sfpush_")
|
|
try:
|
|
shutil.copytree(src_dir, tmp, dirs_exist_ok=True)
|
|
def g(*args):
|
|
subprocess.run(["git", *args], cwd=tmp, check=True,
|
|
capture_output=True, text=True)
|
|
g("init", "-b", "main")
|
|
g("config", "user.name", "skillfactor-pipeline")
|
|
g("config", "user.email", "pipeline@noreply.zeiterfassung.cloud")
|
|
g("add", "-A")
|
|
g("commit", "-m", message)
|
|
g("push", "--force", push_url, "main")
|
|
finally:
|
|
shutil.rmtree(tmp, ignore_errors=True)
|
|
|
|
|
|
def build_marketplace_dir(slugs):
|
|
tmp = tempfile.mkdtemp(prefix="sfmarket_")
|
|
# globale Provenienz aus den Manifests aggregieren (fuer das README-Pie)
|
|
prov_labels = {"esco": "ESCO (occupations & competences)",
|
|
"onet": "O*NET (tasks & tools)",
|
|
"jobads": "Job boards (market evidence)",
|
|
"wiki_ai": "Wikipedia & AI expert curation"}
|
|
prov = {k: 0 for k in prov_labels}
|
|
for slug in slugs:
|
|
m = json.load(open(os.path.join(SKILLS_DIR, slug, "manifest.json"), encoding="utf-8"))
|
|
for k in prov:
|
|
prov[k] += (m.get("provenance", {}).get("items", {}) or {}).get(k, 0)
|
|
prov_rows = "\n".join(f' "{prov_labels[k]}" : {v}' for k, v in prov.items() if v)
|
|
index = []
|
|
for slug in slugs:
|
|
manifest = json.load(open(os.path.join(SKILLS_DIR, slug, "manifest.json"), encoding="utf-8"))
|
|
index.append({
|
|
"name": manifest["name"],
|
|
"title": manifest["title"],
|
|
"slug": slug,
|
|
"esco_uri": manifest["ids"]["esco_uri"],
|
|
"onet_soc": manifest["ids"]["onet_soc"],
|
|
"version": manifest["version"],
|
|
"layer": manifest["layer"],
|
|
"repo": f"{GITEA_URL}/skills-core/{slug}",
|
|
})
|
|
json.dump(index, open(os.path.join(tmp, "marketplace.json"), "w", encoding="utf-8"),
|
|
indent=2)
|
|
os.makedirs(os.path.join(tmp, "assets"), exist_ok=True)
|
|
for logo in ("skillfactor_logo_red.png", "skillfactor_mark_red.png"):
|
|
shutil.copy(os.path.join(BASE, "assets", logo), os.path.join(tmp, "assets", logo))
|
|
readme = f"""<p align="center"><img src="assets/skillfactor_logo_red.png" alt="skillfactor" width="360"></p>
|
|
|
|
# skillfactor marketplace
|
|
|
|
**AI with work experience.** The occupational skill layer: public occupation
|
|
databases (ESCO, O*NET) + live job-ad evidence + expert knowledge, compiled
|
|
into one ready-to-use skill package per occupation. Every employee agent loads
|
|
the package for its role — and starts with the profession built in.
|
|
|
|
**{len(index)} occupations** are currently published in
|
|
[`skills-core`]({GITEA_URL}/skills-core). The machine-readable index is
|
|
[`marketplace.json`](marketplace.json).
|
|
|
|
## Architecture
|
|
|
|
```mermaid
|
|
flowchart LR
|
|
A["ESCO 1.2.1 / O*NET 30.3<br/>+ official crosswalk"] --> DB[("evidence store<br/>Microsoft SQL Server")]
|
|
J["job ads<br/>(JSearch / Adzuna APIs)"] --> DB
|
|
W["Wikipedia / Wikidata<br/>+ literature"] --> G
|
|
DB --> G["generator"]
|
|
G --> CORE["skills-core<br/>1 repo per occupation"]
|
|
CORE -->|fork| T["tenant-acme<br/>(private extensions)"]
|
|
T -->|sanitize gate:<br/>PII scan + eval + human PR review| COMM["skills-community"]
|
|
```
|
|
|
|
## Layer model
|
|
|
|
| Layer | Content | Visibility |
|
|
|---|---|---|
|
|
| community | shared, anonymized best practices | public |
|
|
| company | tenant processes (e.g. `tenant-acme`) | private |
|
|
| role | the role's memory | tenant |
|
|
| person | personal style | employee |
|
|
|
|
Every boundary is a gate: PII/secret scan (`sanitize.yml`), structural eval
|
|
(`eval.yml`), anonymization, and a human pull-request review.
|
|
|
|
## Data provenance
|
|
|
|
Every package documents where its content comes from in its own
|
|
`PROVENANCE.md` (rendered pie chart + table). Library-wide distribution of
|
|
content items:
|
|
|
|
```mermaid
|
|
pie showData
|
|
title Content sources across all {len(slugs)} packages
|
|
{prov_rows}
|
|
```
|
|
|
|
The taxonomy share dominates library-wide because job-ad and expert
|
|
enrichment rolls out occupation by occupation (see the
|
|
[recruiter package]({GITEA_URL}/skills-core/recruitment-consultant/src/branch/main/PROVENANCE.md)
|
|
for a fully enriched split).
|
|
|
|
## Licensing
|
|
|
|
This marketplace includes information from the O*NET Database (v30.3,
|
|
USDOL/ETA, CC BY 4.0) and ESCO (v1.2.1, © European Union —
|
|
[conditions](https://esco.ec.europa.eu/en/use-esco/download)).
|
|
skillfactor is not endorsed by USDOL/ETA or the European Commission.
|
|
"""
|
|
open(os.path.join(tmp, "README.md"), "w", encoding="utf-8", newline="\n").write(readme)
|
|
return tmp
|
|
|
|
|
|
def main():
|
|
only = sys.argv[1] if len(sys.argv) > 1 else None
|
|
ensure_org("skills-core", "public")
|
|
ensure_org("skills-community", "public")
|
|
ensure_org("tenant-acme", "private")
|
|
|
|
slugs = sorted(d for d in os.listdir(SKILLS_DIR)
|
|
if os.path.isdir(os.path.join(SKILLS_DIR, d)))
|
|
if only:
|
|
slugs = [s for s in slugs if s == only]
|
|
print(f"publishing {len(slugs)} packages ...")
|
|
|
|
for i, slug in enumerate(slugs, 1):
|
|
manifest = json.load(open(os.path.join(SKILLS_DIR, slug, "manifest.json"), encoding="utf-8"))
|
|
ensure_repo("skills-core", slug, manifest.get("title", slug))
|
|
push_dir("skills-core", slug, os.path.join(SKILLS_DIR, slug),
|
|
f"feat: {slug} skill package v{manifest['version']}")
|
|
print(f"[{i}/{len(slugs)}] pushed skills-core/{slug}")
|
|
time.sleep(PUSH_PAUSE_S)
|
|
|
|
# community edition of the recruiter package = PR target for the tenant demo
|
|
recruiter = next((s for s in slugs if "recruit" in s), None)
|
|
if recruiter:
|
|
ensure_repo("skills-community", recruiter,
|
|
"community edition — accepts sanitized tenant contributions")
|
|
push_dir("skills-community", recruiter, os.path.join(SKILLS_DIR, recruiter),
|
|
f"feat: community edition of {recruiter}")
|
|
print(f"pushed skills-community/{recruiter}")
|
|
|
|
all_slugs = sorted(d for d in os.listdir(SKILLS_DIR)
|
|
if os.path.isdir(os.path.join(SKILLS_DIR, d)))
|
|
market = build_marketplace_dir(all_slugs)
|
|
try:
|
|
ensure_repo("skills-core", "marketplace", "skillfactor marketplace index")
|
|
push_dir("skills-core", "marketplace", market, "docs: marketplace index + README")
|
|
print("pushed skills-core/marketplace")
|
|
finally:
|
|
shutil.rmtree(market, ignore_errors=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|