skillfactor-pipeline/pipeline/p4c_industry_index.py

"""Phase 4c — industry-navigation index in Gitea.

Creates (or updates) one repo:  skills-core/by-industry

Folder tree in the repo:
  /
  ├── README.md          (top-level: all industries as links)
  ├── it-technology/
  │   ├── README.md      (sub-categories)
  │   ├── professionals/ → README.md (job list + links to skills-core)
  │   └── technicians/   → README.md
  ├── healthcare/
  │   └── ...
  └── ...

Every skill package is referenced by a Markdown link that points to
  {GITEA_URL}/skills-core/{slug}/src/branch/main/SKILL.md
which renders directly in Gitea's DOC mode.

Run:  python pipeline/p4c_industry_index.py
Safe to re-run (idempotent via force-push).
"""
import json, os, shutil, subprocess, tempfile, time
import requests
from dotenv import load_dotenv

BASE     = os.path.join(os.path.dirname(__file__), "..")
SKILLS   = os.path.join(BASE, "skills")
load_dotenv(os.path.join(BASE, ".env"))

GITEA_URL = os.environ["GITEA_URL"].rstrip("/")
TOKEN     = os.environ["GITEA_TOKEN"]
API       = f"{GITEA_URL}/api/v1"
HDR       = {"Authorization": f"token {TOKEN}"}

# ─── ISCO → (industry_slug, industry_label, sub_slug, sub_label) ──────────────

# Major group (1st digit) → default industry for uncategorised
_MAJOR_INDUSTRY = {
    "0": ("military-defense",    "Military & Defense"),
    "1": ("management",          "Management & Leadership"),
    "4": ("administration",      "Administration & Office"),
    "5": ("services-sales",      "Services & Sales"),
    "6": ("agriculture",         "Agriculture & Environment"),
    "7": ("trades-crafts",       "Trades & Crafts"),
    "8": ("production-logistics","Production & Logistics"),
    "9": ("elementary",          "Elementary Occupations"),
}

# 2-digit codes → (industry_slug, industry_label, sub_slug, sub_label)
_2DIGIT = {
    # Professionals (2x)
    "21": ("engineering-science",   "Engineering & Science",     "professionals", "Professionals"),
    "22": ("healthcare",            "Healthcare & Medicine",     "professionals", "Professionals"),
    "23": ("education",             "Education & Training",      "professionals", "Professionals"),
    "24": ("business-finance-law",  "Business, Finance & Law",   "professionals", "Professionals"),
    "25": ("it-technology",         "IT & Technology",           "professionals", "Professionals"),
    "26": ("social-arts-culture",   "Social Science, Arts & Culture", "professionals", "Professionals"),
    # Technicians (3x)
    "31": ("engineering-science",   "Engineering & Science",     "technicians",   "Technicians & Associates"),
    "32": ("healthcare",            "Healthcare & Medicine",     "support",       "Healthcare Support"),
    "33": ("business-finance-law",  "Business, Finance & Law",   "support",       "Business Support"),
    "34": ("social-arts-culture",   "Social Science, Arts & Culture", "technicians", "Technicians & Associates"),
    "35": ("it-technology",         "IT & Technology",           "technicians",   "Technicians & Support"),
    # Management sub-groups
    "11": ("management",            "Management & Leadership",   "chief-executives", "Chief Executives & Legislators"),
    "12": ("management",            "Management & Leadership",   "administrative",   "Administrative & Commercial"),
    "13": ("management",            "Management & Leadership",   "production",       "Production & Specialised"),
    "14": ("management",            "Management & Leadership",   "hospitality",      "Hospitality, Retail & Services"),
    # Administration
    "41": ("administration",        "Administration & Office",   "general-office",   "General & Keyboard Clerks"),
    "42": ("administration",        "Administration & Office",   "customer-service", "Customer Services Clerks"),
    "43": ("administration",        "Administration & Office",   "numerical",        "Numerical & Material Recording"),
    "44": ("administration",        "Administration & Office",   "other-clerical",   "Other Clerical Support"),
    # Services & Sales
    "51": ("services-sales",        "Services & Sales",          "personal-services","Personal Service Workers"),
    "52": ("services-sales",        "Services & Sales",          "sales",            "Sales Workers"),
    "53": ("services-sales",        "Services & Sales",          "personal-care",    "Personal Care Workers"),
    "54": ("services-sales",        "Services & Sales",          "protective",       "Protective Services Workers"),
    # Agriculture
    "61": ("agriculture",           "Agriculture & Environment", "crop-farming",     "Crop Farming"),
    "62": ("agriculture",           "Agriculture & Environment", "animal-farming",   "Animal Farming & Fisheries"),
    "63": ("agriculture",           "Agriculture & Environment", "subsistence",      "Subsistence Farming"),
    # Trades
    "71": ("trades-crafts",         "Trades & Crafts",           "building",         "Building & Construction"),
    "72": ("trades-crafts",         "Trades & Crafts",           "metal-machinery",  "Metal, Machinery & Related"),
    "73": ("trades-crafts",         "Trades & Crafts",           "handicraft",       "Handicraft & Printing"),
    "74": ("trades-crafts",         "Trades & Crafts",           "electrical",       "Electrical & Electronic"),
    "75": ("trades-crafts",         "Trades & Crafts",           "food-processing",  "Food Processing & Related"),
    # Production & Logistics
    "81": ("production-logistics",  "Production & Logistics",    "mining-machinery", "Mining & Building Machinery"),
    "82": ("production-logistics",  "Production & Logistics",    "process-operators","Process Plant Operators"),
    "83": ("production-logistics",  "Production & Logistics",    "transport",        "Transport & Logistics"),
    # Military
    "00": ("military-defense",      "Military & Defense",        "commissioned",     "Commissioned Officers"),
    "01": ("military-defense",      "Military & Defense",        "other-ranks",      "Non-Commissioned Officers & Other Ranks"),
    # Elementary
    "91": ("elementary",            "Elementary Occupations",    "cleaning",         "Cleaning & Helpers"),
    "92": ("elementary",            "Elementary Occupations",    "agriculture-elem", "Agricultural Labourers"),
    "93": ("elementary",            "Elementary Occupations",    "production-elem",  "Production Labourers"),
    "94": ("elementary",            "Elementary Occupations",    "food-elem",        "Food Preparation Assistants"),
    "96": ("elementary",            "Elementary Occupations",    "refuse",           "Refuse & Street Services"),
}

def classify(isco_group: str):
    """Return (ind_slug, ind_label, sub_slug, sub_label) for an ISCO 4-digit code."""
    code = str(isco_group or "").strip().replace(".", "").zfill(4)
    two  = code[:2]
    one  = code[:1]
    if two in _2DIGIT:
        return _2DIGIT[two]
    if one in _MAJOR_INDUSTRY:
        isl, ilbl = _MAJOR_INDUSTRY[one]
        return (isl, ilbl, "general", "General")
    return ("other", "Other Occupations", "general", "General")


# ─── Collect all packages ─────────────────────────────────────────────────────

def load_packages():
    """Yield (slug, title, isco_group) for every package with a manifest."""
    for slug in sorted(os.listdir(SKILLS)):
        mpath = os.path.join(SKILLS, slug, "manifest.json")
        if not os.path.isfile(mpath):
            continue
        try:
            m = json.load(open(mpath, encoding="utf-8"))
            yield slug, m.get("title", slug), m.get("ids", {}).get("isco_group", "")
        except Exception:
            continue


# ─── Build folder tree in a temp dir ─────────────────────────────────────────

GITEA_SKILL_URL = f"{GITEA_URL}/skills-core/{{slug}}/src/branch/main/SKILL.md"

# Root README groups industries by collar class — the catalog's primary
# audience is computer-based (white-collar) work, so it leads.
COLLAR_SECTIONS = [
    ("💻 White collar — computer-based work",
     "The core audience: professions where the computer is the primary work tool.",
     ["it-technology", "business-finance-law", "management", "healthcare",
      "engineering-science", "education", "social-arts-culture", "administration"]),
    ("🛎️ Service & sales",
     "Customer-facing work — partly digital (POS, CRM, booking systems).",
     ["services-sales"]),
    ("🔧 Blue collar — manual & field work",
     "Hands-on professions; packages exist for completeness of the catalog.",
     ["trades-crafts", "production-logistics", "agriculture",
      "military-defense", "elementary", "other"]),
]
INDUSTRY_ORDER = [i for _, _, inds in COLLAR_SECTIONS for i in inds]

def build_tree(packages):
    """Build the full directory structure and return the temp dir path."""
    # groups: { ind_slug: { label, subs: { sub_slug: { label, items: [(slug,title)] } } } }
    groups = {}
    for slug, title, isco in packages:
        ind_s, ind_l, sub_s, sub_l = classify(isco)
        g = groups.setdefault(ind_s, {"label": ind_l, "subs": {}})
        s = g["subs"].setdefault(sub_s, {"label": sub_l, "items": []})
        s["items"].append((slug, title))

    tmp = tempfile.mkdtemp(prefix="sfind_")

    # ── Root README (grouped by collar class, white collar first) ───────────
    root_md = ["# skillfactor — Browse by Industry\n",
               "Navigate skill packages by sector. Each folder below opens a sub-category "
               "listing with direct links to the full skill packages.\n"]

    def _ind_row(ind_s):
        g = groups[ind_s]
        nsubs = len(g["subs"])
        ntotal = sum(len(s["items"]) for s in g["subs"].values())
        return f"| [{g['label']}]({ind_s}/) | {nsubs} | {ntotal} |\n"

    listed = set()
    for title, note, industries in COLLAR_SECTIONS:
        present = [i for i in industries if i in groups]
        if not present:
            continue
        n_section = sum(sum(len(s["items"]) for s in groups[i]["subs"].values())
                        for i in present)
        root_md += [f"\n## {title}\n\n", f"{note} **{n_section} packages.**\n\n",
                    "| Industry | Sub-categories | Packages |\n", "|---|---|---|\n"]
        for ind_s in present:
            root_md.append(_ind_row(ind_s))
            listed.add(ind_s)
    # Any industry not covered by a section
    rest = [i for i in sorted(groups) if i not in listed]
    if rest:
        root_md += ["\n## Further\n\n",
                    "| Industry | Sub-categories | Packages |\n", "|---|---|---|\n"]
        root_md += [_ind_row(i) for i in rest]

    open(os.path.join(tmp, "README.md"), "w", encoding="utf-8", newline="\n").writelines(root_md)

    # ── Per-industry folder ──────────────────────────────────────────────────
    for ind_s, g in groups.items():
        ind_dir = os.path.join(tmp, ind_s)
        os.makedirs(ind_dir, exist_ok=True)

        ind_md = [f"# {g['label']}\n\n",
                  f"[← All industries](../README.md)\n\n",
                  "| Sub-category | Packages |\n",
                  "|---|---|\n"]
        for sub_s in sorted(g["subs"]):
            s = g["subs"][sub_s]
            ind_md.append(f"| [{s['label']}]({sub_s}/) | {len(s['items'])} |\n")
        open(os.path.join(ind_dir, "README.md"), "w", encoding="utf-8", newline="\n").writelines(ind_md)

        # ── Per-sub-category folder ──────────────────────────────────────────
        for sub_s, s in g["subs"].items():
            sub_dir = os.path.join(ind_dir, sub_s)
            os.makedirs(sub_dir, exist_ok=True)

            items_sorted = sorted(s["items"], key=lambda x: x[1])
            sub_md = [f"# {g['label']} — {s['label']}\n\n",
                      f"[← {g['label']}](../README.md) · [← All industries](../../README.md)\n\n",
                      f"{len(items_sorted)} skill packages in this category.\n\n",
                      "| Occupation | Skill Package |\n",
                      "|---|---|\n"]
            for slug, title in items_sorted:
                skill_url = GITEA_SKILL_URL.format(slug=slug)
                repo_url  = f"{GITEA_URL}/skills-core/{slug}"
                sub_md.append(f"| {title.title()} | [SKILL.md]({skill_url}) · [repo]({repo_url}) |\n")
            open(os.path.join(sub_dir, "README.md"), "w", encoding="utf-8", newline="\n").writelines(sub_md)

    return tmp


# ─── Gitea helpers ────────────────────────────────────────────────────────────

def ensure_repo(org, name, description=""):
    r = requests.post(f"{API}/orgs/{org}/repos", headers=HDR, timeout=60,
                      json={"name": name, "description": description[:255],
                            "private": False, "default_branch": "main", "auto_init": False})
    if r.status_code in (200, 201):
        print(f"repo created: {org}/{name}")
    elif r.status_code == 409:
        print(f"repo exists:  {org}/{name}")
    else:
        raise RuntimeError(f"POST /orgs/{org}/repos -> {r.status_code}: {r.text[:200]}")


def push_dir(org, repo, src_dir, message):
    url = GITEA_URL.replace("://", f"://gitadmin:{TOKEN}@") + f"/{org}/{repo}.git"
    tmp = tempfile.mkdtemp(prefix="sfpush_")
    try:
        shutil.copytree(src_dir, tmp, dirs_exist_ok=True)
        def g(*a): subprocess.run(["git", *a], cwd=tmp, check=True, capture_output=True, text=True)
        g("init", "-b", "main")
        g("config", "user.name",  "skillfactor-pipeline")
        g("config", "user.email", "pipeline@noreply.zeiterfassung.cloud")
        g("add", "-A")
        g("commit", "-m", message)
        g("push", "--force", url, "main")
        print(f"pushed {org}/{repo}")
    finally:
        shutil.rmtree(tmp, ignore_errors=True)


# ─── Main ─────────────────────────────────────────────────────────────────────

def main():
    packages = list(load_packages())
    print(f"loaded {len(packages)} packages")

    tree_dir = build_tree(packages)
    try:
        # Count generated files
        n_files = sum(len(fs) for _, _, fs in os.walk(tree_dir))
        print(f"generated {n_files} files in {tree_dir}")

        ensure_repo("skills-core", "by-industry",
                    "Browse skill packages by industry and sub-category")
        push_dir("skills-core", "by-industry", tree_dir,
                 f"docs: industry navigation index ({len(packages)} packages)")
    finally:
        shutil.rmtree(tree_dir, ignore_errors=True)


if __name__ == "__main__":
    main()