skillfactor-pipeline/pipeline/run-status-writer.ps1

# Schreibt alle 5 Minuten den Live-Verarbeitungsstatus als JSON in die
# Gitea-Custom-Assets (sofort serviert, kein Neustart). Die Landingpage
# pollt die Datei mit Cache-Buster.
# Start (detached, persistent):
#   Start-Process powershell -WindowStyle Hidden -ArgumentList
#     '-ExecutionPolicy Bypass -File C:\dev\skillfactor\pipeline\run-status-writer.ps1'

$dev = "C:\dev\skillfactor"
$assets = "C:\Program Files (x86)\TempoBill\Zeiterfassung.Cloud\Tools\gitea\custom\public\assets"
$out = "$assets\status.json"
$outList = "$assets\processed.json"
# Startzeitpunkt des Evidence-Crawls (erste Zeile des Stage-Logs)
$started = "2026-07-08 12:39"
# Statische Slug-Map (Titel + Collar) einmalig laden
$slugMap = Get-Content "$dev\data\slug-map.json" -Raw -Encoding utf8 | ConvertFrom-Json

while ($true) {
    $sw = [System.Diagnostics.Stopwatch]::StartNew()
    try {
        # progress.json wird von der Evidence-Stage laufend ersetzt -> Lese-Retry
        $p = $null
        foreach ($attempt in 1..5) {
            try {
                $p = Get-Content "$dev\data\progress.json" -Raw -Encoding utf8 -ErrorAction Stop | ConvertFrom-Json
                break
            } catch { Start-Sleep -Seconds 2 }
        }
        if (-not $p) { throw "progress.json nicht lesbar" }
        $occ = $p.occupations.PSObject.Properties
        $done = 0; $total = 0
        $doneList = New-Object System.Collections.ArrayList
        foreach ($o in $occ) {
            $total++
            if ($o.Value.evidence -eq "done") {
                $done++
                $meta = $slugMap.($o.Name)
                [void]$doneList.Add([ordered]@{
                    slug   = $o.Name
                    title  = if ($meta) { $meta.t } else { $o.Name }
                    collar = if ($meta) { $meta.c } else { "" }
                    ads    = [int]$o.Value.ads
                })
            }
        }
        # Ollama-Extraktionen: 1 Request je extrahierte Anzeige (JSONL-Zeilen)
        $ollama = 0
        Get-ChildItem "$dev\data\evidence" -Filter "*.jsonl" -ErrorAction SilentlyContinue | ForEach-Object {
            try {
                $rd = New-Object System.IO.StreamReader($_.FullName)
                while ($null -ne $rd.ReadLine()) { $ollama++ }
                $rd.Close()
            } catch {}
        }
        $startDt = [datetime]::ParseExact($started, "yyyy-MM-dd HH:mm", $null)
        $elapsed = (Get-Date) - $startDt

        # ETA: Rate aus der eigenen Verlaufs-Historie (letzte ~6h), plus
        # fester Nachlauf fuer Voll-Re-Push + Abschlussreport (~8h)
        $histFile = "$dev\data\status-writer-history.json"
        $hist = @()
        if (Test-Path $histFile) {
            try { $hist = @(Get-Content $histFile -Raw | ConvertFrom-Json) } catch {}
        }
        $hist += [pscustomobject]@{ t = (Get-Date).ToString("o"); done = $done }
        if ($hist.Count -gt 80) { $hist = $hist[-80..-1] }
        $hist | ConvertTo-Json | Set-Content $histFile -Encoding utf8
        $eta = $null; $etaDays = $null
        $old = $hist | Where-Object { ([datetime]$_.t) -lt (Get-Date).AddHours(-2) } | Select-Object -First 1
        if (-not $old -and $hist.Count -gt 3) { $old = $hist[0] }
        if ($old -and $done -gt $old.done) {
            $hours = ((Get-Date) - [datetime]$old.t).TotalHours
            $rate = ($done - $old.done) / [Math]::Max(0.1, $hours)
            if ($rate -gt 0) {
                $remainH = ($total - $done) / $rate + 8.0
                $etaDt = (Get-Date).AddHours($remainH)
                $eta = $etaDt.ToString("yyyy-MM-dd")
                $etaDays = [Math]::Round($remainH / 24.0, 1)
            }
        }
        $sw.Stop()
        # Stack-Exchange-Wissensschicht: Fortschritt der knowledge-Pipeline
        $se = $null
        $seFile = "$dev\knowledge\data\progress-se.json"
        if (Test-Path $seFile) {
            try { $se = Get-Content $seFile -Raw -Encoding utf8 | ConvertFrom-Json } catch {}
        }
        $obj = [ordered]@{
            done        = $done
            total       = $total
            percent     = [Math]::Round(100.0 * $done / [Math]::Max(1, $total), 1)
            started_at  = $started
            elapsed_h   = [Math]::Floor($elapsed.TotalHours)
            elapsed_m   = $elapsed.Minutes
            ollama_requests = $ollama
            eta_date    = $eta
            eta_days    = $etaDays
            se_step     = $(if ($se) { $se.step } else { $null })
            se_updated  = $(if ($se) { $se.updated_at } else { $null })
            se_detail   = $(if ($se -and $se.steps -and $se.step -and $se.steps.($se.step)) {
                              ($se.steps.($se.step) | ConvertTo-Json -Compress)
                          } else { $null })
            updated_at  = (Get-Date -Format "yyyy-MM-dd HH:mm:ss")
            compute_ms  = $sw.ElapsedMilliseconds
        }
        $json = $obj | ConvertTo-Json -Compress
        [System.IO.File]::WriteAllText($out, $json, (New-Object System.Text.UTF8Encoding($false)))
        $sorted = $doneList | Sort-Object { $_.slug }
        $listObj = [ordered]@{
            updated_at = $obj.updated_at
            done       = $done
            total      = $total
            items      = @($sorted)
        }
        $jsonList = $listObj | ConvertTo-Json -Compress -Depth 4
        [System.IO.File]::WriteAllText($outList, $jsonList, (New-Object System.Text.UTF8Encoding($false)))
    } catch { }
    Start-Sleep -Seconds 300
}