7850baf6bb
Dump stackexchange_20260331 (newest mirror), 6 sites, downloads running; streamed ingest/filter/tags pipeline; status writer + homepage strip show the layer's build progress every 5 minutes. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PDKeXvpT6tENSvyQGLV1Uq
93 lines
4.1 KiB
PowerShell
93 lines
4.1 KiB
PowerShell
# Schreibt alle 5 Minuten den Live-Verarbeitungsstatus als JSON in die
|
|
# Gitea-Custom-Assets (sofort serviert, kein Neustart). Die Landingpage
|
|
# pollt die Datei mit Cache-Buster.
|
|
# Start (detached, persistent):
|
|
# Start-Process powershell -WindowStyle Hidden -ArgumentList
|
|
# '-ExecutionPolicy Bypass -File C:\dev\skillfactor\pipeline\run-status-writer.ps1'
|
|
|
|
$dev = "C:\dev\skillfactor"
|
|
$assets = "C:\Program Files (x86)\TempoBill\Zeiterfassung.Cloud\Tools\gitea\custom\public\assets"
|
|
$out = "$assets\status.json"
|
|
$outList = "$assets\processed.json"
|
|
# Startzeitpunkt des Evidence-Crawls (erste Zeile des Stage-Logs)
|
|
$started = "2026-07-08 12:39"
|
|
# Statische Slug-Map (Titel + Collar) einmalig laden
|
|
$slugMap = Get-Content "$dev\data\slug-map.json" -Raw -Encoding utf8 | ConvertFrom-Json
|
|
|
|
while ($true) {
|
|
$sw = [System.Diagnostics.Stopwatch]::StartNew()
|
|
try {
|
|
# progress.json wird von der Evidence-Stage laufend ersetzt -> Lese-Retry
|
|
$p = $null
|
|
foreach ($attempt in 1..5) {
|
|
try {
|
|
$p = Get-Content "$dev\data\progress.json" -Raw -Encoding utf8 -ErrorAction Stop | ConvertFrom-Json
|
|
break
|
|
} catch { Start-Sleep -Seconds 2 }
|
|
}
|
|
if (-not $p) { throw "progress.json nicht lesbar" }
|
|
$occ = $p.occupations.PSObject.Properties
|
|
$done = 0; $total = 0
|
|
$doneList = New-Object System.Collections.ArrayList
|
|
foreach ($o in $occ) {
|
|
$total++
|
|
if ($o.Value.evidence -eq "done") {
|
|
$done++
|
|
$meta = $slugMap.($o.Name)
|
|
[void]$doneList.Add([ordered]@{
|
|
slug = $o.Name
|
|
title = if ($meta) { $meta.t } else { $o.Name }
|
|
collar = if ($meta) { $meta.c } else { "" }
|
|
ads = [int]$o.Value.ads
|
|
})
|
|
}
|
|
}
|
|
# Ollama-Extraktionen: 1 Request je extrahierte Anzeige (JSONL-Zeilen)
|
|
$ollama = 0
|
|
Get-ChildItem "$dev\data\evidence" -Filter "*.jsonl" -ErrorAction SilentlyContinue | ForEach-Object {
|
|
try {
|
|
$rd = New-Object System.IO.StreamReader($_.FullName)
|
|
while ($null -ne $rd.ReadLine()) { $ollama++ }
|
|
$rd.Close()
|
|
} catch {}
|
|
}
|
|
$startDt = [datetime]::ParseExact($started, "yyyy-MM-dd HH:mm", $null)
|
|
$elapsed = (Get-Date) - $startDt
|
|
$sw.Stop()
|
|
# Stack-Exchange-Wissensschicht: Fortschritt der knowledge-Pipeline
|
|
$se = $null
|
|
$seFile = "$dev\knowledge\data\progress-se.json"
|
|
if (Test-Path $seFile) {
|
|
try { $se = Get-Content $seFile -Raw -Encoding utf8 | ConvertFrom-Json } catch {}
|
|
}
|
|
$obj = [ordered]@{
|
|
done = $done
|
|
total = $total
|
|
percent = [Math]::Round(100.0 * $done / [Math]::Max(1, $total), 1)
|
|
started_at = $started
|
|
elapsed_h = [Math]::Floor($elapsed.TotalHours)
|
|
elapsed_m = $elapsed.Minutes
|
|
ollama_requests = $ollama
|
|
se_step = $(if ($se) { $se.step } else { $null })
|
|
se_updated = $(if ($se) { $se.updated_at } else { $null })
|
|
se_detail = $(if ($se -and $se.steps -and $se.step -and $se.steps.($se.step)) {
|
|
($se.steps.($se.step) | ConvertTo-Json -Compress)
|
|
} else { $null })
|
|
updated_at = (Get-Date -Format "yyyy-MM-dd HH:mm:ss")
|
|
compute_ms = $sw.ElapsedMilliseconds
|
|
}
|
|
$json = $obj | ConvertTo-Json -Compress
|
|
[System.IO.File]::WriteAllText($out, $json, (New-Object System.Text.UTF8Encoding($false)))
|
|
$sorted = $doneList | Sort-Object { $_.slug }
|
|
$listObj = [ordered]@{
|
|
updated_at = $obj.updated_at
|
|
done = $done
|
|
total = $total
|
|
items = @($sorted)
|
|
}
|
|
$jsonList = $listObj | ConvertTo-Json -Compress -Depth 4
|
|
[System.IO.File]::WriteAllText($outList, $jsonList, (New-Object System.Text.UTF8Encoding($false)))
|
|
} catch { }
|
|
Start-Sleep -Seconds 300
|
|
}
|