69a7ba6ea7
lint_package.py checks contamination, alphabetical truncation, count consistency, provenance completeness and intro grammar; wired into both publish paths (p4_publish + batch stage_packages), SKIP_LINT=1 reserved for labeled v1-grade republishes. Heatmap over 3,039 v1 packages: 1,406 clean / 1,633 with findings; C2 truncation is systemic (generator), contamination affects 2.6% of crawled packages; nothing requires re-fetching. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PDKeXvpT6tENSvyQGLV1Uq
68 lines
3.0 KiB
PowerShell
68 lines
3.0 KiB
PowerShell
# Stufe 2 der Automatik: wartet auf das Ende der Enrichment-Nacht
|
|
# (run-enrichment-night.ps1), zieht dann fuer JEDEN Beruf Jobanzeigen
|
|
# (JSearch, Budget-Kappe 33k in progress.spend_request), extrahiert via
|
|
# Ollama gemma3:27b, aggregiert Markt-Evidence in die Pakete, aktualisiert
|
|
# Provenance, re-pusht den Katalog und erzeugt die Abschluss-Auswertung.
|
|
#
|
|
# Start (detached):
|
|
# Start-Process powershell -WindowStyle Hidden -ArgumentList
|
|
# '-ExecutionPolicy Bypass -File C:\dev\skillfactor\pipeline\run-evidence-stage.ps1'
|
|
|
|
$ErrorActionPreference = "Continue"
|
|
$dev = "C:\dev\skillfactor"
|
|
$py = "$dev\.venv\Scripts\python.exe"
|
|
$nlog = "$dev\docs\logs\enrichment_night.log"
|
|
$log = "$dev\docs\logs\evidence_stage.log"
|
|
|
|
function Log($m) {
|
|
"$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss') $m" | Out-File $log -Append -Encoding utf8
|
|
}
|
|
|
|
Log "=== evidence stage orchestrator started ==="
|
|
|
|
# ── 1. Auf Abschluss der Enrichment-Nacht warten ─────────────────────────────
|
|
Log "waiting for enrichment night run to complete..."
|
|
while ($true) {
|
|
if ((Test-Path $nlog) -and
|
|
(Select-String -Path $nlog -Pattern "enrichment night run COMPLETE" -Quiet)) { break }
|
|
Start-Sleep -Seconds 120
|
|
}
|
|
Log "enrichment night complete - starting evidence stage."
|
|
|
|
# ── 2. Evidence fuer ALLE Berufe (resumable; restartet bei Crash) ────────────
|
|
Set-Location $dev
|
|
$attempt = 0
|
|
while ($true) {
|
|
$attempt++
|
|
Log "evidence run attempt $attempt..."
|
|
cmd /c "`"$py`" -u pipeline\batch_run.py --stage evidence >> `"$log`" 2>&1"
|
|
$code = $LASTEXITCODE
|
|
Log "batch_run evidence exit=$code"
|
|
if ($code -eq 2) { Log "JSearch budget exhausted - stopping fetch loop."; break }
|
|
if ($code -eq 0) { break }
|
|
if ($attempt -ge 200) { Log "WARN: 200 attempts reached - stopping."; break }
|
|
Start-Sleep -Seconds 30
|
|
}
|
|
|
|
# ── 3. Provenance-Rebuild (jobads-Segmente in den Pies) ──────────────────────
|
|
cmd /c "`"$py`" pipeline\p3d_provenance.py >> `"$log`" 2>&1"
|
|
Log "p3d_provenance done (exit $LASTEXITCODE)"
|
|
|
|
# ── 4. Katalog re-pushen (Markt-Evidence in den Repos) ───────────────────────
|
|
# SKIP_LINT=1: dieser Refresh publiziert bewusst v1-Qualitaet (Heatmap kennt
|
|
# den Zustand); Referenz-Qualitaet entsteht paketweise ueber das Lint-Gate.
|
|
$env:SKIP_LINT = "1"
|
|
cmd /c "`"$py`" pipeline\reset_push_status.py >> `"$log`" 2>&1"
|
|
Log "reset_push_status done"
|
|
& powershell -NonInteractive -ExecutionPolicy Bypass -File "$dev\push_loop.ps1" *>> $log
|
|
Log "re-push finished (SKIP_LINT=1, v1-grade republish)"
|
|
$env:SKIP_LINT = ""
|
|
|
|
# ── 5. Marketplace + Abschluss-Auswertung ────────────────────────────────────
|
|
cmd /c "`"$py`" pipeline\push_marketplace.py >> `"$log`" 2>&1"
|
|
Log "marketplace pushed"
|
|
cmd /c "`"$py`" pipeline\evidence_report.py --push >> `"$log`" 2>&1"
|
|
Log "evidence report generated + pushed"
|
|
|
|
Log "=== evidence stage orchestrator COMPLETE ==="
|