7850baf6bb
Dump stackexchange_20260331 (newest mirror), 6 sites, downloads running; streamed ingest/filter/tags pipeline; status writer + homepage strip show the layer's build progress every 5 minutes. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PDKeXvpT6tENSvyQGLV1Uq
23 lines
1.1 KiB
PowerShell
23 lines
1.1 KiB
PowerShell
# Lädt die 6 Stack-Exchange-Site-Archive vom archive.org-Mirror (resumable).
|
|
# Version: stackexchange_20260331 (neuester Mirror, geprüft 2026-07-09).
|
|
$base = "https://archive.org/download/stackexchange_20260331/stackexchange_20260331"
|
|
$dest = "C:\dev\skillfactor\knowledge\data\raw"
|
|
New-Item -ItemType Directory -Force $dest | Out-Null
|
|
$files = @(
|
|
"workplace.stackexchange.com.7z",
|
|
"pm.stackexchange.com.7z",
|
|
"law.stackexchange.com.7z",
|
|
"money.stackexchange.com.7z",
|
|
"softwareengineering.stackexchange.com.7z",
|
|
"datascience.stackexchange.com.7z"
|
|
)
|
|
$log = "C:\dev\skillfactor\docs\logs\se_download.log"
|
|
foreach ($f in $files) {
|
|
"$(Get-Date -Format 'HH:mm:ss') start $f" | Out-File $log -Append -Encoding utf8
|
|
# curl.exe: resumable (-C -), folgt Redirects (-L), still (-sS)
|
|
& curl.exe -L -sS -C - -o "$dest\$f" "$base/$f" 2>&1 | Out-File $log -Append -Encoding utf8
|
|
$size = [Math]::Round((Get-Item "$dest\$f").Length / 1MB)
|
|
"$(Get-Date -Format 'HH:mm:ss') done $f ($size MB)" | Out-File $log -Append -Encoding utf8
|
|
}
|
|
"$(Get-Date -Format 'HH:mm:ss') ALL DOWNLOADS COMPLETE" | Out-File $log -Append -Encoding utf8
|