af05383bb1
Snapshot before the quality program (relevance gates, tiered mapping, QA linter). v1 is the immutable before/after reference; evidence crawl was at ~175/3039 occupations when tagged. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PDKeXvpT6tENSvyQGLV1Uq
74 lines
2.5 KiB
Python
74 lines
2.5 KiB
Python
"""Resume + budget bookkeeping for the full-catalog batch runs.
|
|
|
|
Single source of truth: data/progress.json
|
|
{
|
|
"jsearch_requests_total": 123, # lifetime counter, hard cap below
|
|
"occupations": {
|
|
"<slug>": {
|
|
"package": "done|pending", # phase 1 (deterministic generator)
|
|
"evidence": "done|fetching|extracting|aggregating|failed|pending",
|
|
"depth": "done|in_progress|failed|pending", # phases 2b/3
|
|
"tier": 1|2|3, # 1 = top-200, 3 = night batch rest
|
|
"requests": 7, # JSearch requests spent on this slug
|
|
"ads": 61, # unique ads with full text
|
|
"error": "last error message" # only present after a failure
|
|
}, ...
|
|
}
|
|
}
|
|
|
|
Writes are atomic (tmp + os.replace) so an aborted run never corrupts the
|
|
file. Every JSearch request MUST go through spend_request() — it enforces
|
|
the hard budget. Cache hits cost nothing and are not counted.
|
|
"""
|
|
import json
|
|
import os
|
|
|
|
BASE = os.path.join(os.path.dirname(__file__), "..")
|
|
PROGRESS = os.path.join(BASE, "data", "progress.json")
|
|
|
|
JSEARCH_BUDGET_TOTAL = 33000 # hard cap per finalize order (skillfactor_finalize.md)
|
|
|
|
|
|
class BudgetExhausted(RuntimeError):
|
|
pass
|
|
|
|
|
|
def load():
|
|
if os.path.exists(PROGRESS):
|
|
with open(PROGRESS, encoding="utf-8-sig") as f: # utf-8-sig strips BOM if present
|
|
return json.load(f)
|
|
return {"jsearch_requests_total": 0, "occupations": {}}
|
|
|
|
|
|
def save(state):
|
|
import time as _time
|
|
tmp = PROGRESS + ".tmp"
|
|
with open(tmp, "w", encoding="utf-8") as f:
|
|
json.dump(state, f, ensure_ascii=False, indent=1)
|
|
for attempt in range(20):
|
|
try:
|
|
os.replace(tmp, PROGRESS)
|
|
return
|
|
except PermissionError:
|
|
_time.sleep(0.1)
|
|
os.replace(tmp, PROGRESS) # final attempt
|
|
|
|
|
|
def occ(state, slug):
|
|
return state["occupations"].setdefault(slug, {
|
|
"package": "pending", "evidence": "pending", "depth": "pending",
|
|
"tier": 3, "requests": 0, "ads": 0,
|
|
})
|
|
|
|
|
|
def spend_request(state, slug, n=1):
|
|
"""Count n JSearch requests against the global and per-slug budget.
|
|
Call BEFORE the HTTP request; raises BudgetExhausted at the cap."""
|
|
if state["jsearch_requests_total"] + n > JSEARCH_BUDGET_TOTAL:
|
|
raise BudgetExhausted(
|
|
f"JSearch budget {JSEARCH_BUDGET_TOTAL} reached "
|
|
f"({state['jsearch_requests_total']} spent)")
|
|
state["jsearch_requests_total"] += n
|
|
occ(state, slug)["requests"] += n
|
|
save(state)
|