fix(F15-F16): stop losing dates, contacts and whole daily reports
F15 - dates and metadata kept their meaning - The Taleo parser stored an application DEADLINE in `posted`, so a role closing on 30 September looked as though it had been published then - backwards for a freshness signal. It now populates `deadline`, and `posted` stays empty because that board genuinely publishes none. - The Prospective adapter read unit, Lohnklasse and a named contact, folded them into `description` for scoring, and `_process_company` then discarded `description` - losing exactly the fields that decide level, pay and whether a cold submit is necessary. They are now carried as fields through to the report and the JSON dump, alongside `date_provenance`. - Named contacts now appear in report rows. Channel is the rationed variable since 2026-09-05, so a published, reachable name is the difference between a cold submit being a choice and being the only option. F16 - writes stopped destroying delivered work - Every run of a day wrote `<date>.md`/`.json`, so a restricted run replaced the full daily report. This is not hypothetical: it happened twice in this session and both times the 37-company run had to be repeated by hand. Restricted runs now write `<date>.only-kdocy.md` and say so. - `save_seen()` ran in a `finally:` BEFORE the report was written. A failed report write therefore left roles marked seen, so a `--new-only` retry would hide precisely the hits that were never delivered. Seen-state is now committed only after report and dump are on disk - and never by a restricted run, which did not look at the whole board. - Atomic writes used a fixed `.tmp` name; two writers could share it and os.replace the wrong content into place. Unique temp names now. 34/34 tests pass; validated on a live 37-company run. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Qej5SBvhcbGXHwLqNWiskM
This commit is contained in:
dennisthiessenandClaude Opus 5 committed
1 parent
ce8619b8e3
commit
c7e8d4e058
2 files changed
+79
-15
No files matched your search
+67
-9
@@ -38,6 +38,7 @@ import os
|
||||
import re
|
||||
import ssl
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from functools import lru_cache
|
||||
@@ -1172,15 +1173,19 @@ def _parse_taleo_jobsearch_html(html, host, career_section):
|
||||
deadline = _unq(m.group(5))
|
||||
body = _unq(m.group(6))
|
||||
grade = _unq(m.group(7))
|
||||
# List view exposes application deadline only (not a posted-on date). We store
|
||||
# the date part in `posted` so the report surfaces the closing date.
|
||||
# List view exposes an application DEADLINE only, never a posted-on date. It used to
|
||||
# be stored in `posted`, which made a role closing soon look freshly published -
|
||||
# backwards for a freshness signal (F15). It now goes to `deadline`, and `posted`
|
||||
# stays empty because this board genuinely does not publish one.
|
||||
deadline_short = deadline.split(",")[0].strip() if deadline else ""
|
||||
jobs.append({
|
||||
"id": job_no,
|
||||
"title": title,
|
||||
"location": loc,
|
||||
"url": f"{detail_base}?job={job_no}&lang=en",
|
||||
"posted": deadline_short,
|
||||
"posted": "",
|
||||
"deadline": deadline_short,
|
||||
"date_provenance": "taleo list view exposes a closing date only; no publication date",
|
||||
"description": f"{body} {grade}".strip()[:500],
|
||||
})
|
||||
if jobs:
|
||||
@@ -1349,6 +1354,13 @@ def fetch_prospective(args):
|
||||
"url": (j.get("links") or {}).get("directlink", ""),
|
||||
"posted": (j.get("start_date") or "")[:10],
|
||||
"deadline": deadline,
|
||||
"date_provenance": "prospective start_date = publication; end_date = application deadline",
|
||||
# Carried as fields, not just folded into description: these decide level, pay and
|
||||
# whether a cold submit is even necessary (F15).
|
||||
"unit": unit,
|
||||
"compensation": f"Lohnklasse {lohnklasse}" if lohnklasse else "",
|
||||
"contact": re.sub(r"\s+", " ", html.unescape(
|
||||
re.sub(r"<[^>]+>", " ", str(szas.get("sza_contact", ""))))).strip()[:300],
|
||||
"description": re.sub(r"\s+", " ", desc)[:1200],
|
||||
})
|
||||
return jobs
|
||||
@@ -1871,9 +1883,17 @@ def score_job(job, title_only=False):
|
||||
def _atomic_write_json(path, data):
|
||||
"""Write JSON via temp + os.replace so a crash mid-write cannot truncate state files."""
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = path.with_suffix(path.suffix + ".tmp")
|
||||
tmp.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
os.replace(tmp, path)
|
||||
# Unique temp name (F16): a fixed ".tmp" let two concurrent writers share one temp file
|
||||
# and os.replace the wrong content into place.
|
||||
descriptor, tmp_name = tempfile.mkstemp(dir=path.parent, prefix=path.name + ".", suffix=".tmp")
|
||||
tmp = Path(tmp_name)
|
||||
try:
|
||||
with os.fdopen(descriptor, "w", encoding="utf-8") as stream:
|
||||
stream.write(json.dumps(data, indent=2, ensure_ascii=False))
|
||||
os.replace(tmp, path)
|
||||
except BaseException:
|
||||
tmp.unlink(missing_ok=True)
|
||||
raise
|
||||
|
||||
|
||||
def load_seen():
|
||||
@@ -2185,6 +2205,14 @@ def write_report(path, results, errors, new_only, include_weak, stats=None, tota
|
||||
lines.append(f"- Posted: {r['posted']}")
|
||||
if r.get("deadline"):
|
||||
lines.append(f"- **Bewerbungsfrist: {r['deadline']}**")
|
||||
if r.get("unit"):
|
||||
lines.append(f"- Einheit: {r['unit']}")
|
||||
if r.get("compensation"):
|
||||
lines.append(f"- Vergütung: {r['compensation']}")
|
||||
if r.get("contact"):
|
||||
# Channel is the rationed variable: a published, reachable name means a cold
|
||||
# submit is a choice rather than the only option.
|
||||
lines.append(f"- **Ansprechperson: {r['contact']}**")
|
||||
lines.append(f"- URL: {r['url']}")
|
||||
if d:
|
||||
note = f" — {d['note']}" if d.get("note") else ""
|
||||
@@ -2332,6 +2360,10 @@ def _process_company(cid, display, args, jobs, seen, today, last_scrape):
|
||||
"title": j["title"], "location": j["location"],
|
||||
"url": j["url"], "posted": j.get("posted", ""),
|
||||
"deadline": j.get("deadline", ""),
|
||||
"date_provenance": j.get("date_provenance", ""),
|
||||
"unit": j.get("unit", ""),
|
||||
"compensation": j.get("compensation", ""),
|
||||
"contact": j.get("contact", ""),
|
||||
"score": score, "pos": pos, "neg": neg,
|
||||
"in_ch": in_ch, "remote": is_remote, "relocation": is_relocation,
|
||||
"is_new": is_new,
|
||||
@@ -2469,7 +2501,9 @@ def main():
|
||||
"scraped": stat["scraped"], "date": today, "display": display,
|
||||
}
|
||||
finally:
|
||||
save_seen(seen)
|
||||
# Seen-state is NOT committed here (F16). It used to be, which meant a failed report
|
||||
# write left roles marked as seen and a `--new-only` retry hid exactly the hits that
|
||||
# were never delivered. It is committed after the report is on disk.
|
||||
save_last_scrape(new_last_scrape)
|
||||
_close_browser()
|
||||
|
||||
@@ -2481,17 +2515,33 @@ def main():
|
||||
all_results.sort(key=lambda r: (-r["score"], r["company"], r["title"]))
|
||||
|
||||
REPORTS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
report_path = REPORTS_DIR / f"{today}.md"
|
||||
# Run scope in the filename (F16). A partial run used to overwrite the full daily report:
|
||||
# `--only=kdocy` replaced a 37-company report with seven rows, and the full run had to be
|
||||
# repeated by hand. Only an unrestricted run owns the plain `<date>` name.
|
||||
scope = []
|
||||
if only:
|
||||
scope.append(f"only-{only}")
|
||||
if new_only:
|
||||
scope.append("new-only")
|
||||
if hide_decided:
|
||||
scope.append("undecided")
|
||||
stem = today if not scope else f"{today}.{'.'.join(scope)}"
|
||||
if scope:
|
||||
print(f"Restricted run: writing {stem}.md/.json, leaving the full daily report intact.",
|
||||
file=sys.stderr)
|
||||
report_path = REPORTS_DIR / f"{stem}.md"
|
||||
write_report(report_path, all_results, errors, new_only, include_weak,
|
||||
stats=stats, total_secs=total_secs,
|
||||
decisions=dec_index, hide_decided=hide_decided)
|
||||
|
||||
# Plain data dump of every eligible role, unfiltered by keyword score — fit judgment
|
||||
# for these is done in conversation against the profile, not by the keyword scorer.
|
||||
json_path = REPORTS_DIR / f"{today}.json"
|
||||
json_path = REPORTS_DIR / f"{stem}.json"
|
||||
dump = [{
|
||||
"company": r["company"], "title": r["title"], "location": r["location"],
|
||||
"url": r["url"], "posted": r["posted"], "deadline": r.get("deadline", ""),
|
||||
"date_provenance": r.get("date_provenance", ""), "unit": r.get("unit", ""),
|
||||
"compensation": r.get("compensation", ""), "contact": r.get("contact", ""),
|
||||
"is_new": r["is_new"],
|
||||
"decision": lookup_decision(dec_index, decisions, r["url"]),
|
||||
"keyword_signal": {"score": r["score"], "pos": r["pos"], "neg": r["neg"]},
|
||||
@@ -2501,6 +2551,14 @@ def main():
|
||||
} for r in all_results]
|
||||
_atomic_write_json(json_path, dump)
|
||||
|
||||
# Report and dump are on disk, so the run genuinely delivered: now the roles may be
|
||||
# marked seen. A restricted run must not mark anything seen at all - it did not look at
|
||||
# the whole board, and doing so would hide those roles from the next full run.
|
||||
if scope:
|
||||
print("Restricted run: seen-state deliberately NOT updated.", file=sys.stderr)
|
||||
else:
|
||||
save_seen(seen)
|
||||
|
||||
n_new = sum(1 for r in all_results if r["is_new"])
|
||||
print(f"\nReport written: {report_path}", file=sys.stderr)
|
||||
print(f"Total matches: {len(all_results)} ({n_new} new) | "
|
||||
|
||||
Reference in new issue
Block a user