diff --git a/FIX_PROGRESS.md b/FIX_PROGRESS.md index 3f6ed34..cfb13df 100644 --- a/FIX_PROGRESS.md +++ b/FIX_PROGRESS.md @@ -6,12 +6,13 @@ Do not rewrite submitted application archives or invent missing evidence. ## Resume point -F01–F14 complete. **Next: F15** — typed dates and retained contact/compensation metadata -(`job_scout/scout.py`: the Taleo parser writes a deadline into `posted`, and the Prospective -path's unit/Lohnklasse/contact are folded into `description`, which `_process_company` discards). +F01–F16 complete. **Next: F17** — reproducible builds and safe historical-archive +classification. Start from the 13 older packages listed in the F05/F06 verification notes +below; they carry genuine findings under today's rules and must NOT be silently rewritten. +F17 also needs the 45-document LaTeX sweep, which has not been re-run this session. Read this file and inspect `git log`/`git diff` before resuming; do not repeat completed work. -Checkpoints: `afdef7e` (F01+F02) · `fc5f3e4` (F03–F06) · `ce5b7f5` (F07) · `3519b2f` (F08–F10) · `afd8174` (F11–F13) · plus F14. +Checkpoints: `afdef7e` (F01+F02) · `fc5f3e4` (F03–F06) · `ce5b7f5` (F07) · `3519b2f` (F08–F10) · `afd8174` (F11–F13) · `ce8619b` (F14) · plus F15–F16. ## User rulings — 2026-09-11 @@ -40,8 +41,8 @@ Checkpoints: `afdef7e` (F01+F02) · `fc5f3e4` (F03–F06) · `ce5b7f5` (F07) · | F12 | DONE | Whole-word place matching (Bernau no longer reads as Bern); ~25 Swiss towns incl. Thun and Zimmerwald; Danish cities; pan-regional scope no longer beats an explicit onsite/one-country restriction. All 14 review cases correct | | F13 | DONE | Identifying query params kept in the key; read and write paths share `lookup_decision`. **Live run recovered 7 reposted RUAG decisions**, labelled RELATED rather than applied as identical | | F14 | DONE | Buckets renamed to keyword relevance with an explicit 'NOT a fit judgement'; German titles and platform/DevOps vocabulary added. Kdo Cy *ICT-System Engineer (DevSecOps/Platform Automation)* went 0→10; *Senior Dateningenieur* 1→3, level with its English twin | -| F15 | TODO | Typed dates and retained contact/compensation metadata | -| F16 | TODO | Run-scoped reports, transactional persistence and scan coverage | +| F15 | DONE | Taleo deadline moved out of `posted` (a closing date was making roles look freshly published); `unit`, `compensation`, `contact` and `date_provenance` carried as fields through to the report and JSON. Named contacts now surface — directly serving the channel rule | +| F16 | DONE | Restricted runs write `.only-x.md` and leave the daily report intact; seen-state committed only after the report is on disk, and never by a restricted run; unique temp names for atomic writes | | F17 | TODO | Reproducible new builds and safe historical archive classification | | F18 | TODO | Evidence earlier on page one; rendered skills budget | | F19 | TODO | Education layout, phrasing guidance, reproducible word counts | @@ -90,6 +91,11 @@ mid-remediation — it would destroy the ability to attribute a regression to a - F14 note: the score floor on RUAG/SBB/BKW/PostFinance/Kdo Cy was a workaround for missing German vocabulary. The vocabulary is now the fix; the floors can probably be reduced later, but they were left in place to keep this diff attributable. +- F15/F16 validated on a live 37-company run: named contacts with phone numbers now appear in + report rows, NATO/Taleo closing dates render as Bewerbungsfrist rather than Posted, and + `--only=kdocy` wrote `2026-09-11.only-kdocy.md` while leaving the full report untouched. +- F16 context: the daily-report clobber is not hypothetical. It happened twice in this session + (2026-09-10 and 09-11) and both times the full run had to be repeated by hand. - Not re-run this session: the 45-document LaTeX build sweep and the visual sampling. ## Completion rules diff --git a/job_scout/scout.py b/job_scout/scout.py index 606cbe3..d90ff8e 100644 --- a/job_scout/scout.py +++ b/job_scout/scout.py @@ -38,6 +38,7 @@ import os import re import ssl import sys +import tempfile import time from concurrent.futures import ThreadPoolExecutor, as_completed from functools import lru_cache @@ -1172,15 +1173,19 @@ def _parse_taleo_jobsearch_html(html, host, career_section): deadline = _unq(m.group(5)) body = _unq(m.group(6)) grade = _unq(m.group(7)) - # List view exposes application deadline only (not a posted-on date). We store - # the date part in `posted` so the report surfaces the closing date. + # List view exposes an application DEADLINE only, never a posted-on date. It used to + # be stored in `posted`, which made a role closing soon look freshly published - + # backwards for a freshness signal (F15). It now goes to `deadline`, and `posted` + # stays empty because this board genuinely does not publish one. deadline_short = deadline.split(",")[0].strip() if deadline else "" jobs.append({ "id": job_no, "title": title, "location": loc, "url": f"{detail_base}?job={job_no}&lang=en", - "posted": deadline_short, + "posted": "", + "deadline": deadline_short, + "date_provenance": "taleo list view exposes a closing date only; no publication date", "description": f"{body} {grade}".strip()[:500], }) if jobs: @@ -1349,6 +1354,13 @@ def fetch_prospective(args): "url": (j.get("links") or {}).get("directlink", ""), "posted": (j.get("start_date") or "")[:10], "deadline": deadline, + "date_provenance": "prospective start_date = publication; end_date = application deadline", + # Carried as fields, not just folded into description: these decide level, pay and + # whether a cold submit is even necessary (F15). + "unit": unit, + "compensation": f"Lohnklasse {lohnklasse}" if lohnklasse else "", + "contact": re.sub(r"\s+", " ", html.unescape( + re.sub(r"<[^>]+>", " ", str(szas.get("sza_contact", ""))))).strip()[:300], "description": re.sub(r"\s+", " ", desc)[:1200], }) return jobs @@ -1871,9 +1883,17 @@ def score_job(job, title_only=False): def _atomic_write_json(path, data): """Write JSON via temp + os.replace so a crash mid-write cannot truncate state files.""" path.parent.mkdir(parents=True, exist_ok=True) - tmp = path.with_suffix(path.suffix + ".tmp") - tmp.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8") - os.replace(tmp, path) + # Unique temp name (F16): a fixed ".tmp" let two concurrent writers share one temp file + # and os.replace the wrong content into place. + descriptor, tmp_name = tempfile.mkstemp(dir=path.parent, prefix=path.name + ".", suffix=".tmp") + tmp = Path(tmp_name) + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as stream: + stream.write(json.dumps(data, indent=2, ensure_ascii=False)) + os.replace(tmp, path) + except BaseException: + tmp.unlink(missing_ok=True) + raise def load_seen(): @@ -2185,6 +2205,14 @@ def write_report(path, results, errors, new_only, include_weak, stats=None, tota lines.append(f"- Posted: {r['posted']}") if r.get("deadline"): lines.append(f"- **Bewerbungsfrist: {r['deadline']}**") + if r.get("unit"): + lines.append(f"- Einheit: {r['unit']}") + if r.get("compensation"): + lines.append(f"- Vergütung: {r['compensation']}") + if r.get("contact"): + # Channel is the rationed variable: a published, reachable name means a cold + # submit is a choice rather than the only option. + lines.append(f"- **Ansprechperson: {r['contact']}**") lines.append(f"- URL: {r['url']}") if d: note = f" — {d['note']}" if d.get("note") else "" @@ -2332,6 +2360,10 @@ def _process_company(cid, display, args, jobs, seen, today, last_scrape): "title": j["title"], "location": j["location"], "url": j["url"], "posted": j.get("posted", ""), "deadline": j.get("deadline", ""), + "date_provenance": j.get("date_provenance", ""), + "unit": j.get("unit", ""), + "compensation": j.get("compensation", ""), + "contact": j.get("contact", ""), "score": score, "pos": pos, "neg": neg, "in_ch": in_ch, "remote": is_remote, "relocation": is_relocation, "is_new": is_new, @@ -2469,7 +2501,9 @@ def main(): "scraped": stat["scraped"], "date": today, "display": display, } finally: - save_seen(seen) + # Seen-state is NOT committed here (F16). It used to be, which meant a failed report + # write left roles marked as seen and a `--new-only` retry hid exactly the hits that + # were never delivered. It is committed after the report is on disk. save_last_scrape(new_last_scrape) _close_browser() @@ -2481,17 +2515,33 @@ def main(): all_results.sort(key=lambda r: (-r["score"], r["company"], r["title"])) REPORTS_DIR.mkdir(parents=True, exist_ok=True) - report_path = REPORTS_DIR / f"{today}.md" + # Run scope in the filename (F16). A partial run used to overwrite the full daily report: + # `--only=kdocy` replaced a 37-company report with seven rows, and the full run had to be + # repeated by hand. Only an unrestricted run owns the plain `` name. + scope = [] + if only: + scope.append(f"only-{only}") + if new_only: + scope.append("new-only") + if hide_decided: + scope.append("undecided") + stem = today if not scope else f"{today}.{'.'.join(scope)}" + if scope: + print(f"Restricted run: writing {stem}.md/.json, leaving the full daily report intact.", + file=sys.stderr) + report_path = REPORTS_DIR / f"{stem}.md" write_report(report_path, all_results, errors, new_only, include_weak, stats=stats, total_secs=total_secs, decisions=dec_index, hide_decided=hide_decided) # Plain data dump of every eligible role, unfiltered by keyword score — fit judgment # for these is done in conversation against the profile, not by the keyword scorer. - json_path = REPORTS_DIR / f"{today}.json" + json_path = REPORTS_DIR / f"{stem}.json" dump = [{ "company": r["company"], "title": r["title"], "location": r["location"], "url": r["url"], "posted": r["posted"], "deadline": r.get("deadline", ""), + "date_provenance": r.get("date_provenance", ""), "unit": r.get("unit", ""), + "compensation": r.get("compensation", ""), "contact": r.get("contact", ""), "is_new": r["is_new"], "decision": lookup_decision(dec_index, decisions, r["url"]), "keyword_signal": {"score": r["score"], "pos": r["pos"], "neg": r["neg"]}, @@ -2501,6 +2551,14 @@ def main(): } for r in all_results] _atomic_write_json(json_path, dump) + # Report and dump are on disk, so the run genuinely delivered: now the roles may be + # marked seen. A restricted run must not mark anything seen at all - it did not look at + # the whole board, and doing so would hide those roles from the next full run. + if scope: + print("Restricted run: seen-state deliberately NOT updated.", file=sys.stderr) + else: + save_seen(seen) + n_new = sum(1 for r in all_results if r["is_new"]) print(f"\nReport written: {report_path}", file=sys.stderr) print(f"Total matches: {len(all_results)} ({n_new} new) | "