From 8a30c68c95500562312e743ebba14cd9d198ec47 Mon Sep 17 00:00:00 2001 From: Dennis Thiessen Date: Fri, 21 Aug 2026 16:33:27 +0200 Subject: [PATCH] fix(scout): key decision lookups by stable job ID, not raw URL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Boards rewrite their own URLs between runs, so a decision recorded under one URL was lost the next time the same req appeared. Amazon truncates the job slug to a fixed width and that width changed, which silently resurfaced the already-skipped Principal Delivery Consultant req at score 7 in the 08-21 run. Adds _decision_key(): "#" via per-board ID patterns, falling back to a normalized path. Host-scoping is deliberate — a wrong ID match can never mark a role decided at a different company. Also collapses genuine duplicates: Workday's /apply suffix, PostFinance's per-locale URLs, Apple's ?team= and /locationPicker variants, the same NVIDIA req under two locations. The log file stays URL-keyed (it is hand-edited and the URL is the readable part); lookups go through index_decisions(). --decide now updates an existing entry when the job is already logged under a drifted URL instead of adding a second row. Verified against all 955 URLs in the decision log plus the 2026-08-21 report: 4 groups collapse, each inspected and correct, no false merges. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01WpMtWFtYkGkjSMShuTgXiB --- job_scout/scout.py | 119 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 112 insertions(+), 7 deletions(-) diff --git a/job_scout/scout.py b/job_scout/scout.py index db4b5e2..145ea68 100644 --- a/job_scout/scout.py +++ b/job_scout/scout.py @@ -1672,15 +1672,111 @@ def save_seen(seen): _atomic_write_json(STATE_FILE, seen) +# Board-specific req-ID patterns, tried in order against the URL path+query. Every key is +# scoped by host (see _decision_key), so an ID only ever has to be unique within one board +# and cross-board collisions are impossible by construction. +_JOB_ID_PATTERNS = [ + # Amazon: the slug is truncated to a fixed width and that width has changed between runs, + # so the same req yields different URLs. This is the drift that motivated the whole helper. + (r"amazon\.jobs$", r"/jobs/(\d+)"), + (r"careers\.microsoft\.com$", r"/job/(\d+)"), + (r"google\.com$", r"/results/(\d+)-"), + # Workday: trailing _REQID on the job slug (Roche 202608-120791-1, Novartis REQ-…, NVIDIA JR…). + # Also collapses the /apply suffix some rows carry and others don't. + (r"myworkdayjobs\.com$", r"_([A-Za-z]*[\d-]+\d)(?:/apply)?/?$"), + (r"smartrecruiters\.com$", r"/(\d{6,})"), + (r"greenhouse\.io$", r"/jobs/(\d+)"), + (r"careers\.cisco\.com$", r"/job/(\d+)"), + # PostFinance publishes one req per locale (…/74421-fr_FR, …/74372-de_DE) — same job. + (r"jobs\.postfinance\.ch$", r"/(\d+)-[a-z]{2}_[A-Z]{2}/?$"), + # Apple: same req appears with ?team=… and with a /locationPicker suffix. + (r"jobs\.apple\.com$", r"/details/([\d-]+)"), + (r"successfactors\.eu$", r"[?&]jobId=(\d+)"), + (r"taleo\.net$", r"[?&]job=(\d+)"), + (r"bis\.org$", r"/vacancies/(jr\d+)"), + (r"metacareers\.com$", r"/job_details/(\d+)"), + (r"finn\.no$", r"[?&]finnkode=(\d+)"), + (r"linkedin\.com$", r"/jobs/view/.*-(\d+)/?$"), + (r"careers\.roche\.com$", r"/job/([\w-]+)/"), + # Greenhouse-backed careers pages that carry the req in a query param (Databricks, + # Fivetran, Datadog, Elastic — Elastic repeats gh_jid twice; the first match wins). + (None, r"[?&]gh_jid=(\d+)"), + # Teamtailor (Axpo, Telenor) — /jobs/-. + (None, r"/jobs/(\d+)-"), + # Ashby / Lever / Recruitee-style boards (RUAG, BKW, BFH, SBB) — trailing UUID. + (None, r"/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-" + r"[0-9a-f]{4}-[0-9a-f]{12})/?$"), +] + + +def _decision_key(url): + """Stable per-job identity for the decision log, resilient to URL drift. + + Boards rewrite their own URLs: Amazon truncates the slug to a width that has changed + between runs, PostFinance publishes one URL per locale, Apple appends ?team= or + /locationPicker, Workday adds /apply. Keying decisions on the raw URL therefore loses + them — an already-skipped role silently resurfaces as undecided (this is what happened + to the AWS Principal Delivery Consultant req between the 2026-08-18 and 08-21 runs). + + Returns "#" when a board pattern matches, else "#". + Host-scoping keeps two boards that use the same numbering scheme from colliding, so a + wrong match can never mark a genuinely undecided role as decided at a *different* company. + """ + try: + parts = urllib.parse.urlsplit(url.strip()) + except ValueError: + return url.strip() + host = (parts.netloc or "").lower().split(":")[0] + if host.startswith("www."): + host = host[4:] + target = parts.path + (f"?{parts.query}" if parts.query else "") + + for host_pat, id_pat in _JOB_ID_PATTERNS: + if host_pat and not re.search(host_pat, host): + continue + m = re.search(id_pat, target) + if m: + return f"{host}#{m.group(1)}" + + # No known ID shape: fall back to the path with cosmetic variation stripped. + path = parts.path.rstrip("/") + if path.endswith("/apply"): + path = path[: -len("/apply")] + return f"{host}#{path.lower()}" + + def load_decisions(): """Decision log keyed by job URL: {url: {company, title, decision, note, date}}. Decisions persist across runs so we don't re-evaluate roles we've already judged - (shortlist / skip / applied / paused / rejected — free-text, not enforced).""" + (shortlist / skip / applied / paused / rejected — free-text, not enforced). + + The file stays URL-keyed — it is hand-edited and the URL is the useful thing to read. + Lookups go through index_decisions() instead, which is drift-proof.""" if DECISIONS_FILE.exists(): return json.loads(DECISIONS_FILE.read_text(encoding="utf-8")) return {} +def index_decisions(decisions): + """Build {stable key -> entry} for lookup. On collision the newest decision wins.""" + idx = {} + for url, entry in decisions.items(): + key = _decision_key(url) + prev = idx.get(key) + if prev is None or str(entry.get("date", "")) >= str(prev.get("date", "")): + idx[key] = entry + return idx + + +def find_decision_url(decisions, url): + """Existing log URL for the same job, if one is already recorded under a drifted URL.""" + key = _decision_key(url) + for existing in decisions: + if _decision_key(existing) == key: + return existing + return None + + def save_decisions(decisions): _atomic_write_json(DECISIONS_FILE, decisions) @@ -1750,6 +1846,8 @@ def write_stats_table(stats, total_secs): def write_report(path, results, errors, new_only, include_weak, stats=None, total_secs=0.0, decisions=None, hide_decided=False): + # `decisions` here is the stable-key index from index_decisions(), not the raw + # URL-keyed log — job URLs drift between runs, the keys do not. decisions = decisions or {} today = datetime.now().strftime("%Y-%m-%d") n_new = sum(1 for r in results if r["is_new"]) @@ -1774,7 +1872,7 @@ def write_report(path, results, errors, new_only, include_weak, stats=None, tota if not include_weak and weak: lines.append(f"\n_Hiding {len(weak)} weak/noise roles (score < 2). Use --include-weak to show._") - n_decided = sum(1 for r in results if r["url"] in decisions) + n_decided = sum(1 for r in results if _decision_key(r["url"]) in decisions) if n_decided: shown = "hidden" if hide_decided else "tagged inline" lines.append(f"_{n_decided} role(s) already in the decision log ({shown}; " @@ -1786,12 +1884,13 @@ def write_report(path, results, errors, new_only, include_weak, stats=None, tota buckets.append(("Weak / noise (score < 2)", weak)) for bucket_name, bucket in buckets: - shown = [r for r in bucket if not (hide_decided and r["url"] in decisions)] + shown = [r for r in bucket + if not (hide_decided and _decision_key(r["url"]) in decisions)] if not shown: continue lines.append(f"\n## {bucket_name} - {len(shown)} role(s)\n") for r in shown: - d = decisions.get(r["url"]) + d = decisions.get(_decision_key(r["url"])) new_tag = " [NEW]" if r["is_new"] else "" decided_tag = f" — 🗂 {d['decision'].upper()}" if d else "" loc_tag = ("CH" if r["in_ch"] else @@ -1927,7 +2026,10 @@ def main(): return url, status, note = rest[0], rest[1], " ".join(rest[2:]) decisions = load_decisions() - prev = decisions.get(url, {}) + # Update in place if this job is already logged under a drifted URL (truncated + # slug, other locale, /apply suffix) rather than adding a second row for it. + existing = find_decision_url(decisions, url) + prev = decisions.pop(existing, {}) if existing else {} # Preserve any company/title already stored (filled when a report run tagged the URL). decisions[url] = { "company": prev.get("company", ""), @@ -1935,6 +2037,8 @@ def main(): "decision": status, "note": note, "date": today, } save_decisions(decisions) + if existing and existing != url: + print(f"(replaced earlier URL for the same job: {existing})", file=sys.stderr) print(f"Recorded: {status} — {url}", file=sys.stderr) return @@ -1961,6 +2065,7 @@ def main(): seen = load_seen() decisions = load_decisions() + dec_index = index_decisions(decisions) last_scrape = load_last_scrape() all_results, errors, stats = [], [], [] new_last_scrape = dict(last_scrape) @@ -2040,7 +2145,7 @@ def main(): report_path = REPORTS_DIR / f"{today}.md" write_report(report_path, all_results, errors, new_only, include_weak, stats=stats, total_secs=total_secs, - decisions=decisions, hide_decided=hide_decided) + decisions=dec_index, hide_decided=hide_decided) # Plain data dump of every eligible role, unfiltered by keyword score — fit judgment # for these is done in conversation against the profile, not by the keyword scorer. @@ -2048,7 +2153,7 @@ def main(): dump = [{ "company": r["company"], "title": r["title"], "location": r["location"], "url": r["url"], "posted": r["posted"], "is_new": r["is_new"], - "decision": decisions.get(r["url"]), + "decision": dec_index.get(_decision_key(r["url"])), "keyword_signal": {"score": r["score"], "pos": r["pos"], "neg": r["neg"]}, "loc_flags": { "in_ch": r["in_ch"], "remote": r["remote"], "relocation": r["relocation"],