From afd8174bbdb310084b6c1c48cdc76a130f5abbab Mon Sep 17 00:00:00 2001
From: Dennis Thiessen <dennis@thiessen.io>
Date: Fri, 11 Sep 2026 19:50:17 +0200
Subject: [PATCH] fix(F11-F13): scout selection bugs that dropped roles before
 any judgement

These three all cost roles BEFORE a human ever read them, which is the
expensive kind of defect: a job that never reaches the report cannot be
evaluated at all.

F11 - dedupe by requisition, not title
- The old rule collapsed everything sharing a normalised title within one
  employer. Two different reqs called "Senior Data Engineer" in Bern and Zurich
  became one row, and different teams hiring the same title vanished.
- Requisition identity is now the key, parsed from the URL with the same
  patterns the decision log uses, employer-scoped so two boards cannot collide.
- The legitimate case the old rule existed for - one role republished per
  location - is a SAME-REQUISITION case, so those merge their locations.
- A shared title across different requisitions is now a warning, never a
  deletion.

F12 - location filters
- Substring matching accepted "Bernau bei Berlin" as Bern. Place matching is
  now whole-word.
- Thun, Zimmerwald, Baar, Olten and ~25 more Swiss towns were rejected outright
  when the adapter supplied no country. Zimmerwald is the Kdo Cy site he applied
  to; Thun is where he lives.
- Aarhus, Odense and other Danish cities added alongside Copenhagen.
- A stray "Europe" no longer implies remote eligibility next to an explicit
  onsite marker or a single-country residence restriction.
- All 14 cases named in the review now resolve correctly.

F13 - decision identity
- The generic fallback key dropped the query string, so /vacancy?job=1 and
  ?job=2 were the same key and one decision silently covered both. Identifying
  query parameters are now kept; tracking noise is not.
- The READ path had no slug fallback although the WRITE path got one on
  2026-09-04. Every RUAG repost therefore came back undecided despite a recorded
  decision - which is what happened to Lead-Architekt Network- und Richtfunk on
  2026-09-10, days after 63 RUAG rule-outs were written.
- Both paths now share `lookup_decision`. Verified on a live RUAG run: 7
  reposted requisitions recovered their decisions.
- A slug match is reported as RELATED with the matched URL, never applied as if
  it were the same requisition: a repost can differ materially, as RUAG
  demonstrated when it moved a disqualifier into a job title.

tests/test_scout_matching.py added. Full suite 34/34 pass.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Qej5SBvhcbGXHwLqNWiskM
---
 job_scout/scout.py           | 159 +++++++++++++++++++++++++++++++----
 tests/test_scout_matching.py |  65 ++++++++++++++
 2 files changed, 207 insertions(+), 17 deletions(-)
 create mode 100644 tests/test_scout_matching.py

diff --git a/job_scout/scout.py b/job_scout/scout.py
index 1039d9d..db6c9a3 100644
--- a/job_scout/scout.py
+++ b/job_scout/scout.py
@@ -58,10 +58,19 @@ API_FETCH_WORKERS = 8
 CH_LOCATION_KEYWORDS = [
     "switzerland", "zurich", "zürich", "basel", "bern", "geneva", "genf",
     "lausanne", "zug", "rüschlikon", "stäfa", "schweiz", "suisse",
+    # Added 2026-09-11 (F12). These were being rejected outright when the adapter gave no
+    # country: Zimmerwald is the Kdo Cy site he applied to, Thun is where he lives.
+    "thun", "zimmerwald", "baar", "olten", "winterthur", "luzern", "lucerne",
+    "st. gallen", "sankt gallen", "aarau", "baden", "emmen", "duebendorf", "dübendorf",
+    "frauenfeld", "biel", "bienne", "solothurn", "chur", "sion", "neuchâtel", "neuchatel",
+    "fribourg", "freiburg", "lugano", "bellinzona", "kloten", "wallisellen", "opfikon",
+    "lupfig", "villigen", "rotkreuz", "allschwil", "kaiseraugst", "payerne", "kastellaun",
 ]
 
 DK_LOCATION_KEYWORDS = [
     "denmark", "copenhagen", "danmark", "københavn", "koebenhavn",
+    # Added 2026-09-11 (F12): the two other Danish cities that actually hire here.
+    "aarhus", "århus", "odense", "aalborg", "ålborg", "lyngby", "roskilde",
 ]
 
 NO_LOCATION_KEYWORDS = [
@@ -1725,6 +1734,24 @@ ADAPTERS = {
 }
 
 
+# An explicit onsite marker or a single-country restriction beats a stray pan-regional word.
+_ONSITE = re.compile(r"\b(?:on-?site|in-?office|vor ort|presence required|100% office)\b", re.I)
+_COUNTRY_ONLY = re.compile(
+    r"\b(?:only|nur|exclusively|must (?:be|reside) in|residents? of|based in)\b", re.I)
+
+
+def _place_match(low, keywords):
+    """Whole-word place matching.
+
+    Substring matching accepted "Bernau bei Berlin" as Bern, which put German roles into a
+    Swiss-only result set. Hyphens and slashes count as boundaries so "Bern/Zurich" still hits.
+    """
+    for keyword in keywords:
+        if re.search(r"(?<![\w])" + re.escape(keyword) + r"(?![\w])", low):
+            return True
+    return False
+
+
 def location_eligibility(loc_text, policy="default"):
     """Decide whether a posting's location is in-scope.
 
@@ -1745,15 +1772,19 @@ def location_eligibility(loc_text, policy="default"):
     if not loc_text:
         return False, False, False, False
     low = loc_text.lower()
-    in_ch = any(k in low for k in CH_LOCATION_KEYWORDS)
-    in_dk = any(k in low for k in DK_LOCATION_KEYWORDS)
-    in_no = any(k in low for k in NO_LOCATION_KEYWORDS)
+    in_ch = _place_match(low, CH_LOCATION_KEYWORDS)
+    in_dk = _place_match(low, DK_LOCATION_KEYWORDS)
+    in_no = _place_match(low, NO_LOCATION_KEYWORDS)
     is_us_only = any(p in low for p in US_ONLY_PATTERNS) and not in_ch
     has_remote = bool(re.search(r"\bremote\b", low)) or "work from anywhere" in low
     europe_scope = any(k in low for k in EUROPE_SCOPE_KEYWORDS)
     global_scope = any(k in low for k in GLOBAL_SCOPE_KEYWORDS) and not is_us_only
-    # Pan-regional scope counts even without the word "remote" (e.g. QuantCo "Europe").
-    is_europe_or_global = (europe_scope or global_scope) and not is_us_only
+    # Pan-regional scope counts even without the word "remote" (e.g. QuantCo "Europe"),
+    # but NOT when the posting also says onsite or restricts residence to one country
+    # (F12: "Europe" beside "London / Onsite" used to read as remote-eligible).
+    restricted = bool(_ONSITE.search(low)) or (
+        bool(_COUNTRY_ONLY.search(low)) and not (in_ch or in_dk))
+    is_europe_or_global = (europe_scope or global_scope) and not is_us_only and not restricted
     is_ch_remote = in_ch and has_remote
 
     if policy == "norway_only":
@@ -1899,11 +1930,20 @@ def _decision_key(url):
         if m:
             return f"{host}#{m.group(1)}"
 
-    # No known ID shape: fall back to the path with cosmetic variation stripped.
+    # No known ID shape: fall back to the path with cosmetic variation stripped. The QUERY
+    # is kept (F13): dropping it made /vacancy?job=1 and /vacancy?job=2 the same key, so one
+    # decision silently covered two different requisitions.
     path = parts.path.rstrip("/")
     if path.endswith("/apply"):
         path = path[: -len("/apply")]
-    return f"{host}#{path.lower()}"
+    query = urllib.parse.parse_qs(parts.query)
+    # Only identifying parameters; tracking noise must not fragment the key.
+    keep = {k: v for k, v in query.items()
+            if k.lower() in {"job", "jobid", "id", "req", "reqid", "requisition", "gh_jid", "vacancy"}}
+    suffix = ""
+    if keep:
+        suffix = "?" + "&".join(f"{k}={sorted(v)[0]}" for k, v in sorted(keep.items()))
+    return f"{host}#{path.lower()}{suffix}"
 
 
 def load_decisions():
@@ -1958,6 +1998,36 @@ def find_decision_url(decisions, url):
     return None
 
 
+@lru_cache(maxsize=1)
+def _raw_decisions():
+    """Raw decision log for the report path's slug fallback (cached per run)."""
+    return load_decisions()
+
+
+def lookup_decision(dec_index, decisions, url):
+    """Resolve a decision for a report row, using the SAME identity rules as the write path.
+
+    The 2026-09-04 fix taught `find_decision_url` (used by --decide) that RUAG regenerates the
+    UUID on every repost, but the REPORT path kept doing a bare `_decision_key` lookup. Result:
+    a reposted RUAG requisition came back as undecided although the decision was in the log -
+    which is exactly what happened to `Lead-Architekt Network- und Richtfunk` on 2026-09-10,
+    days after 63 RUAG rule-outs had been written.
+
+    A slug match is reported as RELATED, not as the same requisition: a repost can differ
+    materially, so an old rule-out must not be applied to it automatically.
+    """
+    hit = dec_index.get(_decision_key(url))
+    if hit is not None:
+        return hit
+    existing = find_decision_url(decisions, url)
+    if existing is None:
+        return None
+    related = dict(decisions[existing])
+    related["matched_by"] = "slug"
+    related["matched_url"] = existing
+    return related
+
+
 def save_decisions(decisions):
     _atomic_write_json(DECISIONS_FILE, decisions)
 
@@ -2053,7 +2123,8 @@ def write_report(path, results, errors, new_only, include_weak, stats=None, tota
 
     if not include_weak and weak:
         lines.append(f"\n_Hiding {len(weak)} weak/noise roles (score < 2). Use --include-weak to show._")
-    n_decided = sum(1 for r in results if _decision_key(r["url"]) in decisions)
+    n_decided = sum(1 for r in results
+                    if lookup_decision(decisions, _raw_decisions(), r["url"]) is not None)
     if n_decided:
         shown = "hidden" if hide_decided else "tagged inline"
         lines.append(f"_{n_decided} role(s) already in the decision log ({shown}; "
@@ -2066,12 +2137,13 @@ def write_report(path, results, errors, new_only, include_weak, stats=None, tota
 
     for bucket_name, bucket in buckets:
         shown = [r for r in bucket
-                 if not (hide_decided and _decision_key(r["url"]) in decisions)]
+                 if not (hide_decided and
+                         lookup_decision(decisions, _raw_decisions(), r["url"]) is not None)]
         if not shown:
             continue
         lines.append(f"\n## {bucket_name} - {len(shown)} role(s)\n")
         for r in shown:
-            d = decisions.get(_decision_key(r["url"]))
+            d = lookup_decision(decisions, _raw_decisions(), r["url"])
             new_tag = " [NEW]" if r["is_new"] else ""
             decided_tag = f" — 🗂 {d['decision'].upper()}" if d else ""
             loc_tag = ("CH" if r["in_ch"] else
@@ -2086,7 +2158,13 @@ def write_report(path, results, errors, new_only, include_weak, stats=None, tota
             lines.append(f"- URL: {r['url']}")
             if d:
                 note = f" — {d['note']}" if d.get("note") else ""
-                lines.append(f"- 🗂 Decision: **{d['decision']}**{note} ({d.get('date','')})")
+                if d.get("matched_by") == "slug":
+                    lines.append(
+                        f"- 🗂 RELATED decision (same slug, **new requisition id** — a repost can "
+                        f"differ, so re-read before reusing): **{d['decision']}**{note} "
+                        f"({d.get('date','')}) · matched `{d.get('matched_url','')}`")
+                else:
+                    lines.append(f"- 🗂 Decision: **{d['decision']}**{note} ({d.get('date','')})")
             if r["pos"]:
                 lines.append(f"- Positive: {', '.join(r['pos'])}")
             if r["neg"]:
@@ -2118,12 +2196,38 @@ def _fetch_company(cid, display, adapter, args):
         return cid, display, adapter, args, None, msg, time.perf_counter() - t0
 
 
+def _requisition_key(cid, job):
+    """Stable per-requisition identity within one employer.
+
+    Prefers a board requisition id parsed out of the URL (the same patterns the decision log
+    uses), then the adapter's own id, then the URL. Employer-scoped so two boards that number
+    requisitions the same way can never collide.
+    """
+    url = (job.get("url") or "").strip()
+    if url:
+        for host_pat, id_pat in _JOB_ID_PATTERNS:
+            try:
+                parts = urllib.parse.urlsplit(url)
+            except ValueError:
+                break
+            host = (parts.netloc or "").lower()
+            if host_pat and not re.search(host_pat, host):
+                continue
+            match = re.search(id_pat, parts.path + (f"?{parts.query}" if parts.query else ""))
+            if match:
+                return f"{cid}#{match.group(1)}"
+    identifier = job.get("id")
+    return f"{cid}#{identifier}" if identifier else f"{cid}#{url or job.get('title', '')}"
+
+
 def _process_company(cid, display, args, jobs, seen, today, last_scrape):
     """Filter, score, and update seen for one company's job list.
 
     Returns (results, stat_dict, warnings) where warnings is a list of (display, msg).
     """
     warnings = []
+    req_seen: dict[str, dict] = {}
+    title_collisions: dict[str, int] = {}
     scraped = len(jobs)
     prev = (last_scrape.get(cid) or {}).get("scraped", 0)
     if scraped == 0:
@@ -2164,11 +2268,24 @@ def _process_company(cid, display, args, jobs, seen, today, last_scrape):
         )
         if not ok:
             continue
-        # Collapse the same role posted once per remote country (title differs only
-        # by a "| Country | Remote" suffix) — dedupe on the title before the first "|".
+        # Requisition identity is the primary key (F11). Title alone is NOT identity: two
+        # different reqs called "Senior Data Engineer" in Bern and Zurich used to collapse
+        # into one, and different teams hiring the same title vanished silently.
+        req_key = _requisition_key(cid, j)
+        if req_key in req_seen:
+            # Same requisition seen again - the case the old rule existed for, a role
+            # republished once per location. Merge the location instead of dropping it.
+            first = req_seen[req_key]
+            extra = (j.get("location") or "").strip()
+            if extra and extra.lower() not in first["location"].lower():
+                first["location"] = f"{first['location']} / {extra}"
+            continue
         norm_title = re.sub(r"\s+", " ", (j.get("title") or "").split("|")[0]).strip().lower()
         if norm_title in title_seen:
-            continue
+            # Same title, DIFFERENT requisition. Keep it and flag it; removing it here is
+            # what lost real roles.
+            title_collisions.setdefault(norm_title, 0)
+            title_collisions[norm_title] += 1
         title_seen.add(norm_title)
         eligible += 1
         is_new = jid not in company_seen
@@ -2180,7 +2297,7 @@ def _process_company(cid, display, args, jobs, seen, today, last_scrape):
             score = floor
         if score >= 2:
             match += 1
-        results.append({
+        row = {
             "company": display, "company_id": cid,
             "title": j["title"], "location": j["location"],
             "url": j["url"], "posted": j.get("posted", ""),
@@ -2188,10 +2305,18 @@ def _process_company(cid, display, args, jobs, seen, today, last_scrape):
             "score": score, "pos": pos, "neg": neg,
             "in_ch": in_ch, "remote": is_remote, "relocation": is_relocation,
             "is_new": is_new,
-        })
+        }
+        results.append(row)
+        req_seen[req_key] = row
         if is_new:
             company_seen[jid] = {"title": j["title"], "first_seen": today}
 
+    if title_collisions:
+        detail = ", ".join(f"{title!r} x{count + 1}" for title, count in sorted(title_collisions.items())[:4])
+        warnings.append((
+            display,
+            f"{len(title_collisions)} title(s) shared by different requisitions, all KEPT "
+            f"(check for real duplicates): {detail}"))
     stat = {
         "company": display, "scraped": scraped, "eligible": eligible,
         "match": match, "newest": newest, "error": False,
@@ -2338,7 +2463,7 @@ def main():
         "company": r["company"], "title": r["title"], "location": r["location"],
         "url": r["url"], "posted": r["posted"], "deadline": r.get("deadline", ""),
         "is_new": r["is_new"],
-        "decision": dec_index.get(_decision_key(r["url"])),
+        "decision": lookup_decision(dec_index, decisions, r["url"]),
         "keyword_signal": {"score": r["score"], "pos": r["pos"], "neg": r["neg"]},
         "loc_flags": {
             "in_ch": r["in_ch"], "remote": r["remote"], "relocation": r["relocation"],
diff --git a/tests/test_scout_matching.py b/tests/test_scout_matching.py
new file mode 100644
index 0000000..c477dd6
--- /dev/null
+++ b/tests/test_scout_matching.py
@@ -0,0 +1,65 @@
+"""Scout selection regressions from the 2026-09-11 review (F11 onwards).
+
+These are the cases where a bug silently removed or mis-ranked a role *before* any human
+judgement happened, which is the expensive kind: a job that never reaches the report cannot
+be evaluated at all.
+"""
+from __future__ import annotations
+
+import sys
+import unittest
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT / "job_scout"))
+
+import scout  # noqa: E402
+
+
+def process(jobs, cid="acme", display="Acme"):
+    results, stat, warnings = scout._process_company(
+        cid, display, {}, jobs, {}, "2026-09-11", {})
+    return results, stat, warnings
+
+
+class RequisitionDedupeTests(unittest.TestCase):
+    """F11: title is not identity."""
+
+    def test_two_requisitions_sharing_a_title_both_survive(self):
+        jobs = [
+            {"id": "R-1", "title": "Senior Data Engineer",
+             "location": "Bern, Switzerland", "url": "https://acme.example/jobs/R-1"},
+            {"id": "R-2", "title": "Senior Data Engineer",
+             "location": "Zurich, Switzerland", "url": "https://acme.example/jobs/R-2"},
+        ]
+        results, _, warnings = process(jobs)
+        self.assertEqual(2, len(results), "a different requisition must never be dropped")
+        self.assertTrue(any("title(s) shared by different requisitions" in message
+                            for _, message in warnings),
+                        "the collision should be reported, not silently resolved")
+
+    def test_same_requisition_republished_per_location_is_merged(self):
+        jobs = [
+            {"id": "R-9", "title": "Platform Engineer | Switzerland | Remote",
+             "location": "Bern, Switzerland", "url": "https://acme.example/jobs/R-9"},
+            {"id": "R-9", "title": "Platform Engineer | Germany | Remote",
+             "location": "Munich, Germany", "url": "https://acme.example/jobs/R-9"},
+        ]
+        results, _, _ = process(jobs)
+        self.assertEqual(1, len(results), "one requisition should stay one row")
+        self.assertIn("Bern", results[0]["location"])
+
+    def test_distinct_titles_are_unaffected(self):
+        jobs = [
+            {"id": "A", "title": "Data Engineer", "location": "Bern, Switzerland",
+             "url": "https://acme.example/jobs/A"},
+            {"id": "B", "title": "Platform Engineer", "location": "Bern, Switzerland",
+             "url": "https://acme.example/jobs/B"},
+        ]
+        results, _, warnings = process(jobs)
+        self.assertEqual(2, len(results))
+        self.assertEqual([], [m for _, m in warnings if "title(s) shared" in m])
+
+
+if __name__ == "__main__":
+    unittest.main()