feat: rebuild evidence-first application workflow

job_scout/scout.py

@@ -255,6 +255,11 @@ COMPANIES = [
         "scroll_count": 5,
         "use_inner_text_as_blob": True,
         "cookie_accept": ["button:has-text('Accept all')", "button:has-text('Reject all')"],
+        # Results are paged 20 at a time and "page" is 1-indexed — without this only the
+        # first 20 of ~48 Zürich roles were ever scraped.
+        "page_param": "page",
+        "page_param_start": 2,
+        "max_pages": 6,
     }),
     ("apple",     "Apple",     "playwright", {
         "url": "https://jobs.apple.com/en-us/search?location=switzerland-CHE",
@@ -534,7 +539,8 @@ def fetch_pcsx(args):
     while True:
         url = f"{base}?domain={domain}&query=&location={urllib.parse.quote(location)}&start={start}&num=50"
         data = http_get_json(url, headers={"Referer": f"https://apply.careers.microsoft.com/careers?location={urllib.parse.quote(location)}"})
-        positions = (data.get("data") or {}).get("positions", []) or []
+        payload = data.get("data") or {}
+        positions = payload.get("positions", []) or []
         for p in positions:
             locs = p.get("locations") or []
             jobs.append({
@@ -545,9 +551,16 @@ def fetch_pcsx(args):
                 "posted": p.get("postedTs", ""),
                 "description": (p.get("description") or "")[:2000],
             })
-        if not positions or len(positions) < 50:
+        # The API serves ~10 rows per call regardless of `num`, so a short page is NOT
+        # end-of-results — page until a call comes back empty or `count` is reached.
+        # (The old `len(positions) < 50` break capped Microsoft at its first 10 CH roles
+        # and hid the entire Zürich Principal-FDE cluster sitting on page 2.)
+        if not positions:
             break
         start += len(positions)
+        total = payload.get("count")
+        if isinstance(total, int) and start >= total:
+            break
         if start >= 500:
             break
     return jobs
@@ -1016,6 +1029,23 @@ def _absolutize(href, prefix):
     return prefix.rstrip("/") + "/" + cleaned
 
 
+def _strip_query_param(url, param):
+    """Drop a single query parameter from a URL, preserving the rest.
+
+    Job-card hrefs inherit the listing page's query string, so with query-param
+    pagination the same posting would get a different URL (and therefore a different
+    id) depending on which page it happened to land on — making every reshuffle look
+    like a brand-new job and orphaning its decision-log entry.
+    """
+    if not url or f"{param}=" not in url:
+        return url
+    head, _, query = url.partition("?")
+    if not query:
+        return url
+    kept = [kv for kv in query.split("&") if kv and not kv.startswith(f"{param}=")]
+    return f"{head}?{'&'.join(kept)}" if kept else head
+
+
 def _extract_location_from_blob(full, default=""):
     """Pull a short location line out of a job-card text blob.
 
@@ -1127,6 +1157,8 @@ def fetch_playwright(args):
                 link_el = card if not args.get("link_sel") else card.locator(args["link_sel"]).first
                 href = (link_el.get_attribute(args.get("link_attr", "href")) or "") if link_el.count() else ""
                 href = _absolutize(href, args.get("url_prefix", ""))
+                if args.get("page_param"):
+                    href = _strip_query_param(href, args["page_param"])
 
                 if not title:
                     continue
@@ -1172,15 +1204,18 @@ def fetch_playwright(args):
                     page.wait_for_timeout(500)
             except Exception:
                 pass
-        # Optional query-param pagination (e.g. Drupal "?page=N", 0-indexed). The base URL is
-        # page 0 (already loaded); fetch successive pages until one adds no new cards.
+        # Optional query-param pagination (e.g. Drupal "?page=N"). The base URL is the first
+        # page (already loaded); fetch successive pages until one adds no new cards.
+        # `page_param_start` is the value the SECOND page takes: 1 for 0-indexed boards
+        # (Drupal), 2 for 1-indexed ones (Google, where "?page=1" is just page one again).
         page_param = args.get("page_param")
         if page_param:
             base = args["url"]
             joiner = "&" if "?" in base else "?"
+            second = args.get("page_param_start", 1)
             for p in range(args.get("max_pages", 8)):
                 if p > 0:
-                    page.goto(f"{base}{joiner}{page_param}={p}", timeout=45000,
+                    page.goto(f"{base}{joiner}{page_param}={second + p - 1}", timeout=45000,
                               wait_until="domcontentloaded")
                 added = scrape_current()
                 if p > 0 and added == 0: