diff --git a/job_scout/scout.py b/job_scout/scout.py index 4ad12df..0822a06 100644 --- a/job_scout/scout.py +++ b/job_scout/scout.py @@ -2,8 +2,18 @@ Pulls latest openings from companies via public ATS APIs (Workday/Ashby/Greenhouse/ SmartRecruiters/Lever/Eightfold/RSS) and, for JS-rendered careers sites, a headless-browser -(playwright) adapter. Filters by Swiss location or remote eligibility, scores fit against -profile keywords, tracks which job IDs we've already seen, writes a markdown report. +(playwright) adapter. Filters by location policy, scores fit against profile keywords, +tracks which job IDs we've already seen, writes a markdown report. + +Location policy (default for most companies): + - Switzerland onsite/hybrid (CH) + - Denmark onsite/reloc + - CH-remote and Europe/EMEA/global remote (work-from CH or DK is realistic) + - NOT multi-country allowlists that omit CH/DK/Europe (e.g. "UK | Brazil | Ireland | ...") + +Company overrides via args["_location_policy"]: + - "norway_only" — Equinor: Norway only + - "dk_no" — NATO-style: Denmark + Norway only (no CH — no NATO seat in CH) Usage: py scout.py # Pull all configured companies (strong + medium only) @@ -14,8 +24,8 @@ Usage: py scout.py --decide "" [note...] # Record a decision and exit # status is free-text: shortlist | skip | applied | paused | ... -State : state/seen_jobs.json (job IDs seen) · state/decisions.json (per-URL decisions) -Output: reports/YYYY-MM-DD.md (scan-stats table + scored roles, decisions tagged inline) +State : state/seen_jobs.json · state/decisions.json · state/last_scrape.json +Output: reports/YYYY-MM-DD.md (+ .json dump of every eligible role) To add a company: append to COMPANIES with one of the existing adapter types. A few sites resist scraping even headless and stay in MANUAL_CHECK (surfaced as a report checklist). @@ -23,10 +33,12 @@ See the adapter-coverage notes at the bottom for the current automated/manual sp """ import json +import os import re import ssl import sys import time +from concurrent.futures import ThreadPoolExecutor, as_completed from functools import lru_cache import urllib.error import urllib.parse @@ -37,39 +49,38 @@ from pathlib import Path ROOT = Path(__file__).parent STATE_FILE = ROOT / "state" / "seen_jobs.json" DECISIONS_FILE = ROOT / "state" / "decisions.json" +LAST_SCRAPE_FILE = ROOT / "state" / "last_scrape.json" REPORTS_DIR = ROOT / "reports" USER_AGENT = "Mozilla/5.0 (compatible; job-scout/0.1)" +API_FETCH_WORKERS = 8 CH_LOCATION_KEYWORDS = [ "switzerland", "zurich", "zürich", "basel", "bern", "geneva", "genf", "lausanne", "zug", "rüschlikon", "stäfa", "schweiz", "suisse", ] +DK_LOCATION_KEYWORDS = [ + "denmark", "copenhagen", "danmark", "københavn", "koebenhavn", +] + +NO_LOCATION_KEYWORDS = [ + "norway", "oslo", "stavanger", "bergen", "trondheim", "norge", +] + US_ONLY_PATTERNS = [ "remote - us", "remote, us", "remote-us", "us remote", "us-remote", "remote-friendly us", "remote (us)", "united states - remote", "remote, united states", ] -# Pan-regional/global remote — genuinely reachable from Switzerland regardless of which -# single country the listing happens to name. -GLOBAL_REMOTE_KEYWORDS = ["europe", "emea", "global", "worldwide"] - -# Single-country hints, grouped so synonyms ("uk"/"united kingdom") count as one country. -# A listing naming only ONE of these is a residency lock (e.g. Databricks' "Remote - United -# Kingdom Remote - United Kingdom"), not something open to a CH-based candidate, even though -# it reads as "remote." Naming 2+ reads as a broad multi-country remote policy instead (e.g. -# Grafana's "Germany | Ireland | Spain | Sweden | UK") — that's treated as eligible. -COUNTRY_HINT_GROUPS = [ - ("germany",), ("france",), ("spain",), ("portugal",), ("ireland",), - ("netherlands",), ("sweden",), ("norway",), ("finland",), ("poland",), - ("czech",), ("romania",), ("italy",), ("austria",), ("belgium",), - ("uk", "united kingdom"), +# Pan-regional remote scope: work-from CH or DK is realistic. +EUROPE_SCOPE_KEYWORDS = [ + "europe", "emea", "european", "eu-wide", "eu wide", "remote europe", + "europe remote", "remote - europe", "remote, europe", +] +GLOBAL_SCOPE_KEYWORDS = [ + "global", "worldwide", "work from anywhere", "anywhere in the world", ] - -# Countries outside CH where relocation is genuinely on the table for a strong enough offer -# (see project_target_companies memory) — accepted for every company, on-site or remote. -RELOCATION_COUNTRY_KEYWORDS = ["denmark", "copenhagen"] POSITIVE_KEYWORDS = { "genai": 3, "generative ai": 3, "llm": 3, "large language model": 3, @@ -172,7 +183,7 @@ COMPANIES = [ # Added 2026-06-06 (Tier A/B data-infra). Databricks/Snowflake/Datadog have Zürich offices # (Swiss-scale comp, clears bar); Elastic/dbt Labs are remote-EU (verify CH-equiv comp — # may be geo-banded below 180k, like Grafana). All title-filtered (boards are 160-760 roles). - ("databricks","Databricks","greenhouse", {"board": "databricks", "_title_filter": ENG_TITLE_FILTER}), # Zürich SWE + remote-EU; Denmark relocation handled globally, see RELOCATION_COUNTRY_KEYWORDS + ("databricks","Databricks","greenhouse", {"board": "databricks", "_title_filter": ENG_TITLE_FILTER}), # Zürich SWE + remote-EU; DK via default location policy ("snowflake", "Snowflake", "ashby", {"slug": "snowflake", "_title_filter": ENG_TITLE_FILTER}), # Zürich "Observe" observability SWE roles ("datadog", "Datadog", "greenhouse", {"board": "datadog", "_title_filter": ENG_TITLE_FILTER}), # Zürich branch + remote-EU ("elastic", "Elastic", "greenhouse", {"board": "elastic", "_title_filter": ENG_TITLE_FILTER}), # remote-first; ELK = his stack @@ -182,15 +193,31 @@ COMPANIES = [ # Dropped: Sygnum (Glassdoor 3.4, 51% recommend, comp 2.3/5 — below 180k bar — 2026-05). ("metgroup", "MET Group", "smartrecruiters", {"company": "METGroup", "_title_filter": ENG_TITLE_FILTER}), ("ldc", "Louis Dreyfus","smartrecruiters",{"company": "LouisDreyfusCompany", "_title_filter": ENG_TITLE_FILTER}), - # Equinor (Workday) — Norway energy major; he's lived/worked in NO before (Equinor AI - # Architect req was applied to manually — see CLAUDE.md Active Sessions) and it's an - # energy-trading-adjacent target. Live board is small (~15 total) and mostly NO/US/BR - # on-site ops/trading roles, so no server-side search_text filter or title filter — - # let location_matches do the CH/remote-eligibility work like the other low-volume boards. + # Equinor (Workday) — Norway energy major; lived/worked in NO before. Outlier location + # policy: Norway only (not CH / Europe-remote). Small board (~15); no title filter. ("equinor", "Equinor", "workday", { "host": "equinor.wd3.myworkdayjobs.com", "tenant": "equinor", "site": "EQNR", + "_location_policy": "norway_only", + # Norway-only filter already scopes the board; titles are often generic/energy + # ops and score 0 under the English eng keyword list — keep them visible. + "_score_floor": 2, + }), + # NATO civilian vacancies (Oracle Taleo). Public marketing page is + # https://www.nato.int/en/work-with-us/careers/vacancies — jobs actually live on + # nato.taleo.net career section 2. Location policy dk_no: Denmark + Norway only + # (no NATO seat in CH). Server-side location taxonomy IDs filter the board before + # parse; dk_no is a second safety net on the location string. + # Taleo option values (jobsearch.ftl dropdown): Norway=14470035151, Denmark=16270035151. + ("nato", "NATO", "taleo", { + "host": "nato.taleo.net", + "career_section": "2", + "location_ids": ["14470035151", "16270035151"], # Norway, Denmark + "_location_policy": "dk_no", + # Staff Officer / NATO-body titles score unevenly under eng keywords — keep + # pre-filtered NO/DK roles visible (same pattern as Equinor/SBB). + "_score_floor": 2, }), # International org — BIS (Basel), commutable from Bern, salary net of Swiss tax. # Low-volume RSS feed; no title filter (Innovation Hub roles can be oddly titled). @@ -288,10 +315,9 @@ COMPANIES = [ # Deployment Strategist) aren't in ENG_TITLE_FILTER, so filtering would hide them. ("palantir", "Palantir", "lever", {"slug": "palantir"}), # QuantCo (Lever — note the trailing-hyphen slug "quantco-"). ~16 roles, most tagged - # "Europe" (hybrid); QuantCo's continental hub is Zürich, so the EU-wide rule in - # location_matches surfaces them. No title filter: the target band is DS/Quant/AI/Cloud - # (see comp analysis), which ENG_TITLE_FILTER would drop; interns/frontend are caught by - # NEGATIVE_KEYWORDS instead. + # "Europe" (hybrid); QuantCo's continental hub is Zürich, so Europe-scope in + # location_eligibility surfaces them. No title filter: target band is DS/Quant/AI/Cloud + # (ENG_TITLE_FILTER would drop some); interns/frontend caught by NEGATIVE_KEYWORDS. ("quantco", "QuantCo", "lever", {"slug": "quantco-"}), # --- Bern/Thun local tier — WLB & proximity exception (comp bar relaxed; 2026-06-01) --- # Wired after live endpoint discovery. ⚠️ German citizen: RUAG classified work may require @@ -375,12 +401,8 @@ COMPANIES = [ }), ] -# Companies where adapter probing did not yield a reliable scrape. Reasons noted. -# These surface as a clickable checklist in the report so they're not forgotten. -# Companies that resist scraping stay here as a clickable report checklist. Currently empty — -# every target company is automated. (Dropped 2026-06-01: BFH — academic FH pay below even the -# relaxed Bern/Thun floor, research-leaning, 403s anyway; Dialectic — ~50-person crypto VC, -# 0 open roles, crypto angle already covered by Kraken/Bitcoin Suisse/Coinbase Ventures.) +# Companies that resist scraping — clickable checklist in the report. +# (BFH re-automated 2026-07-14 on www.bfh.ch. Dialectic dropped — crypto covered elsewhere.) MANUAL_CHECK = [ # Oracle (Tier C, requested 2026-06-06). Oracle Recruiting Cloud (ORC) resists clean # scraping: careers.oracle.com renders job tiles that are NOT anchors, so the playwright @@ -473,8 +495,12 @@ def fetch_ashby(args): def fetch_greenhouse(args): + # Title-filtered boards score title_only — skip heavy JD HTML (content=true) for those. board = args["board"] - url = f"https://boards-api.greenhouse.io/v1/boards/{board}/jobs?content=true" + want_content = not args.get("_title_filter") + url = f"https://boards-api.greenhouse.io/v1/boards/{board}/jobs" + if want_content: + url += "?content=true" data = http_get_json(url) jobs = [] for j in data.get("jobs", []): @@ -482,9 +508,11 @@ def fetch_greenhouse(args): offices = j.get("offices") or [] office_names = " | ".join(o.get("name", "") for o in offices if isinstance(o, dict)) loc_blob = " ".join(x for x in [loc, office_names] if x) - desc = j.get("content", "") or "" - desc = re.sub(r"<[^>]+>", " ", desc) - desc = re.sub(r"\s+", " ", desc).strip() + desc = "" + if want_content: + desc = j.get("content", "") or "" + desc = re.sub(r"<[^>]+>", " ", desc) + desc = re.sub(r"\s+", " ", desc).strip() jobs.append({ "id": str(j.get("id")), "title": j.get("title", ""), @@ -587,25 +615,6 @@ def fetch_rss(args): return jobs -def fetch_wp_ajax(args): - """WordPress admin-ajax style endpoint. Sygnum uses this pattern.""" - url = args["url"] - data = http_get_json(url) - if not isinstance(data, list): - return [] - jobs = [] - for j in data: - jobs.append({ - "id": (j.get("title", "") + "|" + j.get("location", ""))[:120], - "title": j.get("title", ""), - "location": " ".join(filter(None, [j.get("location", ""), j.get("work_type", "")])), - "url": j.get("application_url") or args["url"], - "posted": "", - "description": " ".join(filter(None, [j.get("department", ""), j.get("role_type", "")])), - }) - return jobs - - def fetch_getro(args): """Getro network job-board search API (POST JSON). Powers VC portfolio talent networks — here the Coinbase Ventures web3 network (collection 1625). Returns roles @@ -669,10 +678,15 @@ def fetch_onlyfy(args): page = resp.read().decode("utf-8", "replace") titles = re.findall(r'(.*?)', page, re.S) locs = re.findall(r'icon-map-marker[^>]*>\s*([^<]+)', page) + if titles and locs and len(titles) != len(locs): + # Misaligned regex pairs would invent wrong locations; keep titles, drop bad locs. + print(f" onlyfy: title/loc count mismatch ({len(titles)} vs {len(locs)}); " + f"using titles only", file=sys.stderr) + locs = [""] * len(titles) jobs = [] - for (href, raw_title), raw_loc in zip(titles, locs): + for (href, raw_title), raw_loc in zip(titles, locs if locs else [""] * len(titles)): title = _html.unescape(re.sub(r"<[^>]+>", "", raw_title)).strip() - loc = _html.unescape(raw_loc).strip() + loc = _html.unescape(raw_loc).strip() if raw_loc else "" jobs.append({ "id": href.rsplit("/", 1)[-1], "title": title, @@ -711,6 +725,141 @@ def fetch_lever(args): return jobs +def _parse_taleo_jobsearch_html(html, host, career_section): + """Extract jobs from a Taleo jobsearch.ftl HTML response. + + Taleo does not expose a reliable public JSON search API for this board (REST + returns careerSectionUnAvailable). Jobs are embedded in the page as + api.fillInterface arrays and as a pipe-delimited initialHistory field. + """ + import html as _html + + jobs, seen = [], set() + detail_base = f"https://{host}/careersection/{career_section}/jobdetail.ftl" + + # Quoted fillInterface form (already-unescaped dates in modern Taleo builds). + # Observed record shape (NATO 2026-07): + # '','','<id>','<title>', five more '<id>', + # '<jobNo>','<Country-City>','false','','','','', + # '<deadline>','<NATO body>','<grade>' + pat = re.compile( + r"'(\d+)','((?:[^'\\]|\\.)+)'," + r"'\1','\2'," + r"(?:'\1',){5}" + r"'(\d+)','((?:[^'\\]|\\.)+)'," + r"'false','','','',''," + r"'((?:[^'\\]|\\.)*)','((?:[^'\\]|\\.)*)','((?:[^'\\]|\\.)*)'" + ) + for m in pat.finditer(html): + job_no = m.group(3) + if job_no in seen: + continue + seen.add(job_no) + + def _unq(s): + return _html.unescape(s.replace("\\'", "'").replace("\\:", ":")) + + title = _unq(m.group(2)) + loc = _unq(m.group(4)).replace("-", ", ") + deadline = _unq(m.group(5)) + body = _unq(m.group(6)) + grade = _unq(m.group(7)) + # List view exposes application deadline only (not a posted-on date). We store + # the date part in `posted` so the report surfaces the closing date. + deadline_short = deadline.split(",")[0].strip() if deadline else "" + jobs.append({ + "id": job_no, + "title": title, + "location": loc, + "url": f"{detail_base}?job={job_no}&lang=en", + "posted": deadline_short, + "description": f"{body} {grade}".strip()[:500], + }) + if jobs: + return jobs + + # Fallback: pipe-delimited initialHistory hidden field + m = re.search( + r'(?:name|id)="initialHistory"[^>]*value="([^"]*)"', html, re.I + ) or re.search( + r'value="([^"]*)"[^>]*(?:name|id)="initialHistory"', html, re.I + ) + if not m: + return [] + hist = urllib.parse.unquote(m.group(1).replace("%5C:", ":").replace("\\:", ":")) + parts = hist.split("!|!") + for i, p in enumerate(parts): + if not re.fullmatch(r"\d{5,6}", p) or i + 1 >= len(parts): + continue + loc_raw = parts[i + 1] + if not re.search( + r"Norway|Denmark|Belgium|Germany|Italy|France|United|Luxembourg|" + r"Netherlands|Poland|Spain|Portugal|Canada|Turkey|Türkiye", + loc_raw, re.I, + ): + continue + job_no = p + if job_no in seen: + continue + title = "" + for j in range(i - 1, max(-1, i - 12), -1): + cand = parts[j] + if (cand and not re.fullmatch(r"\d+", cand) + and cand not in ("false", "true", "Apply") + and len(cand) > 8): + title = cand + break + if not title: + continue + seen.add(job_no) + deadline = parts[i + 6] if i + 6 < len(parts) else "" + body = parts[i + 7] if i + 7 < len(parts) else "" + grade = parts[i + 8] if i + 8 < len(parts) else "" + deadline_short = deadline.split(",")[0].strip() if deadline else "" + jobs.append({ + "id": job_no, + "title": _html.unescape(title), + "location": loc_raw.replace("-", ", "), + "url": f"{detail_base}?job={job_no}&lang=en", + "posted": deadline_short, + "description": f"{body} {grade}".strip()[:500], + }) + return jobs + + +def fetch_taleo(args): + """Oracle Taleo careersection board (used by NATO civilian vacancies). + + The marketing page https://www.nato.int/en/work-with-us/careers/vacancies is an + AEM shell; the live board is Taleo. Public REST search is unavailable, so we + GET jobsearch.ftl (optionally filtered by Taleo location taxonomy IDs from the + board's location dropdown) and parse embedded job records. + + Args: host (default nato.taleo.net), career_section (default "2"), + location_ids (optional list of Taleo option values — e.g. Norway/Denmark). + """ + host = args.get("host", "nato.taleo.net") + section = str(args.get("career_section", "2")) + location_ids = args.get("location_ids") + # One request per location id (Taleo query param takes a single id); merge by job no. + # If no location_ids, one unfiltered first-page fetch. + fetches = location_ids if location_ids else [None] + by_id = {} + for loc_id in fetches: + url = f"https://{host}/careersection/{section}/jobsearch.ftl?lang=en" + if loc_id: + url += f"&location={urllib.parse.quote(str(loc_id))}" + req = urllib.request.Request(url, headers={ + "User-Agent": USER_AGENT, + "Accept": "text/html,application/xhtml+xml", + }) + with urllib.request.urlopen(req, timeout=60, context=_ssl_context()) as resp: + html = resp.read().decode("utf-8", "replace") + for j in _parse_taleo_jobsearch_html(html, host, section): + by_id[j["id"]] = j + return list(by_id.values()) + + def fetch_json(args): """Generic JSON jobs API with configurable field names, for employer sites that expose a clean public endpoint. Verified use: Swissgrid (Magnolia CMS @@ -867,6 +1016,40 @@ def _absolutize(href, prefix): return prefix.rstrip("/") + "/" + cleaned +def _extract_location_from_blob(full, default=""): + """Pull a short location line out of a job-card text blob. + + Google/Meta/Roche-style cards often dump Material-icon chrome into inner_text + (e.g. 'corporate_fare' / 'place' / 'Zürich, Switzerland' / 'bar_chart'). Prefer the + line after a 'place'/'location' label, else the first short line that looks like a + place, else the configured default — never the first 300 chars of the whole card. + """ + if not full: + return default + lines = [ln.strip() for ln in full.splitlines() if ln.strip()] + skip = { + "place", "location", "corporate_fare", "bar_chart", "work", "schedule", + "google", "meta", "apple", "roche", "cisco", + } + for i, ln in enumerate(lines): + if ln.lower() in ("place", "location") and i + 1 < len(lines): + cand = lines[i + 1].strip() + if cand and cand.lower() not in skip and len(cand) < 120: + return cand[:200] + place_hints = ( + CH_LOCATION_KEYWORDS + DK_LOCATION_KEYWORDS + NO_LOCATION_KEYWORDS + + list(EUROPE_SCOPE_KEYWORDS) + list(GLOBAL_SCOPE_KEYWORDS) + + ["remote", "hybrid", "onsite", "on-site"] + ) + for ln in lines: + low = ln.lower() + if len(ln) > 120 or low in skip: + continue + if any(k in low for k in place_hints): + return ln[:200] + return default + + def _close_browser(): if _playwright_singleton["browser"]: try: @@ -954,11 +1137,18 @@ def fetch_playwright(args): added += 1 description = "" if args.get("use_inner_text_as_blob"): - # Use the full card text as both location source and description + # Full card text for keyword scoring; location via structured extract. full = (card.inner_text() or "") description = full[:2000] - if not location: - location = full[:300] + extracted = _extract_location_from_blob( + full, default=args.get("default_location", "") + ) + # Prefer a real location line over a default that is empty or a + # polluted leftover; keep default_location when extract finds nothing. + if extracted: + location = extracted + elif not location: + location = args.get("default_location", "") jobs.append({ "id": jid, "title": title, @@ -1028,12 +1218,12 @@ ADAPTERS = { "ashby": fetch_ashby, "greenhouse": fetch_greenhouse, "pcsx": fetch_pcsx, - "wp_ajax": fetch_wp_ajax, "smartrecruiters": fetch_smartrecruiters, "rss": fetch_rss, "getro": fetch_getro, "onlyfy": fetch_onlyfy, "lever": fetch_lever, + "taleo": fetch_taleo, "json": fetch_json, "sbb": fetch_sbb, "bkw": fetch_bkw, @@ -1041,26 +1231,50 @@ ADAPTERS = { } -def location_matches(loc_text): +def location_eligibility(loc_text, policy="default"): + """Decide whether a posting's location is in-scope. + + Returns (eligible, in_ch, is_remote, is_relocation). + + Default policy (most companies): + - CH onsite/hybrid + - Denmark onsite/reloc + - Europe/EMEA/global remote scope (work-from CH or DK is realistic) + - CH-remote (Switzerland + remote wording) + - Multi-country allowlists WITHOUT CH/DK/Europe/global scope are NOT eligible + (e.g. Kraken "UK | Brazil | Ireland | ..." with no Switzerland/Europe) + + Overrides (args["_location_policy"]): + - norway_only — Equinor: Norway only + - dk_no — NATO-style: Denmark + Norway only (no CH) + """ if not loc_text: - return False, False + return False, False, False, False low = loc_text.lower() in_ch = any(k in low for k in CH_LOCATION_KEYWORDS) + in_dk = any(k in low for k in DK_LOCATION_KEYWORDS) + in_no = any(k in low for k in NO_LOCATION_KEYWORDS) is_us_only = any(p in low for p in US_ONLY_PATTERNS) and not in_ch - is_global_wide = any(k in low for k in GLOBAL_REMOTE_KEYWORDS) and not is_us_only - country_hits = sum(1 for group in COUNTRY_HINT_GROUPS if any(k in low for k in group)) - is_multi_country = country_hits >= 2 and not is_us_only - is_remote = is_global_wide or is_multi_country - return in_ch, is_remote + has_remote = bool(re.search(r"\bremote\b", low)) or "work from anywhere" in low + europe_scope = any(k in low for k in EUROPE_SCOPE_KEYWORDS) + global_scope = any(k in low for k in GLOBAL_SCOPE_KEYWORDS) and not is_us_only + # Pan-regional scope counts even without the word "remote" (e.g. QuantCo "Europe"). + is_europe_or_global = (europe_scope or global_scope) and not is_us_only + is_ch_remote = in_ch and has_remote + if policy == "norway_only": + eligible = in_no and not is_us_only + return eligible, False, False, eligible + if policy == "dk_no": + # NATO etc.: Denmark + Norway only — CH onsite is out (no NATO seat in CH). + eligible = (in_dk or in_no) and not is_us_only + return eligible, False, False, eligible -def is_relocation_location(loc_text): - """Denmark-only relocation acceptance (see RELOCATION_COUNTRY_KEYWORDS), applied to - every company — not a per-company opt-in.""" - if not loc_text: - return False - low = loc_text.lower() - return any(k in low for k in RELOCATION_COUNTRY_KEYWORDS) + # default + is_relocation = in_dk + is_remote = is_europe_or_global or is_ch_remote + eligible = (in_ch or in_dk or is_europe_or_global) and not (is_us_only and not in_ch) + return eligible, in_ch, is_remote, is_relocation @lru_cache(maxsize=512) @@ -1103,6 +1317,14 @@ def score_job(job, title_only=False): return score, pos, neg +def _atomic_write_json(path, data): + """Write JSON via temp + os.replace so a crash mid-write cannot truncate state files.""" + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_suffix(path.suffix + ".tmp") + tmp.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8") + os.replace(tmp, path) + + def load_seen(): if STATE_FILE.exists(): return json.loads(STATE_FILE.read_text(encoding="utf-8")) @@ -1110,8 +1332,7 @@ def load_seen(): def save_seen(seen): - STATE_FILE.parent.mkdir(parents=True, exist_ok=True) - STATE_FILE.write_text(json.dumps(seen, indent=2, ensure_ascii=False), encoding="utf-8") + _atomic_write_json(STATE_FILE, seen) def load_decisions(): @@ -1124,8 +1345,21 @@ def load_decisions(): def save_decisions(decisions): - DECISIONS_FILE.parent.mkdir(parents=True, exist_ok=True) - DECISIONS_FILE.write_text(json.dumps(decisions, indent=2, ensure_ascii=False), encoding="utf-8") + _atomic_write_json(DECISIONS_FILE, decisions) + + +def load_last_scrape(): + """Per-company last successful scraped counts — used to flag sudden 0-job deaths.""" + if LAST_SCRAPE_FILE.exists(): + try: + return json.loads(LAST_SCRAPE_FILE.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError): + return {} + return {} + + +def save_last_scrape(data): + _atomic_write_json(LAST_SCRAPE_FILE, data) def _parse_posted(s): @@ -1139,9 +1373,11 @@ def _parse_posted(s): return datetime.fromisoformat(s.replace("Z", "+00:00")).date() except ValueError: pass - for fmt in ("%Y-%m-%d", "%d.%m.%Y", "%Y/%m/%d", "%d/%m/%Y"): + for fmt in ("%Y-%m-%d", "%d.%m.%Y", "%Y/%m/%d", "%d/%m/%Y", "%d-%b-%Y", "%d-%B-%Y"): try: - return datetime.strptime(s[:10], fmt).date() + # %b/%B need full token (e.g. 09-Aug-2026), not a fixed 10-char slice + token = s.split(",")[0].strip() + return datetime.strptime(token if "%b" in fmt or "%B" in fmt else s[:10], fmt).date() except ValueError: pass m = re.search(r"\d{4}-\d{2}-\d{2}", s) @@ -1249,6 +1485,94 @@ def write_report(path, results, errors, new_only, include_weak, stats=None, tota path.write_text("\n".join(lines), encoding="utf-8") +def _fetch_company(cid, display, adapter, args): + """Fetch one company; returns (cid, display, adapter, args, jobs|None, error|None, secs).""" + t0 = time.perf_counter() + try: + jobs = ADAPTERS[adapter](args) + return cid, display, adapter, args, jobs, None, time.perf_counter() - t0 + except Exception as e: + kind = "unexpected" if not isinstance( + e, (urllib.error.URLError, urllib.error.HTTPError, ValueError, RuntimeError) + ) else "error" + msg = f"{kind}: {e!r}" if kind == "unexpected" else repr(e) + return cid, display, adapter, args, None, msg, time.perf_counter() - t0 + + +def _process_company(cid, display, args, jobs, seen, today, last_scrape): + """Filter, score, and update seen for one company's job list. + + Returns (results, stat_dict, warnings) where warnings is a list of (display, msg). + """ + warnings = [] + scraped = len(jobs) + prev = (last_scrape.get(cid) or {}).get("scraped", 0) + if scraped == 0: + if prev and prev > 0: + warnings.append(( + display, + f"0 jobs returned (was {prev} last run — possible dead board/slug/selectors)", + )) + else: + warnings.append(( + display, + "0 jobs returned (verify board slug/selectors if this is unexpected)", + )) + + title_filter = args.get("_title_filter") + if title_filter: + jobs = [j for j in jobs + if any(_kw_in(k, (j.get("title") or "").lower()) for k in title_filter)] + + dates = [d for j in jobs if (d := _parse_posted(j.get("posted")))] + newest = max(dates) if dates else None + + policy = args.get("_location_policy", "default") + company_seen = seen.setdefault(cid, {}) + title_seen = set() + results = [] + eligible = match = 0 + for j in jobs: + jid = str(j.get("id") or j.get("url")) + ok, in_ch, is_remote, is_relocation = location_eligibility( + j.get("location", ""), policy=policy + ) + if not ok: + continue + # Collapse the same role posted once per remote country (title differs only + # by a "| Country | Remote" suffix) — dedupe on the title before the first "|". + norm_title = re.sub(r"\s+", " ", (j.get("title") or "").split("|")[0]).strip().lower() + if norm_title in title_seen: + continue + title_seen.add(norm_title) + eligible += 1 + is_new = jid not in company_seen + score, pos, neg = score_job(j, title_only=bool(title_filter)) + # Pre-filtered boards (e.g. SBB) with German/generic titles: _score_floor keeps + # already-relevant results out of the hidden weak bucket. + floor = args.get("_score_floor") + if floor is not None and score < floor: + score = floor + if score >= 2: + match += 1 + results.append({ + "company": display, "company_id": cid, + "title": j["title"], "location": j["location"], + "url": j["url"], "posted": j.get("posted", ""), + "score": score, "pos": pos, "neg": neg, + "in_ch": in_ch, "remote": is_remote, "relocation": is_relocation, + "is_new": is_new, + }) + if is_new: + company_seen[jid] = {"title": j["title"], "first_seen": today} + + stat = { + "company": display, "scraped": scraped, "eligible": eligible, + "match": match, "newest": newest, "error": False, + } + return results, stat, warnings + + def main(): today = datetime.now(timezone.utc).strftime("%Y-%m-%d") @@ -1261,14 +1585,21 @@ def main(): url, status, note = rest[0], rest[1], " ".join(rest[2:]) decisions = load_decisions() prev = decisions.get(url, {}) - decisions[url] = {"company": prev.get("company", ""), "title": prev.get("title", ""), - "decision": status, "note": note, "date": today} + # Preserve any company/title already stored (filled when a report run tagged the URL). + decisions[url] = { + "company": prev.get("company", ""), + "title": prev.get("title", ""), + "decision": status, "note": note, "date": today, + } save_decisions(decisions) print(f"Recorded: {status} — {url}", file=sys.stderr) return only, new_only, include_weak, hide_decided = None, False, False, False for arg in sys.argv[1:]: + if arg in ("--help", "-h"): + print(__doc__) + return if arg == "--new-only": new_only = True elif arg == "--include-weak": @@ -1278,85 +1609,83 @@ def main(): elif arg.startswith("--only="): only = arg.split("=", 1)[1] + targets = [(cid, display, adapter, args) for cid, display, adapter, args in COMPANIES + if not only or cid == only] + if only and not targets: + known = ", ".join(c[0] for c in COMPANIES) + print(f"Unknown --only={only!r}. Known ids: {known}", file=sys.stderr) + sys.exit(2) + seen = load_seen() decisions = load_decisions() + last_scrape = load_last_scrape() all_results, errors, stats = [], [], [] + new_last_scrape = dict(last_scrape) run_start = time.perf_counter() - for cid, display, adapter, args in COMPANIES: - if only and cid != only: - continue - print(f"Fetching {display}...", file=sys.stderr) - t0 = time.perf_counter() - try: - jobs = ADAPTERS[adapter](args) - except (urllib.error.URLError, urllib.error.HTTPError, ValueError) as e: - errors.append((display, repr(e))) - stats.append({"company": display, "scraped": 0, "eligible": 0, - "match": 0, "newest": None, "secs": time.perf_counter() - t0, - "error": True}) - continue - except Exception as e: - errors.append((display, f"unexpected: {e!r}")) - stats.append({"company": display, "scraped": 0, "eligible": 0, - "match": 0, "newest": None, "secs": time.perf_counter() - t0, - "error": True}) - continue + # API adapters in parallel; playwright serial (shared browser is not thread-safe). + api_targets = [t for t in targets if t[2] != "playwright"] + pw_targets = [t for t in targets if t[2] == "playwright"] + # Preserve COMPANIES order in stats/report via ordered result map. + fetch_results = {} - scraped = len(jobs) - # Optional per-company title prefilter for high-volume boards - title_filter = args.get("_title_filter") - if title_filter: - jobs = [j for j in jobs - if any(_kw_in(k, (j.get("title") or "").lower()) for k in title_filter)] + try: + if api_targets: + print(f"Fetching {len(api_targets)} API boards " + f"({API_FETCH_WORKERS} workers)...", file=sys.stderr) + with ThreadPoolExecutor(max_workers=API_FETCH_WORKERS) as pool: + futs = { + pool.submit(_fetch_company, cid, display, adapter, args): cid + for cid, display, adapter, args in api_targets + } + for fut in as_completed(futs): + cid, display, adapter, args, jobs, err, secs = fut.result() + print(f" {display}: " + f"{'ERROR' if err else f'{len(jobs)} jobs'} ({secs:.1f}s)", + file=sys.stderr) + fetch_results[cid] = (display, args, jobs, err, secs) - # Newest posting on the board (board freshness), across parseable dates. - dates = [d for j in jobs if (d := _parse_posted(j.get("posted")))] - newest = max(dates) if dates else None + for cid, display, adapter, args in pw_targets: + print(f"Fetching {display} (playwright)...", file=sys.stderr) + _, display, _, args, jobs, err, secs = _fetch_company( + cid, display, adapter, args + ) + print(f" {display}: " + f"{'ERROR' if err else f'{len(jobs)} jobs'} ({secs:.1f}s)", + file=sys.stderr) + fetch_results[cid] = (display, args, jobs, err, secs) - company_seen = seen.setdefault(cid, {}) - title_seen = set() - eligible = match = 0 - for j in jobs: - jid = str(j.get("id") or j.get("url")) - in_ch, is_remote = location_matches(j.get("location", "")) - is_relocation = is_relocation_location(j.get("location", "")) - if not (in_ch or is_remote or is_relocation): + # Process in COMPANIES order for stable stats tables. + for cid, display, adapter, args in targets: + display, args, jobs, err, secs = fetch_results[cid] + if err is not None: + errors.append((display, err)) + stats.append({ + "company": display, "scraped": 0, "eligible": 0, "match": 0, + "newest": None, "secs": secs, "error": True, + }) continue - # Collapse the same role posted once per remote country (title differs only - # by a "| Country | Remote" suffix) — dedupe on the title before the first "|". - norm_title = re.sub(r"\s+", " ", (j.get("title") or "").split("|")[0]).strip().lower() - if norm_title in title_seen: - continue - title_seen.add(norm_title) - eligible += 1 - is_new = jid not in company_seen - score, pos, neg = score_job(j, title_only=bool(title_filter)) - # Pre-filtered boards (e.g. SBB, already narrowed to IT+Bern by the adapter) carry - # German/generic titles the profile scorer can't read; a _score_floor keeps their - # already-relevant results out of the hidden weak bucket. - floor = args.get("_score_floor") - if floor is not None and score < floor: - score = floor - if score >= 2: - match += 1 - all_results.append({ - "company": display, "company_id": cid, - "title": j["title"], "location": j["location"], - "url": j["url"], "posted": j.get("posted", ""), - "score": score, "pos": pos, "neg": neg, - "in_ch": in_ch, "remote": is_remote, "relocation": is_relocation, - "is_new": is_new, - }) - if is_new: - company_seen[jid] = {"title": j["title"], "first_seen": today} - stats.append({"company": display, "scraped": scraped, "eligible": eligible, - "match": match, "newest": newest, - "secs": time.perf_counter() - t0, "error": False}) + results, stat, warnings = _process_company( + cid, display, args, jobs, seen, today, last_scrape + ) + stat["secs"] = secs + for w in warnings: + errors.append(w) + # Soft warning only when 0 jobs — still mark row with ⚠️ if was previously >0 + prev = (last_scrape.get(cid) or {}).get("scraped", 0) + if stat["scraped"] == 0 and prev > 0: + stat["error"] = True + stats.append(stat) + all_results.extend(results) + new_last_scrape[cid] = { + "scraped": stat["scraped"], "date": today, "display": display, + } + finally: + save_seen(seen) + save_last_scrape(new_last_scrape) + _close_browser() - save_seen(seen) - _close_browser() total_secs = time.perf_counter() - run_start if new_only: @@ -1371,84 +1700,52 @@ def main(): decisions=decisions, hide_decided=hide_decided) # Plain data dump of every eligible role, unfiltered by keyword score — fit judgment - # for these is done by Claude in conversation against the profile memory, not by the - # keyword scorer (which is a rough legacy signal, kept below as "pos"/"neg" hints only). + # for these is done in conversation against the profile, not by the keyword scorer. json_path = REPORTS_DIR / f"{today}.json" dump = [{ "company": r["company"], "title": r["title"], "location": r["location"], "url": r["url"], "posted": r["posted"], "is_new": r["is_new"], "decision": decisions.get(r["url"]), "keyword_signal": {"score": r["score"], "pos": r["pos"], "neg": r["neg"]}, + "loc_flags": { + "in_ch": r["in_ch"], "remote": r["remote"], "relocation": r["relocation"], + }, } for r in all_results] - json_path.write_text(json.dumps(dump, indent=2, ensure_ascii=False), encoding="utf-8") + _atomic_write_json(json_path, dump) n_new = sum(1 for r in all_results if r["is_new"]) print(f"\nReport written: {report_path}", file=sys.stderr) print(f"Total matches: {len(all_results)} ({n_new} new) | " f"scanned {len(stats)} companies in {total_secs:.1f}s", file=sys.stderr) if errors: - print(f"Errors: {len(errors)} - see report", file=sys.stderr) + print(f"Errors/warnings: {len(errors)} - see report", file=sys.stderr) -# === Adapter coverage (refreshed 2026-06-06) ================================== -# 31 companies automated across 13 adapter types; 1 in MANUAL_CHECK (Oracle). +# === Adapter coverage (refreshed 2026-07-18) ================================== +# 34 companies automated across 14 adapter types; 1 in MANUAL_CHECK (Oracle). +# +# Location policy: +# default — CH + Denmark + Europe/EMEA/global remote (work-from CH or DK) +# norway_only — Equinor +# dk_no — NATO (Denmark + Norway only; no CH) # # Automated (COMPANIES above): -# workday nvidia, novartis +# workday nvidia, novartis, equinor (norway_only) # ashby kraken, openai, confluent, snowflake # greenhouse anthropic, gitlab, grafana, databricks, datadog, elastic, dbtlabs -# pcsx microsoft (Eightfold position-search endpoint) +# pcsx microsoft # smartrecruiters metgroup, ldc -# rss bis (vacancies.rss — RSS 1.0/RDF) -# getro coinbase_ventures (web3 portfolio network, collection 1625) -# onlyfy bitcoin_suisse (onlyfy.jobs ajax_list HTML fragment) -# lever palantir, quantco (api.lever.co; QuantCo slug is "quantco-") -# json swissgrid (Magnolia /.rest/cloud/component-data) -# sbb sbb (company.sbb.ch AEM jobfilter.results.json) -# bkw bkw (jobs.bkw.com PMS structureddata API) -# playwright google, apple, meta, roche, cisco, ruag (headless browser, 3-15s each) +# rss bis +# getro coinbase_ventures +# onlyfy bitcoin_suisse +# lever palantir, quantco +# taleo nato (dk_no; nato.taleo.net §2 — marketing page is nato.int/vacancies) +# json swissgrid +# sbb sbb +# bkw bkw +# playwright google, apple, meta, roche, cisco, ruag, postfinance, bfh # -# 2026-06-01 list review (verified live): -# - Palantir (lever): 221 postings, US/London-heavy so Swiss/Schwyz roles are rare but -# self-surface (FDSE/Deployment-Strategist titles map to his FDE drafts). -# - Swissgrid (json): Magnolia CMS endpoint; placeOfWork is bare city, so loc_suffix tags -# it Switzerland for the CH filter. ~13 roles incl. Data Scientist / Applied-ML. -# - RUAG (playwright + page_param): Drupal portal, 20 jobs/page, paginated ?page=N. Page 0 -# is apprenticeship-heavy; eng roles (DevOps/Data/Software) are on later pages, so we -# page through (max_pages). ENG_TITLE_FILTER cuts the Lehrstelle bulk. ⚠️ DE-citizen -# limits on RUAG classified roles — verify per-role. -# - SBB (sbb): correct host is company.sbb.ch (not company-jobs.sbb.ch). Flat JSON list; -# fetch_sbb replicates the user's IT + Bern-region filter. German/generic titles, so a -# _score_floor keeps the pre-filtered results visible. ⚠️ DE-citizen limits possible. -# - BKW (bkw): real host is jobs.bkw.com (PMS structureddata API), ~600 group-wide roles; -# fetch_bkw keeps Berufsfeld categories Informatik/Trading/Finanzen (IT/data + energy -# trading: Quant Risk, Solution Architect Energiehandel, ...). _score_floor as above. -# - QuantCo (lever, slug "quantco-"): ~16 roles, most tagged "Europe" (hybrid; Zürich is -# QuantCo's continental hub), surfaced via the EU-wide rule in location_matches. Strong: -# AI Engineer; medium: Cloud Engineer, AI Applied Scientist, Data Scientist, Quant -# Researcher, Software Engineer. Interns/frontend suppressed by NEGATIVE_KEYWORDS. -# The Bern/Thun tier intentionally relaxes the comp bar (see user_comp_bar memory). -# -# 2026-06-06 additions (FAANG-adjacent data-infra, Tier A/B from the Zürich/Bern review): -# - greenhouse: Databricks (board "databricks", 762 roles, Zürich SWE), Datadog ("datadog", -# Zürich branch + remote-EU), Elastic ("elastic", remote-first ELK), dbt Labs ("dbtlabsinc", -# remote-EU). ashby: Snowflake (slug "snowflake", 392 roles incl. Zürich "Observe by -# Snowflake" observability SWE). All title-filtered (ENG_TITLE_FILTER) — large boards. -# - ⚠️ Comp split: Databricks/Snowflake/Datadog pay Swiss-scale (Zürich offices, clears bar); -# Elastic/dbt are remote-EU and may be geo-banded below 180k CHF (like Grafana — verify). -# - HashiCorp: NOT added — IBM acquisition (2025) killed its public boards; on IBM careers now. -# - Oracle: in MANUAL_CHECK — ORC SPA resists scraping; REST endpoint documented there. -# -# MANUAL_CHECK: Oracle (ORC needs CH geographyId resolved). Dropped 2026-06-01: BFH -# (academic FH pay below the relaxed Bern/Thun floor, research-leaning, 403s anyway) and -# Dialectic (~50-person crypto VC, 0 open roles; crypto already covered by Kraken / Bitcoin -# Suisse / Coinbase Ventures). -# -# Earlier history: Google/Apple/Meta/Roche/Cisco automated via playwright; Microsoft via -# pcsx; BIS via rss; Coinbase Ventures via getro; Bitcoin Suisse via onlyfy. Dropped: -# ClickHouse, Vitol, Sygnum (Glassdoor/comp red flags), IBM Research + Sonova (low fit), -# Coinbase-the-employer (hiring freeze), AMINA (poor Glassdoor), Canonical (pay+culture). -# The Coinbase Ventures board (getro) covers PORTFOLIO companies, not Coinbase itself. +# MANUAL_CHECK: Oracle (ORC needs CH geographyId). # ============================================================================== diff --git a/job_scout/state/decisions.json b/job_scout/state/decisions.json index a770831..1a47e68 100644 --- a/job_scout/state/decisions.json +++ b/job_scout/state/decisions.json @@ -1650,5 +1650,19 @@ "decision": "skip", "note": "Claude judgment 2026-07-06: internship, comms", "date": "2026-07-06" + }, + "https://nato.taleo.net/careersection/2/jobdetail.ftl?job=261160&lang=en": { + "company": "NATO", + "title": "Staff Officer (2030 Digitalisation - Artificial Intelligence Engineer)", + "decision": "applied", + "note": "SENT 2026-07-10 via NTAP (Staff Officer AI Engineer, JWC Stavanger). See Agents.md Active Sessions.", + "date": "2026-07-18" + }, + "https://nato.taleo.net/careersection/2/jobdetail.ftl?job=261143&lang=en": { + "company": "NATO", + "title": "Staff Officer (2030 Digitalisation - Cloud Artificial Intelligence Architect)", + "decision": "skip", + "note": "Cloud AI Architect — above experience level (user judgment 2026-07-18).", + "date": "2026-07-18" } } \ No newline at end of file