Improve job scout location policy, reliability, and add NATO Taleo.
Tighten eligibility to CH/DK/Europe-remote (Equinor Norway-only, NATO DK+NO), add Taleo adapter, parallel API fetches, atomic state writes, and empty-board warnings. Log NATO AI Engineer as applied and Architect as skip.
This commit is contained in:
+511
-214
@@ -2,8 +2,18 @@
|
|||||||
|
|
||||||
Pulls latest openings from companies via public ATS APIs (Workday/Ashby/Greenhouse/
|
Pulls latest openings from companies via public ATS APIs (Workday/Ashby/Greenhouse/
|
||||||
SmartRecruiters/Lever/Eightfold/RSS) and, for JS-rendered careers sites, a headless-browser
|
SmartRecruiters/Lever/Eightfold/RSS) and, for JS-rendered careers sites, a headless-browser
|
||||||
(playwright) adapter. Filters by Swiss location or remote eligibility, scores fit against
|
(playwright) adapter. Filters by location policy, scores fit against profile keywords,
|
||||||
profile keywords, tracks which job IDs we've already seen, writes a markdown report.
|
tracks which job IDs we've already seen, writes a markdown report.
|
||||||
|
|
||||||
|
Location policy (default for most companies):
|
||||||
|
- Switzerland onsite/hybrid (CH)
|
||||||
|
- Denmark onsite/reloc
|
||||||
|
- CH-remote and Europe/EMEA/global remote (work-from CH or DK is realistic)
|
||||||
|
- NOT multi-country allowlists that omit CH/DK/Europe (e.g. "UK | Brazil | Ireland | ...")
|
||||||
|
|
||||||
|
Company overrides via args["_location_policy"]:
|
||||||
|
- "norway_only" — Equinor: Norway only
|
||||||
|
- "dk_no" — NATO-style: Denmark + Norway only (no CH — no NATO seat in CH)
|
||||||
|
|
||||||
Usage:
|
Usage:
|
||||||
py scout.py # Pull all configured companies (strong + medium only)
|
py scout.py # Pull all configured companies (strong + medium only)
|
||||||
@@ -14,8 +24,8 @@ Usage:
|
|||||||
py scout.py --decide "<url>" <status> [note...] # Record a decision and exit
|
py scout.py --decide "<url>" <status> [note...] # Record a decision and exit
|
||||||
# status is free-text: shortlist | skip | applied | paused | ...
|
# status is free-text: shortlist | skip | applied | paused | ...
|
||||||
|
|
||||||
State : state/seen_jobs.json (job IDs seen) · state/decisions.json (per-URL decisions)
|
State : state/seen_jobs.json · state/decisions.json · state/last_scrape.json
|
||||||
Output: reports/YYYY-MM-DD.md (scan-stats table + scored roles, decisions tagged inline)
|
Output: reports/YYYY-MM-DD.md (+ .json dump of every eligible role)
|
||||||
|
|
||||||
To add a company: append to COMPANIES with one of the existing adapter types. A few sites
|
To add a company: append to COMPANIES with one of the existing adapter types. A few sites
|
||||||
resist scraping even headless and stay in MANUAL_CHECK (surfaced as a report checklist).
|
resist scraping even headless and stay in MANUAL_CHECK (surfaced as a report checklist).
|
||||||
@@ -23,10 +33,12 @@ See the adapter-coverage notes at the bottom for the current automated/manual sp
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
import os
|
||||||
import re
|
import re
|
||||||
import ssl
|
import ssl
|
||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||||
from functools import lru_cache
|
from functools import lru_cache
|
||||||
import urllib.error
|
import urllib.error
|
||||||
import urllib.parse
|
import urllib.parse
|
||||||
@@ -37,39 +49,38 @@ from pathlib import Path
|
|||||||
ROOT = Path(__file__).parent
|
ROOT = Path(__file__).parent
|
||||||
STATE_FILE = ROOT / "state" / "seen_jobs.json"
|
STATE_FILE = ROOT / "state" / "seen_jobs.json"
|
||||||
DECISIONS_FILE = ROOT / "state" / "decisions.json"
|
DECISIONS_FILE = ROOT / "state" / "decisions.json"
|
||||||
|
LAST_SCRAPE_FILE = ROOT / "state" / "last_scrape.json"
|
||||||
REPORTS_DIR = ROOT / "reports"
|
REPORTS_DIR = ROOT / "reports"
|
||||||
USER_AGENT = "Mozilla/5.0 (compatible; job-scout/0.1)"
|
USER_AGENT = "Mozilla/5.0 (compatible; job-scout/0.1)"
|
||||||
|
API_FETCH_WORKERS = 8
|
||||||
|
|
||||||
CH_LOCATION_KEYWORDS = [
|
CH_LOCATION_KEYWORDS = [
|
||||||
"switzerland", "zurich", "zürich", "basel", "bern", "geneva", "genf",
|
"switzerland", "zurich", "zürich", "basel", "bern", "geneva", "genf",
|
||||||
"lausanne", "zug", "rüschlikon", "stäfa", "schweiz", "suisse",
|
"lausanne", "zug", "rüschlikon", "stäfa", "schweiz", "suisse",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
DK_LOCATION_KEYWORDS = [
|
||||||
|
"denmark", "copenhagen", "danmark", "københavn", "koebenhavn",
|
||||||
|
]
|
||||||
|
|
||||||
|
NO_LOCATION_KEYWORDS = [
|
||||||
|
"norway", "oslo", "stavanger", "bergen", "trondheim", "norge",
|
||||||
|
]
|
||||||
|
|
||||||
US_ONLY_PATTERNS = [
|
US_ONLY_PATTERNS = [
|
||||||
"remote - us", "remote, us", "remote-us", "us remote", "us-remote",
|
"remote - us", "remote, us", "remote-us", "us remote", "us-remote",
|
||||||
"remote-friendly us", "remote (us)", "united states - remote",
|
"remote-friendly us", "remote (us)", "united states - remote",
|
||||||
"remote, united states",
|
"remote, united states",
|
||||||
]
|
]
|
||||||
|
|
||||||
# Pan-regional/global remote — genuinely reachable from Switzerland regardless of which
|
# Pan-regional remote scope: work-from CH or DK is realistic.
|
||||||
# single country the listing happens to name.
|
EUROPE_SCOPE_KEYWORDS = [
|
||||||
GLOBAL_REMOTE_KEYWORDS = ["europe", "emea", "global", "worldwide"]
|
"europe", "emea", "european", "eu-wide", "eu wide", "remote europe",
|
||||||
|
"europe remote", "remote - europe", "remote, europe",
|
||||||
# Single-country hints, grouped so synonyms ("uk"/"united kingdom") count as one country.
|
]
|
||||||
# A listing naming only ONE of these is a residency lock (e.g. Databricks' "Remote - United
|
GLOBAL_SCOPE_KEYWORDS = [
|
||||||
# Kingdom Remote - United Kingdom"), not something open to a CH-based candidate, even though
|
"global", "worldwide", "work from anywhere", "anywhere in the world",
|
||||||
# it reads as "remote." Naming 2+ reads as a broad multi-country remote policy instead (e.g.
|
|
||||||
# Grafana's "Germany | Ireland | Spain | Sweden | UK") — that's treated as eligible.
|
|
||||||
COUNTRY_HINT_GROUPS = [
|
|
||||||
("germany",), ("france",), ("spain",), ("portugal",), ("ireland",),
|
|
||||||
("netherlands",), ("sweden",), ("norway",), ("finland",), ("poland",),
|
|
||||||
("czech",), ("romania",), ("italy",), ("austria",), ("belgium",),
|
|
||||||
("uk", "united kingdom"),
|
|
||||||
]
|
]
|
||||||
|
|
||||||
# Countries outside CH where relocation is genuinely on the table for a strong enough offer
|
|
||||||
# (see project_target_companies memory) — accepted for every company, on-site or remote.
|
|
||||||
RELOCATION_COUNTRY_KEYWORDS = ["denmark", "copenhagen"]
|
|
||||||
|
|
||||||
POSITIVE_KEYWORDS = {
|
POSITIVE_KEYWORDS = {
|
||||||
"genai": 3, "generative ai": 3, "llm": 3, "large language model": 3,
|
"genai": 3, "generative ai": 3, "llm": 3, "large language model": 3,
|
||||||
@@ -172,7 +183,7 @@ COMPANIES = [
|
|||||||
# Added 2026-06-06 (Tier A/B data-infra). Databricks/Snowflake/Datadog have Zürich offices
|
# Added 2026-06-06 (Tier A/B data-infra). Databricks/Snowflake/Datadog have Zürich offices
|
||||||
# (Swiss-scale comp, clears bar); Elastic/dbt Labs are remote-EU (verify CH-equiv comp —
|
# (Swiss-scale comp, clears bar); Elastic/dbt Labs are remote-EU (verify CH-equiv comp —
|
||||||
# may be geo-banded below 180k, like Grafana). All title-filtered (boards are 160-760 roles).
|
# may be geo-banded below 180k, like Grafana). All title-filtered (boards are 160-760 roles).
|
||||||
("databricks","Databricks","greenhouse", {"board": "databricks", "_title_filter": ENG_TITLE_FILTER}), # Zürich SWE + remote-EU; Denmark relocation handled globally, see RELOCATION_COUNTRY_KEYWORDS
|
("databricks","Databricks","greenhouse", {"board": "databricks", "_title_filter": ENG_TITLE_FILTER}), # Zürich SWE + remote-EU; DK via default location policy
|
||||||
("snowflake", "Snowflake", "ashby", {"slug": "snowflake", "_title_filter": ENG_TITLE_FILTER}), # Zürich "Observe" observability SWE roles
|
("snowflake", "Snowflake", "ashby", {"slug": "snowflake", "_title_filter": ENG_TITLE_FILTER}), # Zürich "Observe" observability SWE roles
|
||||||
("datadog", "Datadog", "greenhouse", {"board": "datadog", "_title_filter": ENG_TITLE_FILTER}), # Zürich branch + remote-EU
|
("datadog", "Datadog", "greenhouse", {"board": "datadog", "_title_filter": ENG_TITLE_FILTER}), # Zürich branch + remote-EU
|
||||||
("elastic", "Elastic", "greenhouse", {"board": "elastic", "_title_filter": ENG_TITLE_FILTER}), # remote-first; ELK = his stack
|
("elastic", "Elastic", "greenhouse", {"board": "elastic", "_title_filter": ENG_TITLE_FILTER}), # remote-first; ELK = his stack
|
||||||
@@ -182,15 +193,31 @@ COMPANIES = [
|
|||||||
# Dropped: Sygnum (Glassdoor 3.4, 51% recommend, comp 2.3/5 — below 180k bar — 2026-05).
|
# Dropped: Sygnum (Glassdoor 3.4, 51% recommend, comp 2.3/5 — below 180k bar — 2026-05).
|
||||||
("metgroup", "MET Group", "smartrecruiters", {"company": "METGroup", "_title_filter": ENG_TITLE_FILTER}),
|
("metgroup", "MET Group", "smartrecruiters", {"company": "METGroup", "_title_filter": ENG_TITLE_FILTER}),
|
||||||
("ldc", "Louis Dreyfus","smartrecruiters",{"company": "LouisDreyfusCompany", "_title_filter": ENG_TITLE_FILTER}),
|
("ldc", "Louis Dreyfus","smartrecruiters",{"company": "LouisDreyfusCompany", "_title_filter": ENG_TITLE_FILTER}),
|
||||||
# Equinor (Workday) — Norway energy major; he's lived/worked in NO before (Equinor AI
|
# Equinor (Workday) — Norway energy major; lived/worked in NO before. Outlier location
|
||||||
# Architect req was applied to manually — see CLAUDE.md Active Sessions) and it's an
|
# policy: Norway only (not CH / Europe-remote). Small board (~15); no title filter.
|
||||||
# energy-trading-adjacent target. Live board is small (~15 total) and mostly NO/US/BR
|
|
||||||
# on-site ops/trading roles, so no server-side search_text filter or title filter —
|
|
||||||
# let location_matches do the CH/remote-eligibility work like the other low-volume boards.
|
|
||||||
("equinor", "Equinor", "workday", {
|
("equinor", "Equinor", "workday", {
|
||||||
"host": "equinor.wd3.myworkdayjobs.com",
|
"host": "equinor.wd3.myworkdayjobs.com",
|
||||||
"tenant": "equinor",
|
"tenant": "equinor",
|
||||||
"site": "EQNR",
|
"site": "EQNR",
|
||||||
|
"_location_policy": "norway_only",
|
||||||
|
# Norway-only filter already scopes the board; titles are often generic/energy
|
||||||
|
# ops and score 0 under the English eng keyword list — keep them visible.
|
||||||
|
"_score_floor": 2,
|
||||||
|
}),
|
||||||
|
# NATO civilian vacancies (Oracle Taleo). Public marketing page is
|
||||||
|
# https://www.nato.int/en/work-with-us/careers/vacancies — jobs actually live on
|
||||||
|
# nato.taleo.net career section 2. Location policy dk_no: Denmark + Norway only
|
||||||
|
# (no NATO seat in CH). Server-side location taxonomy IDs filter the board before
|
||||||
|
# parse; dk_no is a second safety net on the location string.
|
||||||
|
# Taleo option values (jobsearch.ftl dropdown): Norway=14470035151, Denmark=16270035151.
|
||||||
|
("nato", "NATO", "taleo", {
|
||||||
|
"host": "nato.taleo.net",
|
||||||
|
"career_section": "2",
|
||||||
|
"location_ids": ["14470035151", "16270035151"], # Norway, Denmark
|
||||||
|
"_location_policy": "dk_no",
|
||||||
|
# Staff Officer / NATO-body titles score unevenly under eng keywords — keep
|
||||||
|
# pre-filtered NO/DK roles visible (same pattern as Equinor/SBB).
|
||||||
|
"_score_floor": 2,
|
||||||
}),
|
}),
|
||||||
# International org — BIS (Basel), commutable from Bern, salary net of Swiss tax.
|
# International org — BIS (Basel), commutable from Bern, salary net of Swiss tax.
|
||||||
# Low-volume RSS feed; no title filter (Innovation Hub roles can be oddly titled).
|
# Low-volume RSS feed; no title filter (Innovation Hub roles can be oddly titled).
|
||||||
@@ -288,10 +315,9 @@ COMPANIES = [
|
|||||||
# Deployment Strategist) aren't in ENG_TITLE_FILTER, so filtering would hide them.
|
# Deployment Strategist) aren't in ENG_TITLE_FILTER, so filtering would hide them.
|
||||||
("palantir", "Palantir", "lever", {"slug": "palantir"}),
|
("palantir", "Palantir", "lever", {"slug": "palantir"}),
|
||||||
# QuantCo (Lever — note the trailing-hyphen slug "quantco-"). ~16 roles, most tagged
|
# QuantCo (Lever — note the trailing-hyphen slug "quantco-"). ~16 roles, most tagged
|
||||||
# "Europe" (hybrid); QuantCo's continental hub is Zürich, so the EU-wide rule in
|
# "Europe" (hybrid); QuantCo's continental hub is Zürich, so Europe-scope in
|
||||||
# location_matches surfaces them. No title filter: the target band is DS/Quant/AI/Cloud
|
# location_eligibility surfaces them. No title filter: target band is DS/Quant/AI/Cloud
|
||||||
# (see comp analysis), which ENG_TITLE_FILTER would drop; interns/frontend are caught by
|
# (ENG_TITLE_FILTER would drop some); interns/frontend caught by NEGATIVE_KEYWORDS.
|
||||||
# NEGATIVE_KEYWORDS instead.
|
|
||||||
("quantco", "QuantCo", "lever", {"slug": "quantco-"}),
|
("quantco", "QuantCo", "lever", {"slug": "quantco-"}),
|
||||||
# --- Bern/Thun local tier — WLB & proximity exception (comp bar relaxed; 2026-06-01) ---
|
# --- Bern/Thun local tier — WLB & proximity exception (comp bar relaxed; 2026-06-01) ---
|
||||||
# Wired after live endpoint discovery. ⚠️ German citizen: RUAG classified work may require
|
# Wired after live endpoint discovery. ⚠️ German citizen: RUAG classified work may require
|
||||||
@@ -375,12 +401,8 @@ COMPANIES = [
|
|||||||
}),
|
}),
|
||||||
]
|
]
|
||||||
|
|
||||||
# Companies where adapter probing did not yield a reliable scrape. Reasons noted.
|
# Companies that resist scraping — clickable checklist in the report.
|
||||||
# These surface as a clickable checklist in the report so they're not forgotten.
|
# (BFH re-automated 2026-07-14 on www.bfh.ch. Dialectic dropped — crypto covered elsewhere.)
|
||||||
# Companies that resist scraping stay here as a clickable report checklist. Currently empty —
|
|
||||||
# every target company is automated. (Dropped 2026-06-01: BFH — academic FH pay below even the
|
|
||||||
# relaxed Bern/Thun floor, research-leaning, 403s anyway; Dialectic — ~50-person crypto VC,
|
|
||||||
# 0 open roles, crypto angle already covered by Kraken/Bitcoin Suisse/Coinbase Ventures.)
|
|
||||||
MANUAL_CHECK = [
|
MANUAL_CHECK = [
|
||||||
# Oracle (Tier C, requested 2026-06-06). Oracle Recruiting Cloud (ORC) resists clean
|
# Oracle (Tier C, requested 2026-06-06). Oracle Recruiting Cloud (ORC) resists clean
|
||||||
# scraping: careers.oracle.com renders job tiles that are NOT anchors, so the playwright
|
# scraping: careers.oracle.com renders job tiles that are NOT anchors, so the playwright
|
||||||
@@ -473,8 +495,12 @@ def fetch_ashby(args):
|
|||||||
|
|
||||||
|
|
||||||
def fetch_greenhouse(args):
|
def fetch_greenhouse(args):
|
||||||
|
# Title-filtered boards score title_only — skip heavy JD HTML (content=true) for those.
|
||||||
board = args["board"]
|
board = args["board"]
|
||||||
url = f"https://boards-api.greenhouse.io/v1/boards/{board}/jobs?content=true"
|
want_content = not args.get("_title_filter")
|
||||||
|
url = f"https://boards-api.greenhouse.io/v1/boards/{board}/jobs"
|
||||||
|
if want_content:
|
||||||
|
url += "?content=true"
|
||||||
data = http_get_json(url)
|
data = http_get_json(url)
|
||||||
jobs = []
|
jobs = []
|
||||||
for j in data.get("jobs", []):
|
for j in data.get("jobs", []):
|
||||||
@@ -482,9 +508,11 @@ def fetch_greenhouse(args):
|
|||||||
offices = j.get("offices") or []
|
offices = j.get("offices") or []
|
||||||
office_names = " | ".join(o.get("name", "") for o in offices if isinstance(o, dict))
|
office_names = " | ".join(o.get("name", "") for o in offices if isinstance(o, dict))
|
||||||
loc_blob = " ".join(x for x in [loc, office_names] if x)
|
loc_blob = " ".join(x for x in [loc, office_names] if x)
|
||||||
desc = j.get("content", "") or ""
|
desc = ""
|
||||||
desc = re.sub(r"<[^>]+>", " ", desc)
|
if want_content:
|
||||||
desc = re.sub(r"\s+", " ", desc).strip()
|
desc = j.get("content", "") or ""
|
||||||
|
desc = re.sub(r"<[^>]+>", " ", desc)
|
||||||
|
desc = re.sub(r"\s+", " ", desc).strip()
|
||||||
jobs.append({
|
jobs.append({
|
||||||
"id": str(j.get("id")),
|
"id": str(j.get("id")),
|
||||||
"title": j.get("title", ""),
|
"title": j.get("title", ""),
|
||||||
@@ -587,25 +615,6 @@ def fetch_rss(args):
|
|||||||
return jobs
|
return jobs
|
||||||
|
|
||||||
|
|
||||||
def fetch_wp_ajax(args):
|
|
||||||
"""WordPress admin-ajax style endpoint. Sygnum uses this pattern."""
|
|
||||||
url = args["url"]
|
|
||||||
data = http_get_json(url)
|
|
||||||
if not isinstance(data, list):
|
|
||||||
return []
|
|
||||||
jobs = []
|
|
||||||
for j in data:
|
|
||||||
jobs.append({
|
|
||||||
"id": (j.get("title", "") + "|" + j.get("location", ""))[:120],
|
|
||||||
"title": j.get("title", ""),
|
|
||||||
"location": " ".join(filter(None, [j.get("location", ""), j.get("work_type", "")])),
|
|
||||||
"url": j.get("application_url") or args["url"],
|
|
||||||
"posted": "",
|
|
||||||
"description": " ".join(filter(None, [j.get("department", ""), j.get("role_type", "")])),
|
|
||||||
})
|
|
||||||
return jobs
|
|
||||||
|
|
||||||
|
|
||||||
def fetch_getro(args):
|
def fetch_getro(args):
|
||||||
"""Getro network job-board search API (POST JSON). Powers VC portfolio talent
|
"""Getro network job-board search API (POST JSON). Powers VC portfolio talent
|
||||||
networks — here the Coinbase Ventures web3 network (collection 1625). Returns roles
|
networks — here the Coinbase Ventures web3 network (collection 1625). Returns roles
|
||||||
@@ -669,10 +678,15 @@ def fetch_onlyfy(args):
|
|||||||
page = resp.read().decode("utf-8", "replace")
|
page = resp.read().decode("utf-8", "replace")
|
||||||
titles = re.findall(r'<a href="(/job/[a-z0-9]+)">(.*?)</a>', page, re.S)
|
titles = re.findall(r'<a href="(/job/[a-z0-9]+)">(.*?)</a>', page, re.S)
|
||||||
locs = re.findall(r'icon-map-marker[^>]*></i>\s*([^<]+)', page)
|
locs = re.findall(r'icon-map-marker[^>]*></i>\s*([^<]+)', page)
|
||||||
|
if titles and locs and len(titles) != len(locs):
|
||||||
|
# Misaligned regex pairs would invent wrong locations; keep titles, drop bad locs.
|
||||||
|
print(f" onlyfy: title/loc count mismatch ({len(titles)} vs {len(locs)}); "
|
||||||
|
f"using titles only", file=sys.stderr)
|
||||||
|
locs = [""] * len(titles)
|
||||||
jobs = []
|
jobs = []
|
||||||
for (href, raw_title), raw_loc in zip(titles, locs):
|
for (href, raw_title), raw_loc in zip(titles, locs if locs else [""] * len(titles)):
|
||||||
title = _html.unescape(re.sub(r"<[^>]+>", "", raw_title)).strip()
|
title = _html.unescape(re.sub(r"<[^>]+>", "", raw_title)).strip()
|
||||||
loc = _html.unescape(raw_loc).strip()
|
loc = _html.unescape(raw_loc).strip() if raw_loc else ""
|
||||||
jobs.append({
|
jobs.append({
|
||||||
"id": href.rsplit("/", 1)[-1],
|
"id": href.rsplit("/", 1)[-1],
|
||||||
"title": title,
|
"title": title,
|
||||||
@@ -711,6 +725,141 @@ def fetch_lever(args):
|
|||||||
return jobs
|
return jobs
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_taleo_jobsearch_html(html, host, career_section):
|
||||||
|
"""Extract jobs from a Taleo jobsearch.ftl HTML response.
|
||||||
|
|
||||||
|
Taleo does not expose a reliable public JSON search API for this board (REST
|
||||||
|
returns careerSectionUnAvailable). Jobs are embedded in the page as
|
||||||
|
api.fillInterface arrays and as a pipe-delimited initialHistory field.
|
||||||
|
"""
|
||||||
|
import html as _html
|
||||||
|
|
||||||
|
jobs, seen = [], set()
|
||||||
|
detail_base = f"https://{host}/careersection/{career_section}/jobdetail.ftl"
|
||||||
|
|
||||||
|
# Quoted fillInterface form (already-unescaped dates in modern Taleo builds).
|
||||||
|
# Observed record shape (NATO 2026-07):
|
||||||
|
# '<id>','<title>','<id>','<title>', five more '<id>',
|
||||||
|
# '<jobNo>','<Country-City>','false','','','','',
|
||||||
|
# '<deadline>','<NATO body>','<grade>'
|
||||||
|
pat = re.compile(
|
||||||
|
r"'(\d+)','((?:[^'\\]|\\.)+)',"
|
||||||
|
r"'\1','\2',"
|
||||||
|
r"(?:'\1',){5}"
|
||||||
|
r"'(\d+)','((?:[^'\\]|\\.)+)',"
|
||||||
|
r"'false','','','','',"
|
||||||
|
r"'((?:[^'\\]|\\.)*)','((?:[^'\\]|\\.)*)','((?:[^'\\]|\\.)*)'"
|
||||||
|
)
|
||||||
|
for m in pat.finditer(html):
|
||||||
|
job_no = m.group(3)
|
||||||
|
if job_no in seen:
|
||||||
|
continue
|
||||||
|
seen.add(job_no)
|
||||||
|
|
||||||
|
def _unq(s):
|
||||||
|
return _html.unescape(s.replace("\\'", "'").replace("\\:", ":"))
|
||||||
|
|
||||||
|
title = _unq(m.group(2))
|
||||||
|
loc = _unq(m.group(4)).replace("-", ", ")
|
||||||
|
deadline = _unq(m.group(5))
|
||||||
|
body = _unq(m.group(6))
|
||||||
|
grade = _unq(m.group(7))
|
||||||
|
# List view exposes application deadline only (not a posted-on date). We store
|
||||||
|
# the date part in `posted` so the report surfaces the closing date.
|
||||||
|
deadline_short = deadline.split(",")[0].strip() if deadline else ""
|
||||||
|
jobs.append({
|
||||||
|
"id": job_no,
|
||||||
|
"title": title,
|
||||||
|
"location": loc,
|
||||||
|
"url": f"{detail_base}?job={job_no}&lang=en",
|
||||||
|
"posted": deadline_short,
|
||||||
|
"description": f"{body} {grade}".strip()[:500],
|
||||||
|
})
|
||||||
|
if jobs:
|
||||||
|
return jobs
|
||||||
|
|
||||||
|
# Fallback: pipe-delimited initialHistory hidden field
|
||||||
|
m = re.search(
|
||||||
|
r'(?:name|id)="initialHistory"[^>]*value="([^"]*)"', html, re.I
|
||||||
|
) or re.search(
|
||||||
|
r'value="([^"]*)"[^>]*(?:name|id)="initialHistory"', html, re.I
|
||||||
|
)
|
||||||
|
if not m:
|
||||||
|
return []
|
||||||
|
hist = urllib.parse.unquote(m.group(1).replace("%5C:", ":").replace("\\:", ":"))
|
||||||
|
parts = hist.split("!|!")
|
||||||
|
for i, p in enumerate(parts):
|
||||||
|
if not re.fullmatch(r"\d{5,6}", p) or i + 1 >= len(parts):
|
||||||
|
continue
|
||||||
|
loc_raw = parts[i + 1]
|
||||||
|
if not re.search(
|
||||||
|
r"Norway|Denmark|Belgium|Germany|Italy|France|United|Luxembourg|"
|
||||||
|
r"Netherlands|Poland|Spain|Portugal|Canada|Turkey|Türkiye",
|
||||||
|
loc_raw, re.I,
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
job_no = p
|
||||||
|
if job_no in seen:
|
||||||
|
continue
|
||||||
|
title = ""
|
||||||
|
for j in range(i - 1, max(-1, i - 12), -1):
|
||||||
|
cand = parts[j]
|
||||||
|
if (cand and not re.fullmatch(r"\d+", cand)
|
||||||
|
and cand not in ("false", "true", "Apply")
|
||||||
|
and len(cand) > 8):
|
||||||
|
title = cand
|
||||||
|
break
|
||||||
|
if not title:
|
||||||
|
continue
|
||||||
|
seen.add(job_no)
|
||||||
|
deadline = parts[i + 6] if i + 6 < len(parts) else ""
|
||||||
|
body = parts[i + 7] if i + 7 < len(parts) else ""
|
||||||
|
grade = parts[i + 8] if i + 8 < len(parts) else ""
|
||||||
|
deadline_short = deadline.split(",")[0].strip() if deadline else ""
|
||||||
|
jobs.append({
|
||||||
|
"id": job_no,
|
||||||
|
"title": _html.unescape(title),
|
||||||
|
"location": loc_raw.replace("-", ", "),
|
||||||
|
"url": f"{detail_base}?job={job_no}&lang=en",
|
||||||
|
"posted": deadline_short,
|
||||||
|
"description": f"{body} {grade}".strip()[:500],
|
||||||
|
})
|
||||||
|
return jobs
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_taleo(args):
|
||||||
|
"""Oracle Taleo careersection board (used by NATO civilian vacancies).
|
||||||
|
|
||||||
|
The marketing page https://www.nato.int/en/work-with-us/careers/vacancies is an
|
||||||
|
AEM shell; the live board is Taleo. Public REST search is unavailable, so we
|
||||||
|
GET jobsearch.ftl (optionally filtered by Taleo location taxonomy IDs from the
|
||||||
|
board's location dropdown) and parse embedded job records.
|
||||||
|
|
||||||
|
Args: host (default nato.taleo.net), career_section (default "2"),
|
||||||
|
location_ids (optional list of Taleo option values — e.g. Norway/Denmark).
|
||||||
|
"""
|
||||||
|
host = args.get("host", "nato.taleo.net")
|
||||||
|
section = str(args.get("career_section", "2"))
|
||||||
|
location_ids = args.get("location_ids")
|
||||||
|
# One request per location id (Taleo query param takes a single id); merge by job no.
|
||||||
|
# If no location_ids, one unfiltered first-page fetch.
|
||||||
|
fetches = location_ids if location_ids else [None]
|
||||||
|
by_id = {}
|
||||||
|
for loc_id in fetches:
|
||||||
|
url = f"https://{host}/careersection/{section}/jobsearch.ftl?lang=en"
|
||||||
|
if loc_id:
|
||||||
|
url += f"&location={urllib.parse.quote(str(loc_id))}"
|
||||||
|
req = urllib.request.Request(url, headers={
|
||||||
|
"User-Agent": USER_AGENT,
|
||||||
|
"Accept": "text/html,application/xhtml+xml",
|
||||||
|
})
|
||||||
|
with urllib.request.urlopen(req, timeout=60, context=_ssl_context()) as resp:
|
||||||
|
html = resp.read().decode("utf-8", "replace")
|
||||||
|
for j in _parse_taleo_jobsearch_html(html, host, section):
|
||||||
|
by_id[j["id"]] = j
|
||||||
|
return list(by_id.values())
|
||||||
|
|
||||||
|
|
||||||
def fetch_json(args):
|
def fetch_json(args):
|
||||||
"""Generic JSON jobs API with configurable field names, for employer sites that expose
|
"""Generic JSON jobs API with configurable field names, for employer sites that expose
|
||||||
a clean public endpoint. Verified use: Swissgrid (Magnolia CMS
|
a clean public endpoint. Verified use: Swissgrid (Magnolia CMS
|
||||||
@@ -867,6 +1016,40 @@ def _absolutize(href, prefix):
|
|||||||
return prefix.rstrip("/") + "/" + cleaned
|
return prefix.rstrip("/") + "/" + cleaned
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_location_from_blob(full, default=""):
|
||||||
|
"""Pull a short location line out of a job-card text blob.
|
||||||
|
|
||||||
|
Google/Meta/Roche-style cards often dump Material-icon chrome into inner_text
|
||||||
|
(e.g. 'corporate_fare' / 'place' / 'Zürich, Switzerland' / 'bar_chart'). Prefer the
|
||||||
|
line after a 'place'/'location' label, else the first short line that looks like a
|
||||||
|
place, else the configured default — never the first 300 chars of the whole card.
|
||||||
|
"""
|
||||||
|
if not full:
|
||||||
|
return default
|
||||||
|
lines = [ln.strip() for ln in full.splitlines() if ln.strip()]
|
||||||
|
skip = {
|
||||||
|
"place", "location", "corporate_fare", "bar_chart", "work", "schedule",
|
||||||
|
"google", "meta", "apple", "roche", "cisco",
|
||||||
|
}
|
||||||
|
for i, ln in enumerate(lines):
|
||||||
|
if ln.lower() in ("place", "location") and i + 1 < len(lines):
|
||||||
|
cand = lines[i + 1].strip()
|
||||||
|
if cand and cand.lower() not in skip and len(cand) < 120:
|
||||||
|
return cand[:200]
|
||||||
|
place_hints = (
|
||||||
|
CH_LOCATION_KEYWORDS + DK_LOCATION_KEYWORDS + NO_LOCATION_KEYWORDS
|
||||||
|
+ list(EUROPE_SCOPE_KEYWORDS) + list(GLOBAL_SCOPE_KEYWORDS)
|
||||||
|
+ ["remote", "hybrid", "onsite", "on-site"]
|
||||||
|
)
|
||||||
|
for ln in lines:
|
||||||
|
low = ln.lower()
|
||||||
|
if len(ln) > 120 or low in skip:
|
||||||
|
continue
|
||||||
|
if any(k in low for k in place_hints):
|
||||||
|
return ln[:200]
|
||||||
|
return default
|
||||||
|
|
||||||
|
|
||||||
def _close_browser():
|
def _close_browser():
|
||||||
if _playwright_singleton["browser"]:
|
if _playwright_singleton["browser"]:
|
||||||
try:
|
try:
|
||||||
@@ -954,11 +1137,18 @@ def fetch_playwright(args):
|
|||||||
added += 1
|
added += 1
|
||||||
description = ""
|
description = ""
|
||||||
if args.get("use_inner_text_as_blob"):
|
if args.get("use_inner_text_as_blob"):
|
||||||
# Use the full card text as both location source and description
|
# Full card text for keyword scoring; location via structured extract.
|
||||||
full = (card.inner_text() or "")
|
full = (card.inner_text() or "")
|
||||||
description = full[:2000]
|
description = full[:2000]
|
||||||
if not location:
|
extracted = _extract_location_from_blob(
|
||||||
location = full[:300]
|
full, default=args.get("default_location", "")
|
||||||
|
)
|
||||||
|
# Prefer a real location line over a default that is empty or a
|
||||||
|
# polluted leftover; keep default_location when extract finds nothing.
|
||||||
|
if extracted:
|
||||||
|
location = extracted
|
||||||
|
elif not location:
|
||||||
|
location = args.get("default_location", "")
|
||||||
jobs.append({
|
jobs.append({
|
||||||
"id": jid,
|
"id": jid,
|
||||||
"title": title,
|
"title": title,
|
||||||
@@ -1028,12 +1218,12 @@ ADAPTERS = {
|
|||||||
"ashby": fetch_ashby,
|
"ashby": fetch_ashby,
|
||||||
"greenhouse": fetch_greenhouse,
|
"greenhouse": fetch_greenhouse,
|
||||||
"pcsx": fetch_pcsx,
|
"pcsx": fetch_pcsx,
|
||||||
"wp_ajax": fetch_wp_ajax,
|
|
||||||
"smartrecruiters": fetch_smartrecruiters,
|
"smartrecruiters": fetch_smartrecruiters,
|
||||||
"rss": fetch_rss,
|
"rss": fetch_rss,
|
||||||
"getro": fetch_getro,
|
"getro": fetch_getro,
|
||||||
"onlyfy": fetch_onlyfy,
|
"onlyfy": fetch_onlyfy,
|
||||||
"lever": fetch_lever,
|
"lever": fetch_lever,
|
||||||
|
"taleo": fetch_taleo,
|
||||||
"json": fetch_json,
|
"json": fetch_json,
|
||||||
"sbb": fetch_sbb,
|
"sbb": fetch_sbb,
|
||||||
"bkw": fetch_bkw,
|
"bkw": fetch_bkw,
|
||||||
@@ -1041,26 +1231,50 @@ ADAPTERS = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def location_matches(loc_text):
|
def location_eligibility(loc_text, policy="default"):
|
||||||
|
"""Decide whether a posting's location is in-scope.
|
||||||
|
|
||||||
|
Returns (eligible, in_ch, is_remote, is_relocation).
|
||||||
|
|
||||||
|
Default policy (most companies):
|
||||||
|
- CH onsite/hybrid
|
||||||
|
- Denmark onsite/reloc
|
||||||
|
- Europe/EMEA/global remote scope (work-from CH or DK is realistic)
|
||||||
|
- CH-remote (Switzerland + remote wording)
|
||||||
|
- Multi-country allowlists WITHOUT CH/DK/Europe/global scope are NOT eligible
|
||||||
|
(e.g. Kraken "UK | Brazil | Ireland | ..." with no Switzerland/Europe)
|
||||||
|
|
||||||
|
Overrides (args["_location_policy"]):
|
||||||
|
- norway_only — Equinor: Norway only
|
||||||
|
- dk_no — NATO-style: Denmark + Norway only (no CH)
|
||||||
|
"""
|
||||||
if not loc_text:
|
if not loc_text:
|
||||||
return False, False
|
return False, False, False, False
|
||||||
low = loc_text.lower()
|
low = loc_text.lower()
|
||||||
in_ch = any(k in low for k in CH_LOCATION_KEYWORDS)
|
in_ch = any(k in low for k in CH_LOCATION_KEYWORDS)
|
||||||
|
in_dk = any(k in low for k in DK_LOCATION_KEYWORDS)
|
||||||
|
in_no = any(k in low for k in NO_LOCATION_KEYWORDS)
|
||||||
is_us_only = any(p in low for p in US_ONLY_PATTERNS) and not in_ch
|
is_us_only = any(p in low for p in US_ONLY_PATTERNS) and not in_ch
|
||||||
is_global_wide = any(k in low for k in GLOBAL_REMOTE_KEYWORDS) and not is_us_only
|
has_remote = bool(re.search(r"\bremote\b", low)) or "work from anywhere" in low
|
||||||
country_hits = sum(1 for group in COUNTRY_HINT_GROUPS if any(k in low for k in group))
|
europe_scope = any(k in low for k in EUROPE_SCOPE_KEYWORDS)
|
||||||
is_multi_country = country_hits >= 2 and not is_us_only
|
global_scope = any(k in low for k in GLOBAL_SCOPE_KEYWORDS) and not is_us_only
|
||||||
is_remote = is_global_wide or is_multi_country
|
# Pan-regional scope counts even without the word "remote" (e.g. QuantCo "Europe").
|
||||||
return in_ch, is_remote
|
is_europe_or_global = (europe_scope or global_scope) and not is_us_only
|
||||||
|
is_ch_remote = in_ch and has_remote
|
||||||
|
|
||||||
|
if policy == "norway_only":
|
||||||
|
eligible = in_no and not is_us_only
|
||||||
|
return eligible, False, False, eligible
|
||||||
|
if policy == "dk_no":
|
||||||
|
# NATO etc.: Denmark + Norway only — CH onsite is out (no NATO seat in CH).
|
||||||
|
eligible = (in_dk or in_no) and not is_us_only
|
||||||
|
return eligible, False, False, eligible
|
||||||
|
|
||||||
def is_relocation_location(loc_text):
|
# default
|
||||||
"""Denmark-only relocation acceptance (see RELOCATION_COUNTRY_KEYWORDS), applied to
|
is_relocation = in_dk
|
||||||
every company — not a per-company opt-in."""
|
is_remote = is_europe_or_global or is_ch_remote
|
||||||
if not loc_text:
|
eligible = (in_ch or in_dk or is_europe_or_global) and not (is_us_only and not in_ch)
|
||||||
return False
|
return eligible, in_ch, is_remote, is_relocation
|
||||||
low = loc_text.lower()
|
|
||||||
return any(k in low for k in RELOCATION_COUNTRY_KEYWORDS)
|
|
||||||
|
|
||||||
|
|
||||||
@lru_cache(maxsize=512)
|
@lru_cache(maxsize=512)
|
||||||
@@ -1103,6 +1317,14 @@ def score_job(job, title_only=False):
|
|||||||
return score, pos, neg
|
return score, pos, neg
|
||||||
|
|
||||||
|
|
||||||
|
def _atomic_write_json(path, data):
|
||||||
|
"""Write JSON via temp + os.replace so a crash mid-write cannot truncate state files."""
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
tmp = path.with_suffix(path.suffix + ".tmp")
|
||||||
|
tmp.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||||
|
os.replace(tmp, path)
|
||||||
|
|
||||||
|
|
||||||
def load_seen():
|
def load_seen():
|
||||||
if STATE_FILE.exists():
|
if STATE_FILE.exists():
|
||||||
return json.loads(STATE_FILE.read_text(encoding="utf-8"))
|
return json.loads(STATE_FILE.read_text(encoding="utf-8"))
|
||||||
@@ -1110,8 +1332,7 @@ def load_seen():
|
|||||||
|
|
||||||
|
|
||||||
def save_seen(seen):
|
def save_seen(seen):
|
||||||
STATE_FILE.parent.mkdir(parents=True, exist_ok=True)
|
_atomic_write_json(STATE_FILE, seen)
|
||||||
STATE_FILE.write_text(json.dumps(seen, indent=2, ensure_ascii=False), encoding="utf-8")
|
|
||||||
|
|
||||||
|
|
||||||
def load_decisions():
|
def load_decisions():
|
||||||
@@ -1124,8 +1345,21 @@ def load_decisions():
|
|||||||
|
|
||||||
|
|
||||||
def save_decisions(decisions):
|
def save_decisions(decisions):
|
||||||
DECISIONS_FILE.parent.mkdir(parents=True, exist_ok=True)
|
_atomic_write_json(DECISIONS_FILE, decisions)
|
||||||
DECISIONS_FILE.write_text(json.dumps(decisions, indent=2, ensure_ascii=False), encoding="utf-8")
|
|
||||||
|
|
||||||
|
def load_last_scrape():
|
||||||
|
"""Per-company last successful scraped counts — used to flag sudden 0-job deaths."""
|
||||||
|
if LAST_SCRAPE_FILE.exists():
|
||||||
|
try:
|
||||||
|
return json.loads(LAST_SCRAPE_FILE.read_text(encoding="utf-8"))
|
||||||
|
except (json.JSONDecodeError, OSError):
|
||||||
|
return {}
|
||||||
|
return {}
|
||||||
|
|
||||||
|
|
||||||
|
def save_last_scrape(data):
|
||||||
|
_atomic_write_json(LAST_SCRAPE_FILE, data)
|
||||||
|
|
||||||
|
|
||||||
def _parse_posted(s):
|
def _parse_posted(s):
|
||||||
@@ -1139,9 +1373,11 @@ def _parse_posted(s):
|
|||||||
return datetime.fromisoformat(s.replace("Z", "+00:00")).date()
|
return datetime.fromisoformat(s.replace("Z", "+00:00")).date()
|
||||||
except ValueError:
|
except ValueError:
|
||||||
pass
|
pass
|
||||||
for fmt in ("%Y-%m-%d", "%d.%m.%Y", "%Y/%m/%d", "%d/%m/%Y"):
|
for fmt in ("%Y-%m-%d", "%d.%m.%Y", "%Y/%m/%d", "%d/%m/%Y", "%d-%b-%Y", "%d-%B-%Y"):
|
||||||
try:
|
try:
|
||||||
return datetime.strptime(s[:10], fmt).date()
|
# %b/%B need full token (e.g. 09-Aug-2026), not a fixed 10-char slice
|
||||||
|
token = s.split(",")[0].strip()
|
||||||
|
return datetime.strptime(token if "%b" in fmt or "%B" in fmt else s[:10], fmt).date()
|
||||||
except ValueError:
|
except ValueError:
|
||||||
pass
|
pass
|
||||||
m = re.search(r"\d{4}-\d{2}-\d{2}", s)
|
m = re.search(r"\d{4}-\d{2}-\d{2}", s)
|
||||||
@@ -1249,6 +1485,94 @@ def write_report(path, results, errors, new_only, include_weak, stats=None, tota
|
|||||||
path.write_text("\n".join(lines), encoding="utf-8")
|
path.write_text("\n".join(lines), encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def _fetch_company(cid, display, adapter, args):
|
||||||
|
"""Fetch one company; returns (cid, display, adapter, args, jobs|None, error|None, secs)."""
|
||||||
|
t0 = time.perf_counter()
|
||||||
|
try:
|
||||||
|
jobs = ADAPTERS[adapter](args)
|
||||||
|
return cid, display, adapter, args, jobs, None, time.perf_counter() - t0
|
||||||
|
except Exception as e:
|
||||||
|
kind = "unexpected" if not isinstance(
|
||||||
|
e, (urllib.error.URLError, urllib.error.HTTPError, ValueError, RuntimeError)
|
||||||
|
) else "error"
|
||||||
|
msg = f"{kind}: {e!r}" if kind == "unexpected" else repr(e)
|
||||||
|
return cid, display, adapter, args, None, msg, time.perf_counter() - t0
|
||||||
|
|
||||||
|
|
||||||
|
def _process_company(cid, display, args, jobs, seen, today, last_scrape):
|
||||||
|
"""Filter, score, and update seen for one company's job list.
|
||||||
|
|
||||||
|
Returns (results, stat_dict, warnings) where warnings is a list of (display, msg).
|
||||||
|
"""
|
||||||
|
warnings = []
|
||||||
|
scraped = len(jobs)
|
||||||
|
prev = (last_scrape.get(cid) or {}).get("scraped", 0)
|
||||||
|
if scraped == 0:
|
||||||
|
if prev and prev > 0:
|
||||||
|
warnings.append((
|
||||||
|
display,
|
||||||
|
f"0 jobs returned (was {prev} last run — possible dead board/slug/selectors)",
|
||||||
|
))
|
||||||
|
else:
|
||||||
|
warnings.append((
|
||||||
|
display,
|
||||||
|
"0 jobs returned (verify board slug/selectors if this is unexpected)",
|
||||||
|
))
|
||||||
|
|
||||||
|
title_filter = args.get("_title_filter")
|
||||||
|
if title_filter:
|
||||||
|
jobs = [j for j in jobs
|
||||||
|
if any(_kw_in(k, (j.get("title") or "").lower()) for k in title_filter)]
|
||||||
|
|
||||||
|
dates = [d for j in jobs if (d := _parse_posted(j.get("posted")))]
|
||||||
|
newest = max(dates) if dates else None
|
||||||
|
|
||||||
|
policy = args.get("_location_policy", "default")
|
||||||
|
company_seen = seen.setdefault(cid, {})
|
||||||
|
title_seen = set()
|
||||||
|
results = []
|
||||||
|
eligible = match = 0
|
||||||
|
for j in jobs:
|
||||||
|
jid = str(j.get("id") or j.get("url"))
|
||||||
|
ok, in_ch, is_remote, is_relocation = location_eligibility(
|
||||||
|
j.get("location", ""), policy=policy
|
||||||
|
)
|
||||||
|
if not ok:
|
||||||
|
continue
|
||||||
|
# Collapse the same role posted once per remote country (title differs only
|
||||||
|
# by a "| Country | Remote" suffix) — dedupe on the title before the first "|".
|
||||||
|
norm_title = re.sub(r"\s+", " ", (j.get("title") or "").split("|")[0]).strip().lower()
|
||||||
|
if norm_title in title_seen:
|
||||||
|
continue
|
||||||
|
title_seen.add(norm_title)
|
||||||
|
eligible += 1
|
||||||
|
is_new = jid not in company_seen
|
||||||
|
score, pos, neg = score_job(j, title_only=bool(title_filter))
|
||||||
|
# Pre-filtered boards (e.g. SBB) with German/generic titles: _score_floor keeps
|
||||||
|
# already-relevant results out of the hidden weak bucket.
|
||||||
|
floor = args.get("_score_floor")
|
||||||
|
if floor is not None and score < floor:
|
||||||
|
score = floor
|
||||||
|
if score >= 2:
|
||||||
|
match += 1
|
||||||
|
results.append({
|
||||||
|
"company": display, "company_id": cid,
|
||||||
|
"title": j["title"], "location": j["location"],
|
||||||
|
"url": j["url"], "posted": j.get("posted", ""),
|
||||||
|
"score": score, "pos": pos, "neg": neg,
|
||||||
|
"in_ch": in_ch, "remote": is_remote, "relocation": is_relocation,
|
||||||
|
"is_new": is_new,
|
||||||
|
})
|
||||||
|
if is_new:
|
||||||
|
company_seen[jid] = {"title": j["title"], "first_seen": today}
|
||||||
|
|
||||||
|
stat = {
|
||||||
|
"company": display, "scraped": scraped, "eligible": eligible,
|
||||||
|
"match": match, "newest": newest, "error": False,
|
||||||
|
}
|
||||||
|
return results, stat, warnings
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
||||||
|
|
||||||
@@ -1261,14 +1585,21 @@ def main():
|
|||||||
url, status, note = rest[0], rest[1], " ".join(rest[2:])
|
url, status, note = rest[0], rest[1], " ".join(rest[2:])
|
||||||
decisions = load_decisions()
|
decisions = load_decisions()
|
||||||
prev = decisions.get(url, {})
|
prev = decisions.get(url, {})
|
||||||
decisions[url] = {"company": prev.get("company", ""), "title": prev.get("title", ""),
|
# Preserve any company/title already stored (filled when a report run tagged the URL).
|
||||||
"decision": status, "note": note, "date": today}
|
decisions[url] = {
|
||||||
|
"company": prev.get("company", ""),
|
||||||
|
"title": prev.get("title", ""),
|
||||||
|
"decision": status, "note": note, "date": today,
|
||||||
|
}
|
||||||
save_decisions(decisions)
|
save_decisions(decisions)
|
||||||
print(f"Recorded: {status} — {url}", file=sys.stderr)
|
print(f"Recorded: {status} — {url}", file=sys.stderr)
|
||||||
return
|
return
|
||||||
|
|
||||||
only, new_only, include_weak, hide_decided = None, False, False, False
|
only, new_only, include_weak, hide_decided = None, False, False, False
|
||||||
for arg in sys.argv[1:]:
|
for arg in sys.argv[1:]:
|
||||||
|
if arg in ("--help", "-h"):
|
||||||
|
print(__doc__)
|
||||||
|
return
|
||||||
if arg == "--new-only":
|
if arg == "--new-only":
|
||||||
new_only = True
|
new_only = True
|
||||||
elif arg == "--include-weak":
|
elif arg == "--include-weak":
|
||||||
@@ -1278,85 +1609,83 @@ def main():
|
|||||||
elif arg.startswith("--only="):
|
elif arg.startswith("--only="):
|
||||||
only = arg.split("=", 1)[1]
|
only = arg.split("=", 1)[1]
|
||||||
|
|
||||||
|
targets = [(cid, display, adapter, args) for cid, display, adapter, args in COMPANIES
|
||||||
|
if not only or cid == only]
|
||||||
|
if only and not targets:
|
||||||
|
known = ", ".join(c[0] for c in COMPANIES)
|
||||||
|
print(f"Unknown --only={only!r}. Known ids: {known}", file=sys.stderr)
|
||||||
|
sys.exit(2)
|
||||||
|
|
||||||
seen = load_seen()
|
seen = load_seen()
|
||||||
decisions = load_decisions()
|
decisions = load_decisions()
|
||||||
|
last_scrape = load_last_scrape()
|
||||||
all_results, errors, stats = [], [], []
|
all_results, errors, stats = [], [], []
|
||||||
|
new_last_scrape = dict(last_scrape)
|
||||||
run_start = time.perf_counter()
|
run_start = time.perf_counter()
|
||||||
|
|
||||||
for cid, display, adapter, args in COMPANIES:
|
# API adapters in parallel; playwright serial (shared browser is not thread-safe).
|
||||||
if only and cid != only:
|
api_targets = [t for t in targets if t[2] != "playwright"]
|
||||||
continue
|
pw_targets = [t for t in targets if t[2] == "playwright"]
|
||||||
print(f"Fetching {display}...", file=sys.stderr)
|
# Preserve COMPANIES order in stats/report via ordered result map.
|
||||||
t0 = time.perf_counter()
|
fetch_results = {}
|
||||||
try:
|
|
||||||
jobs = ADAPTERS[adapter](args)
|
|
||||||
except (urllib.error.URLError, urllib.error.HTTPError, ValueError) as e:
|
|
||||||
errors.append((display, repr(e)))
|
|
||||||
stats.append({"company": display, "scraped": 0, "eligible": 0,
|
|
||||||
"match": 0, "newest": None, "secs": time.perf_counter() - t0,
|
|
||||||
"error": True})
|
|
||||||
continue
|
|
||||||
except Exception as e:
|
|
||||||
errors.append((display, f"unexpected: {e!r}"))
|
|
||||||
stats.append({"company": display, "scraped": 0, "eligible": 0,
|
|
||||||
"match": 0, "newest": None, "secs": time.perf_counter() - t0,
|
|
||||||
"error": True})
|
|
||||||
continue
|
|
||||||
|
|
||||||
scraped = len(jobs)
|
try:
|
||||||
# Optional per-company title prefilter for high-volume boards
|
if api_targets:
|
||||||
title_filter = args.get("_title_filter")
|
print(f"Fetching {len(api_targets)} API boards "
|
||||||
if title_filter:
|
f"({API_FETCH_WORKERS} workers)...", file=sys.stderr)
|
||||||
jobs = [j for j in jobs
|
with ThreadPoolExecutor(max_workers=API_FETCH_WORKERS) as pool:
|
||||||
if any(_kw_in(k, (j.get("title") or "").lower()) for k in title_filter)]
|
futs = {
|
||||||
|
pool.submit(_fetch_company, cid, display, adapter, args): cid
|
||||||
|
for cid, display, adapter, args in api_targets
|
||||||
|
}
|
||||||
|
for fut in as_completed(futs):
|
||||||
|
cid, display, adapter, args, jobs, err, secs = fut.result()
|
||||||
|
print(f" {display}: "
|
||||||
|
f"{'ERROR' if err else f'{len(jobs)} jobs'} ({secs:.1f}s)",
|
||||||
|
file=sys.stderr)
|
||||||
|
fetch_results[cid] = (display, args, jobs, err, secs)
|
||||||
|
|
||||||
# Newest posting on the board (board freshness), across parseable dates.
|
for cid, display, adapter, args in pw_targets:
|
||||||
dates = [d for j in jobs if (d := _parse_posted(j.get("posted")))]
|
print(f"Fetching {display} (playwright)...", file=sys.stderr)
|
||||||
newest = max(dates) if dates else None
|
_, display, _, args, jobs, err, secs = _fetch_company(
|
||||||
|
cid, display, adapter, args
|
||||||
|
)
|
||||||
|
print(f" {display}: "
|
||||||
|
f"{'ERROR' if err else f'{len(jobs)} jobs'} ({secs:.1f}s)",
|
||||||
|
file=sys.stderr)
|
||||||
|
fetch_results[cid] = (display, args, jobs, err, secs)
|
||||||
|
|
||||||
company_seen = seen.setdefault(cid, {})
|
# Process in COMPANIES order for stable stats tables.
|
||||||
title_seen = set()
|
for cid, display, adapter, args in targets:
|
||||||
eligible = match = 0
|
display, args, jobs, err, secs = fetch_results[cid]
|
||||||
for j in jobs:
|
if err is not None:
|
||||||
jid = str(j.get("id") or j.get("url"))
|
errors.append((display, err))
|
||||||
in_ch, is_remote = location_matches(j.get("location", ""))
|
stats.append({
|
||||||
is_relocation = is_relocation_location(j.get("location", ""))
|
"company": display, "scraped": 0, "eligible": 0, "match": 0,
|
||||||
if not (in_ch or is_remote or is_relocation):
|
"newest": None, "secs": secs, "error": True,
|
||||||
|
})
|
||||||
continue
|
continue
|
||||||
# Collapse the same role posted once per remote country (title differs only
|
|
||||||
# by a "| Country | Remote" suffix) — dedupe on the title before the first "|".
|
|
||||||
norm_title = re.sub(r"\s+", " ", (j.get("title") or "").split("|")[0]).strip().lower()
|
|
||||||
if norm_title in title_seen:
|
|
||||||
continue
|
|
||||||
title_seen.add(norm_title)
|
|
||||||
eligible += 1
|
|
||||||
is_new = jid not in company_seen
|
|
||||||
score, pos, neg = score_job(j, title_only=bool(title_filter))
|
|
||||||
# Pre-filtered boards (e.g. SBB, already narrowed to IT+Bern by the adapter) carry
|
|
||||||
# German/generic titles the profile scorer can't read; a _score_floor keeps their
|
|
||||||
# already-relevant results out of the hidden weak bucket.
|
|
||||||
floor = args.get("_score_floor")
|
|
||||||
if floor is not None and score < floor:
|
|
||||||
score = floor
|
|
||||||
if score >= 2:
|
|
||||||
match += 1
|
|
||||||
all_results.append({
|
|
||||||
"company": display, "company_id": cid,
|
|
||||||
"title": j["title"], "location": j["location"],
|
|
||||||
"url": j["url"], "posted": j.get("posted", ""),
|
|
||||||
"score": score, "pos": pos, "neg": neg,
|
|
||||||
"in_ch": in_ch, "remote": is_remote, "relocation": is_relocation,
|
|
||||||
"is_new": is_new,
|
|
||||||
})
|
|
||||||
if is_new:
|
|
||||||
company_seen[jid] = {"title": j["title"], "first_seen": today}
|
|
||||||
|
|
||||||
stats.append({"company": display, "scraped": scraped, "eligible": eligible,
|
results, stat, warnings = _process_company(
|
||||||
"match": match, "newest": newest,
|
cid, display, args, jobs, seen, today, last_scrape
|
||||||
"secs": time.perf_counter() - t0, "error": False})
|
)
|
||||||
|
stat["secs"] = secs
|
||||||
|
for w in warnings:
|
||||||
|
errors.append(w)
|
||||||
|
# Soft warning only when 0 jobs — still mark row with ⚠️ if was previously >0
|
||||||
|
prev = (last_scrape.get(cid) or {}).get("scraped", 0)
|
||||||
|
if stat["scraped"] == 0 and prev > 0:
|
||||||
|
stat["error"] = True
|
||||||
|
stats.append(stat)
|
||||||
|
all_results.extend(results)
|
||||||
|
new_last_scrape[cid] = {
|
||||||
|
"scraped": stat["scraped"], "date": today, "display": display,
|
||||||
|
}
|
||||||
|
finally:
|
||||||
|
save_seen(seen)
|
||||||
|
save_last_scrape(new_last_scrape)
|
||||||
|
_close_browser()
|
||||||
|
|
||||||
save_seen(seen)
|
|
||||||
_close_browser()
|
|
||||||
total_secs = time.perf_counter() - run_start
|
total_secs = time.perf_counter() - run_start
|
||||||
|
|
||||||
if new_only:
|
if new_only:
|
||||||
@@ -1371,84 +1700,52 @@ def main():
|
|||||||
decisions=decisions, hide_decided=hide_decided)
|
decisions=decisions, hide_decided=hide_decided)
|
||||||
|
|
||||||
# Plain data dump of every eligible role, unfiltered by keyword score — fit judgment
|
# Plain data dump of every eligible role, unfiltered by keyword score — fit judgment
|
||||||
# for these is done by Claude in conversation against the profile memory, not by the
|
# for these is done in conversation against the profile, not by the keyword scorer.
|
||||||
# keyword scorer (which is a rough legacy signal, kept below as "pos"/"neg" hints only).
|
|
||||||
json_path = REPORTS_DIR / f"{today}.json"
|
json_path = REPORTS_DIR / f"{today}.json"
|
||||||
dump = [{
|
dump = [{
|
||||||
"company": r["company"], "title": r["title"], "location": r["location"],
|
"company": r["company"], "title": r["title"], "location": r["location"],
|
||||||
"url": r["url"], "posted": r["posted"], "is_new": r["is_new"],
|
"url": r["url"], "posted": r["posted"], "is_new": r["is_new"],
|
||||||
"decision": decisions.get(r["url"]),
|
"decision": decisions.get(r["url"]),
|
||||||
"keyword_signal": {"score": r["score"], "pos": r["pos"], "neg": r["neg"]},
|
"keyword_signal": {"score": r["score"], "pos": r["pos"], "neg": r["neg"]},
|
||||||
|
"loc_flags": {
|
||||||
|
"in_ch": r["in_ch"], "remote": r["remote"], "relocation": r["relocation"],
|
||||||
|
},
|
||||||
} for r in all_results]
|
} for r in all_results]
|
||||||
json_path.write_text(json.dumps(dump, indent=2, ensure_ascii=False), encoding="utf-8")
|
_atomic_write_json(json_path, dump)
|
||||||
|
|
||||||
n_new = sum(1 for r in all_results if r["is_new"])
|
n_new = sum(1 for r in all_results if r["is_new"])
|
||||||
print(f"\nReport written: {report_path}", file=sys.stderr)
|
print(f"\nReport written: {report_path}", file=sys.stderr)
|
||||||
print(f"Total matches: {len(all_results)} ({n_new} new) | "
|
print(f"Total matches: {len(all_results)} ({n_new} new) | "
|
||||||
f"scanned {len(stats)} companies in {total_secs:.1f}s", file=sys.stderr)
|
f"scanned {len(stats)} companies in {total_secs:.1f}s", file=sys.stderr)
|
||||||
if errors:
|
if errors:
|
||||||
print(f"Errors: {len(errors)} - see report", file=sys.stderr)
|
print(f"Errors/warnings: {len(errors)} - see report", file=sys.stderr)
|
||||||
|
|
||||||
|
|
||||||
# === Adapter coverage (refreshed 2026-06-06) ==================================
|
# === Adapter coverage (refreshed 2026-07-18) ==================================
|
||||||
# 31 companies automated across 13 adapter types; 1 in MANUAL_CHECK (Oracle).
|
# 34 companies automated across 14 adapter types; 1 in MANUAL_CHECK (Oracle).
|
||||||
|
#
|
||||||
|
# Location policy:
|
||||||
|
# default — CH + Denmark + Europe/EMEA/global remote (work-from CH or DK)
|
||||||
|
# norway_only — Equinor
|
||||||
|
# dk_no — NATO (Denmark + Norway only; no CH)
|
||||||
#
|
#
|
||||||
# Automated (COMPANIES above):
|
# Automated (COMPANIES above):
|
||||||
# workday nvidia, novartis
|
# workday nvidia, novartis, equinor (norway_only)
|
||||||
# ashby kraken, openai, confluent, snowflake
|
# ashby kraken, openai, confluent, snowflake
|
||||||
# greenhouse anthropic, gitlab, grafana, databricks, datadog, elastic, dbtlabs
|
# greenhouse anthropic, gitlab, grafana, databricks, datadog, elastic, dbtlabs
|
||||||
# pcsx microsoft (Eightfold position-search endpoint)
|
# pcsx microsoft
|
||||||
# smartrecruiters metgroup, ldc
|
# smartrecruiters metgroup, ldc
|
||||||
# rss bis (vacancies.rss — RSS 1.0/RDF)
|
# rss bis
|
||||||
# getro coinbase_ventures (web3 portfolio network, collection 1625)
|
# getro coinbase_ventures
|
||||||
# onlyfy bitcoin_suisse (onlyfy.jobs ajax_list HTML fragment)
|
# onlyfy bitcoin_suisse
|
||||||
# lever palantir, quantco (api.lever.co; QuantCo slug is "quantco-")
|
# lever palantir, quantco
|
||||||
# json swissgrid (Magnolia /.rest/cloud/component-data)
|
# taleo nato (dk_no; nato.taleo.net §2 — marketing page is nato.int/vacancies)
|
||||||
# sbb sbb (company.sbb.ch AEM jobfilter.results.json)
|
# json swissgrid
|
||||||
# bkw bkw (jobs.bkw.com PMS structureddata API)
|
# sbb sbb
|
||||||
# playwright google, apple, meta, roche, cisco, ruag (headless browser, 3-15s each)
|
# bkw bkw
|
||||||
|
# playwright google, apple, meta, roche, cisco, ruag, postfinance, bfh
|
||||||
#
|
#
|
||||||
# 2026-06-01 list review (verified live):
|
# MANUAL_CHECK: Oracle (ORC needs CH geographyId).
|
||||||
# - Palantir (lever): 221 postings, US/London-heavy so Swiss/Schwyz roles are rare but
|
|
||||||
# self-surface (FDSE/Deployment-Strategist titles map to his FDE drafts).
|
|
||||||
# - Swissgrid (json): Magnolia CMS endpoint; placeOfWork is bare city, so loc_suffix tags
|
|
||||||
# it Switzerland for the CH filter. ~13 roles incl. Data Scientist / Applied-ML.
|
|
||||||
# - RUAG (playwright + page_param): Drupal portal, 20 jobs/page, paginated ?page=N. Page 0
|
|
||||||
# is apprenticeship-heavy; eng roles (DevOps/Data/Software) are on later pages, so we
|
|
||||||
# page through (max_pages). ENG_TITLE_FILTER cuts the Lehrstelle bulk. ⚠️ DE-citizen
|
|
||||||
# limits on RUAG classified roles — verify per-role.
|
|
||||||
# - SBB (sbb): correct host is company.sbb.ch (not company-jobs.sbb.ch). Flat JSON list;
|
|
||||||
# fetch_sbb replicates the user's IT + Bern-region filter. German/generic titles, so a
|
|
||||||
# _score_floor keeps the pre-filtered results visible. ⚠️ DE-citizen limits possible.
|
|
||||||
# - BKW (bkw): real host is jobs.bkw.com (PMS structureddata API), ~600 group-wide roles;
|
|
||||||
# fetch_bkw keeps Berufsfeld categories Informatik/Trading/Finanzen (IT/data + energy
|
|
||||||
# trading: Quant Risk, Solution Architect Energiehandel, ...). _score_floor as above.
|
|
||||||
# - QuantCo (lever, slug "quantco-"): ~16 roles, most tagged "Europe" (hybrid; Zürich is
|
|
||||||
# QuantCo's continental hub), surfaced via the EU-wide rule in location_matches. Strong:
|
|
||||||
# AI Engineer; medium: Cloud Engineer, AI Applied Scientist, Data Scientist, Quant
|
|
||||||
# Researcher, Software Engineer. Interns/frontend suppressed by NEGATIVE_KEYWORDS.
|
|
||||||
# The Bern/Thun tier intentionally relaxes the comp bar (see user_comp_bar memory).
|
|
||||||
#
|
|
||||||
# 2026-06-06 additions (FAANG-adjacent data-infra, Tier A/B from the Zürich/Bern review):
|
|
||||||
# - greenhouse: Databricks (board "databricks", 762 roles, Zürich SWE), Datadog ("datadog",
|
|
||||||
# Zürich branch + remote-EU), Elastic ("elastic", remote-first ELK), dbt Labs ("dbtlabsinc",
|
|
||||||
# remote-EU). ashby: Snowflake (slug "snowflake", 392 roles incl. Zürich "Observe by
|
|
||||||
# Snowflake" observability SWE). All title-filtered (ENG_TITLE_FILTER) — large boards.
|
|
||||||
# - ⚠️ Comp split: Databricks/Snowflake/Datadog pay Swiss-scale (Zürich offices, clears bar);
|
|
||||||
# Elastic/dbt are remote-EU and may be geo-banded below 180k CHF (like Grafana — verify).
|
|
||||||
# - HashiCorp: NOT added — IBM acquisition (2025) killed its public boards; on IBM careers now.
|
|
||||||
# - Oracle: in MANUAL_CHECK — ORC SPA resists scraping; REST endpoint documented there.
|
|
||||||
#
|
|
||||||
# MANUAL_CHECK: Oracle (ORC needs CH geographyId resolved). Dropped 2026-06-01: BFH
|
|
||||||
# (academic FH pay below the relaxed Bern/Thun floor, research-leaning, 403s anyway) and
|
|
||||||
# Dialectic (~50-person crypto VC, 0 open roles; crypto already covered by Kraken / Bitcoin
|
|
||||||
# Suisse / Coinbase Ventures).
|
|
||||||
#
|
|
||||||
# Earlier history: Google/Apple/Meta/Roche/Cisco automated via playwright; Microsoft via
|
|
||||||
# pcsx; BIS via rss; Coinbase Ventures via getro; Bitcoin Suisse via onlyfy. Dropped:
|
|
||||||
# ClickHouse, Vitol, Sygnum (Glassdoor/comp red flags), IBM Research + Sonova (low fit),
|
|
||||||
# Coinbase-the-employer (hiring freeze), AMINA (poor Glassdoor), Canonical (pay+culture).
|
|
||||||
# The Coinbase Ventures board (getro) covers PORTFOLIO companies, not Coinbase itself.
|
|
||||||
# ==============================================================================
|
# ==============================================================================
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1650,5 +1650,19 @@
|
|||||||
"decision": "skip",
|
"decision": "skip",
|
||||||
"note": "Claude judgment 2026-07-06: internship, comms",
|
"note": "Claude judgment 2026-07-06: internship, comms",
|
||||||
"date": "2026-07-06"
|
"date": "2026-07-06"
|
||||||
|
},
|
||||||
|
"https://nato.taleo.net/careersection/2/jobdetail.ftl?job=261160&lang=en": {
|
||||||
|
"company": "NATO",
|
||||||
|
"title": "Staff Officer (2030 Digitalisation - Artificial Intelligence Engineer)",
|
||||||
|
"decision": "applied",
|
||||||
|
"note": "SENT 2026-07-10 via NTAP (Staff Officer AI Engineer, JWC Stavanger). See Agents.md Active Sessions.",
|
||||||
|
"date": "2026-07-18"
|
||||||
|
},
|
||||||
|
"https://nato.taleo.net/careersection/2/jobdetail.ftl?job=261143&lang=en": {
|
||||||
|
"company": "NATO",
|
||||||
|
"title": "Staff Officer (2030 Digitalisation - Cloud Artificial Intelligence Architect)",
|
||||||
|
"decision": "skip",
|
||||||
|
"note": "Cloud AI Architect — above experience level (user judgment 2026-07-18).",
|
||||||
|
"date": "2026-07-18"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Reference in New Issue
Block a user