fix(scout): BIS per-vacancy location and Confluent via the IBM careers API

BIS
- The RSS feed carries no location, so every vacancy was stamped with
  default_location "Basel, Switzerland". jr100491 (Hong Kong, BIS Asia office)
  therefore passed the CH filter and reached the 2026-10-14 triage.
- fetch_rss now resolves the location per item through _rss_item_location:
  the vacancy page's static "Geographical location" field
  (location_page_regex), then a "based in ..." sentence in the feed
  description, then default_location with a warning on stderr.
- Verified with --only=bis: 3 scraped, 2 eligible (both Basel); jr100491 now
  reads "8 Finance Street, Central, Hong Kong" and is excluded.

Confluent
- IBM closed the acquisition on 2026-03-17. The Ashby board dropped from 17
  jobs (2026-10-08) to {"jobs":[]} (2026-10-14), and careers.confluent.io is
  behind a Vercel bot checkpoint.
- New `ibm` adapter for the open IBM careers search API
  (www-api.ibm.com/search/api/v2; POST; size capped at 100, paged with
  `from`). Location is built from city, country and work mode, e.g.
  "United States - Remote".
- Confluent keeps its id, so seen-state and decisions still link. Verified
  with --only=confluent: 84 roles, none in CH/Europe-remote scope today, and no
  0-jobs warning.

CLAUDE.md corrections log: 2026-10-15 entry. Also notes that HashiCorp
(dropped 2026-06-06 for lacking a public API after the IBM acquisition) could
return through the same adapter.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dennisthiessenandClaude Opus 5.5 committed 2026-10-15 10:12:04 +02:00
1 parent 2273c3eac8
commit dbf937043e
2 files changed
+104 -5

No files matched your search

+103 -5
View File
@@ -254,7 +254,10 @@ COMPANIES = [
# Dropped: ClickHouse (Glassdoor 3.3, 36% recommend, toxic-culture flag — 2026-05).
# Dropped: HashiCorp — acquired by IBM (closed 2025); greenhouse/ashby/lever boards all 404,
# roles folded into IBM's careers (no clean public ATS API). 2026-06-06.
("confluent", "Confluent", "ashby", {"slug": "confluent", "_title_exclude": NOISE_TITLE_EXCLUDE}),
# Confluent: acquired by IBM (closed 2026-03-17). The Ashby board went empty (17 jobs on
# 2026-10-08, 0 on 2026-10-14); roles are now IBM requisitions. Moved to the `ibm`
# adapter 2026-10-15. Same id, so seen-state and decisions stay keyed to "confluent".
("confluent", "Confluent (IBM)", "ibm", {"query": "confluent", "_title_exclude": NOISE_TITLE_EXCLUDE}),
("gitlab", "GitLab", "greenhouse", {"board": "gitlab", "_title_exclude": NOISE_TITLE_EXCLUDE}),
("grafana", "Grafana Labs","greenhouse",{"board": "grafanalabs", "_title_exclude": NOISE_TITLE_EXCLUDE}),
# Added 2026-06-06 (Tier A/B data-infra). Databricks/Snowflake/Datadog have Zürich offices
@@ -369,6 +372,9 @@ COMPANIES = [
# Low-volume RSS feed; no title filter (Innovation Hub roles can be oddly titled).
("bis", "BIS (Basel)","rss", {
"url": "https://www.bis.org/doclist/vacancies.rss",
# Each vacancy page has a static "Geographical location" sidebar field. BIS also
# hires for Hong Kong, Mexico City and other offices; see _rss_item_location.
"location_page_regex": r"Geographical location</h6>\s*<div[^>]*>\s*<p>([^<]+)</p>",
"default_location": "Basel, Switzerland",
}),
# Dropped: Coinbase Ventures web3 talent network (Getro collection 1625) — 2026-08-18.
@@ -826,10 +832,37 @@ def fetch_smartrecruiters(args):
return jobs
def _rss_item_location(link, description, args):
"""Per-item location for feeds that carry none.
Order: the item's own page via `location_page_regex` (first group), then a
"based in <place>" sentence in the feed description, then `default_location`.
Added 2026-10-15: stamping every BIS vacancy with default_location labelled a Hong Kong
req (jr100491) as Basel, so it passed the CH location filter and reached the triage.
"""
pattern = args.get("location_page_regex")
if pattern and link:
try:
req = urllib.request.Request(link, headers={"User-Agent": USER_AGENT})
with urllib.request.urlopen(req, timeout=30, context=_ssl_context()) as resp:
page = resp.read().decode("utf-8", errors="replace")
match = re.search(pattern, page, re.I | re.S)
if match and match.group(1).strip():
return html.unescape(match.group(1).strip())
except (urllib.error.URLError, OSError, ValueError):
pass
match = re.search(r"\bbased in ([A-Z][^.;:]{2,60})", description)
if match:
return match.group(1).strip()
print(f" ! {link or '?'}: no location on page or in feed, using default_location",
file=sys.stderr)
return args.get("default_location", "")
def fetch_rss(args):
"""Generic RSS/RDF feed parser. BIS publishes vacancies as RSS 1.0 (RDF), whose
<item> elements live in the http://purl.org/rss/1.0/ namespace. Falls back to plain
RSS 2.0 <item> elements. Location isn't in the feed, so default_location is required."""
RSS 2.0 <item> elements. Location isn't in the feed; see _rss_item_location."""
import xml.etree.ElementTree as ET
req = urllib.request.Request(args["url"], headers={"User-Agent": USER_AGENT})
with urllib.request.urlopen(req, timeout=30, context=_ssl_context()) as resp:
@@ -844,13 +877,14 @@ def fetch_rss(args):
el = it.find(tag)
return (el.text or "").strip() if el is not None and el.text else ""
link = field("link")
description = re.sub(r"<[^>]+>", " ", field("description"))
jobs.append({
"id": link or field("title"),
"title": field("title"),
"location": args.get("default_location", ""),
"location": _rss_item_location(link, description, args),
"url": link,
"posted": (it.findtext("dc:date", default="", namespaces=ns) or field("date")),
"description": re.sub(r"<[^>]+>", " ", field("description"))[:1500],
"description": description[:1500],
})
return jobs
@@ -1746,7 +1780,70 @@ def fetch_playwright(args):
return jobs
IBM_SEARCH_API = "https://www-api.ibm.com/search/api/v2"
def fetch_ibm(args):
"""IBM careers search: the open JSON endpoint behind ibm.com/careers/search.
Added 2026-10-15 for Confluent. IBM closed the acquisition on 2026-03-17, and the
Confluent Ashby board (`jobs.ashbyhq.com/confluent`) returns no jobs. Its roles are now
IBM requisitions, mostly titled "... - Confluent". careers.confluent.io sits behind a
Vercel bot checkpoint, so this API is the clean path.
`query` is a keyword search over title, description and body, as the site's own search
box sends it. The API rejects `size` above 100, so pages are fetched with `from`.
Location = city (unless "Multiple Cities" / "No City") + country + work mode, e.g.
"United States - Remote" or "Zurich, CH, Switzerland - Hybrid".
"""
body = {
"appId": "careers", "scopes": ["careers2"], "lang": "zz", "size": 100,
"query": {"bool": {"must": [{"simple_query_string": {
"query": args["query"],
"fields": ["keywords^1", "body^1", "url^2", "description^2",
"h1s_content^2", "title^3", "field_text_01"],
}}]}},
"sort": [{"_score": "desc"}],
"_source": ["title", "url", "description",
"field_keyword_05", "field_keyword_17", "field_keyword_19"],
}
jobs, offset = [], 0
while True:
body["from"] = offset
req = urllib.request.Request(
IBM_SEARCH_API, data=json.dumps(body).encode("utf-8"), method="POST",
headers={"Content-Type": "application/json", "User-Agent": USER_AGENT})
with urllib.request.urlopen(req, timeout=30, context=_ssl_context()) as resp:
data = json.load(resp)
hits = data.get("hits", {}).get("hits", [])
for hit in hits:
src = hit.get("_source", {})
city = (src.get("field_keyword_19") or "").strip()
country = src.get("field_keyword_05") or ""
if isinstance(country, list):
country = ", ".join(country)
place = ", ".join(p for p in (
"" if city.lower().startswith(("multiple cities", "no city")) else city,
country.strip()) if p)
mode = (src.get("field_keyword_17") or "").strip()
url = src.get("url", "")
job_id = urllib.parse.parse_qs(urllib.parse.urlsplit(url).query).get("jobId", [""])[0]
jobs.append({
"id": job_id or url,
"title": (src.get("title") or "").strip(),
"location": f"{place} - {mode}" if mode else place,
"url": url,
"posted": "",
"description": (src.get("description") or "")[:1500],
})
total = data.get("hits", {}).get("total", {}).get("value", 0)
offset += len(hits)
if not hits or offset >= total or offset >= 1000:
return jobs
ADAPTERS = {
"ibm": fetch_ibm,
"workday": fetch_workday,
"ashby": fetch_ashby,
"greenhouse": fetch_greenhouse,
@@ -2613,7 +2710,8 @@ def _main():
#
# Automated (COMPANIES above):
# workday nvidia, novartis, equinor (norway_only)
# ashby kraken, openai, confluent, snowflake
# ashby kraken, openai, snowflake
# ibm confluent (IBM careers search API since 2026-10-15; Ashby board empty)
# greenhouse anthropic, gitlab, grafana, databricks, datadog, elastic, fivetran
# pcsx microsoft
# smartrecruiters metgroup, cern