fix(scout): BIS per-vacancy location and Confluent via the IBM careers API
BIS
- The RSS feed carries no location, so every vacancy was stamped with
default_location "Basel, Switzerland". jr100491 (Hong Kong, BIS Asia office)
therefore passed the CH filter and reached the 2026-10-14 triage.
- fetch_rss now resolves the location per item through _rss_item_location:
the vacancy page's static "Geographical location" field
(location_page_regex), then a "based in ..." sentence in the feed
description, then default_location with a warning on stderr.
- Verified with --only=bis: 3 scraped, 2 eligible (both Basel); jr100491 now
reads "8 Finance Street, Central, Hong Kong" and is excluded.
Confluent
- IBM closed the acquisition on 2026-03-17. The Ashby board dropped from 17
jobs (2026-10-08) to {"jobs":[]} (2026-10-14), and careers.confluent.io is
behind a Vercel bot checkpoint.
- New `ibm` adapter for the open IBM careers search API
(www-api.ibm.com/search/api/v2; POST; size capped at 100, paged with
`from`). Location is built from city, country and work mode, e.g.
"United States - Remote".
- Confluent keeps its id, so seen-state and decisions still link. Verified
with --only=confluent: 84 roles, none in CH/Europe-remote scope today, and no
0-jobs warning.
CLAUDE.md corrections log: 2026-10-15 entry. Also notes that HashiCorp
(dropped 2026-06-06 for lacking a public API after the IBM acquisition) could
return through the same adapter.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
2273c3eac8
commit
dbf937043e
2 files changed
+104
-5
No files matched your search
+103
-5
@@ -254,7 +254,10 @@ COMPANIES = [
|
||||
# Dropped: ClickHouse (Glassdoor 3.3, 36% recommend, toxic-culture flag — 2026-05).
|
||||
# Dropped: HashiCorp — acquired by IBM (closed 2025); greenhouse/ashby/lever boards all 404,
|
||||
# roles folded into IBM's careers (no clean public ATS API). 2026-06-06.
|
||||
("confluent", "Confluent", "ashby", {"slug": "confluent", "_title_exclude": NOISE_TITLE_EXCLUDE}),
|
||||
# Confluent: acquired by IBM (closed 2026-03-17). The Ashby board went empty (17 jobs on
|
||||
# 2026-10-08, 0 on 2026-10-14); roles are now IBM requisitions. Moved to the `ibm`
|
||||
# adapter 2026-10-15. Same id, so seen-state and decisions stay keyed to "confluent".
|
||||
("confluent", "Confluent (IBM)", "ibm", {"query": "confluent", "_title_exclude": NOISE_TITLE_EXCLUDE}),
|
||||
("gitlab", "GitLab", "greenhouse", {"board": "gitlab", "_title_exclude": NOISE_TITLE_EXCLUDE}),
|
||||
("grafana", "Grafana Labs","greenhouse",{"board": "grafanalabs", "_title_exclude": NOISE_TITLE_EXCLUDE}),
|
||||
# Added 2026-06-06 (Tier A/B data-infra). Databricks/Snowflake/Datadog have Zürich offices
|
||||
@@ -369,6 +372,9 @@ COMPANIES = [
|
||||
# Low-volume RSS feed; no title filter (Innovation Hub roles can be oddly titled).
|
||||
("bis", "BIS (Basel)","rss", {
|
||||
"url": "https://www.bis.org/doclist/vacancies.rss",
|
||||
# Each vacancy page has a static "Geographical location" sidebar field. BIS also
|
||||
# hires for Hong Kong, Mexico City and other offices; see _rss_item_location.
|
||||
"location_page_regex": r"Geographical location</h6>\s*<div[^>]*>\s*<p>([^<]+)</p>",
|
||||
"default_location": "Basel, Switzerland",
|
||||
}),
|
||||
# Dropped: Coinbase Ventures web3 talent network (Getro collection 1625) — 2026-08-18.
|
||||
@@ -826,10 +832,37 @@ def fetch_smartrecruiters(args):
|
||||
return jobs
|
||||
|
||||
|
||||
def _rss_item_location(link, description, args):
|
||||
"""Per-item location for feeds that carry none.
|
||||
|
||||
Order: the item's own page via `location_page_regex` (first group), then a
|
||||
"based in <place>" sentence in the feed description, then `default_location`.
|
||||
Added 2026-10-15: stamping every BIS vacancy with default_location labelled a Hong Kong
|
||||
req (jr100491) as Basel, so it passed the CH location filter and reached the triage.
|
||||
"""
|
||||
pattern = args.get("location_page_regex")
|
||||
if pattern and link:
|
||||
try:
|
||||
req = urllib.request.Request(link, headers={"User-Agent": USER_AGENT})
|
||||
with urllib.request.urlopen(req, timeout=30, context=_ssl_context()) as resp:
|
||||
page = resp.read().decode("utf-8", errors="replace")
|
||||
match = re.search(pattern, page, re.I | re.S)
|
||||
if match and match.group(1).strip():
|
||||
return html.unescape(match.group(1).strip())
|
||||
except (urllib.error.URLError, OSError, ValueError):
|
||||
pass
|
||||
match = re.search(r"\bbased in ([A-Z][^.;:]{2,60})", description)
|
||||
if match:
|
||||
return match.group(1).strip()
|
||||
print(f" ! {link or '?'}: no location on page or in feed, using default_location",
|
||||
file=sys.stderr)
|
||||
return args.get("default_location", "")
|
||||
|
||||
|
||||
def fetch_rss(args):
|
||||
"""Generic RSS/RDF feed parser. BIS publishes vacancies as RSS 1.0 (RDF), whose
|
||||
<item> elements live in the http://purl.org/rss/1.0/ namespace. Falls back to plain
|
||||
RSS 2.0 <item> elements. Location isn't in the feed, so default_location is required."""
|
||||
RSS 2.0 <item> elements. Location isn't in the feed; see _rss_item_location."""
|
||||
import xml.etree.ElementTree as ET
|
||||
req = urllib.request.Request(args["url"], headers={"User-Agent": USER_AGENT})
|
||||
with urllib.request.urlopen(req, timeout=30, context=_ssl_context()) as resp:
|
||||
@@ -844,13 +877,14 @@ def fetch_rss(args):
|
||||
el = it.find(tag)
|
||||
return (el.text or "").strip() if el is not None and el.text else ""
|
||||
link = field("link")
|
||||
description = re.sub(r"<[^>]+>", " ", field("description"))
|
||||
jobs.append({
|
||||
"id": link or field("title"),
|
||||
"title": field("title"),
|
||||
"location": args.get("default_location", ""),
|
||||
"location": _rss_item_location(link, description, args),
|
||||
"url": link,
|
||||
"posted": (it.findtext("dc:date", default="", namespaces=ns) or field("date")),
|
||||
"description": re.sub(r"<[^>]+>", " ", field("description"))[:1500],
|
||||
"description": description[:1500],
|
||||
})
|
||||
return jobs
|
||||
|
||||
@@ -1746,7 +1780,70 @@ def fetch_playwright(args):
|
||||
return jobs
|
||||
|
||||
|
||||
IBM_SEARCH_API = "https://www-api.ibm.com/search/api/v2"
|
||||
|
||||
|
||||
def fetch_ibm(args):
|
||||
"""IBM careers search: the open JSON endpoint behind ibm.com/careers/search.
|
||||
|
||||
Added 2026-10-15 for Confluent. IBM closed the acquisition on 2026-03-17, and the
|
||||
Confluent Ashby board (`jobs.ashbyhq.com/confluent`) returns no jobs. Its roles are now
|
||||
IBM requisitions, mostly titled "... - Confluent". careers.confluent.io sits behind a
|
||||
Vercel bot checkpoint, so this API is the clean path.
|
||||
|
||||
`query` is a keyword search over title, description and body, as the site's own search
|
||||
box sends it. The API rejects `size` above 100, so pages are fetched with `from`.
|
||||
Location = city (unless "Multiple Cities" / "No City") + country + work mode, e.g.
|
||||
"United States - Remote" or "Zurich, CH, Switzerland - Hybrid".
|
||||
"""
|
||||
body = {
|
||||
"appId": "careers", "scopes": ["careers2"], "lang": "zz", "size": 100,
|
||||
"query": {"bool": {"must": [{"simple_query_string": {
|
||||
"query": args["query"],
|
||||
"fields": ["keywords^1", "body^1", "url^2", "description^2",
|
||||
"h1s_content^2", "title^3", "field_text_01"],
|
||||
}}]}},
|
||||
"sort": [{"_score": "desc"}],
|
||||
"_source": ["title", "url", "description",
|
||||
"field_keyword_05", "field_keyword_17", "field_keyword_19"],
|
||||
}
|
||||
jobs, offset = [], 0
|
||||
while True:
|
||||
body["from"] = offset
|
||||
req = urllib.request.Request(
|
||||
IBM_SEARCH_API, data=json.dumps(body).encode("utf-8"), method="POST",
|
||||
headers={"Content-Type": "application/json", "User-Agent": USER_AGENT})
|
||||
with urllib.request.urlopen(req, timeout=30, context=_ssl_context()) as resp:
|
||||
data = json.load(resp)
|
||||
hits = data.get("hits", {}).get("hits", [])
|
||||
for hit in hits:
|
||||
src = hit.get("_source", {})
|
||||
city = (src.get("field_keyword_19") or "").strip()
|
||||
country = src.get("field_keyword_05") or ""
|
||||
if isinstance(country, list):
|
||||
country = ", ".join(country)
|
||||
place = ", ".join(p for p in (
|
||||
"" if city.lower().startswith(("multiple cities", "no city")) else city,
|
||||
country.strip()) if p)
|
||||
mode = (src.get("field_keyword_17") or "").strip()
|
||||
url = src.get("url", "")
|
||||
job_id = urllib.parse.parse_qs(urllib.parse.urlsplit(url).query).get("jobId", [""])[0]
|
||||
jobs.append({
|
||||
"id": job_id or url,
|
||||
"title": (src.get("title") or "").strip(),
|
||||
"location": f"{place} - {mode}" if mode else place,
|
||||
"url": url,
|
||||
"posted": "",
|
||||
"description": (src.get("description") or "")[:1500],
|
||||
})
|
||||
total = data.get("hits", {}).get("total", {}).get("value", 0)
|
||||
offset += len(hits)
|
||||
if not hits or offset >= total or offset >= 1000:
|
||||
return jobs
|
||||
|
||||
|
||||
ADAPTERS = {
|
||||
"ibm": fetch_ibm,
|
||||
"workday": fetch_workday,
|
||||
"ashby": fetch_ashby,
|
||||
"greenhouse": fetch_greenhouse,
|
||||
@@ -2613,7 +2710,8 @@ def _main():
|
||||
#
|
||||
# Automated (COMPANIES above):
|
||||
# workday nvidia, novartis, equinor (norway_only)
|
||||
# ashby kraken, openai, confluent, snowflake
|
||||
# ashby kraken, openai, snowflake
|
||||
# ibm confluent (IBM careers search API since 2026-10-15; Ashby board empty)
|
||||
# greenhouse anthropic, gitlab, grafana, databricks, datadog, elastic, fivetran
|
||||
# pcsx microsoft
|
||||
# smartrecruiters metgroup, cern
|
||||
|
||||
Reference in new issue
Block a user