diff --git a/app/services/sec_client.py b/app/services/sec_client.py
index e4e1824..0d9345d 100644
--- a/app/services/sec_client.py
+++ b/app/services/sec_client.py
@@ -8,7 +8,9 @@ the daily filing index — behind one client that honors SEC's fair-access polic
- request spacing well under the 10 req/s limit;
- exponential backoff + retry on 429;
- **403 → alert and stop** (raise ``SecForbiddenError``), never a retry-loop — a
- 403 means the UA or request pattern is wrong and retrying won't fix it.
+ 403 means the UA or request pattern is wrong and retrying won't fix it. The one
+ exception is S3's ``AccessDenied`` on an ``/Archives/`` path, which is how the
+ bucket reports an absent file (``_is_absent_archive_key``).
Parsing lives here (index fixed-width, submissions pagination); DB writes and the
snapshot mapping live in the importer. No conditional GETs — the companyfacts
@@ -53,11 +55,44 @@ class SecForbiddenError(SecError):
class SecNotFoundError(SecError):
- """SEC returned 404 — the resource does not exist (e.g. no index for a day).
+ """The resource does not exist (e.g. no daily index published for a day).
The *only* error a caller may treat as 'missing' — every other SecError
- (403, exhausted retries, 5xx, timeout) must propagate so a fetch failure is
- never mistaken for an empty result."""
+ (fair-access rejection, exhausted retries, 5xx, timeout) must propagate so a
+ fetch failure is never mistaken for an empty result.
+
+ Raised for a 404, and for the one 403 that also means "absent": see
+ ``_is_absent_archive_key``."""
+
+
+def _is_absent_archive_key(url: str, resp: httpx.Response) -> bool:
+ """True when a 403 means "this file does not exist", not "you are blocked".
+
+ ``www.sec.gov/Archives`` is served straight out of an S3 bucket that grants
+ no ``s3:ListBucket``, so a missing key cannot be answered with 404 — S3
+ returns **403 with its ``AccessDenied`` XML** instead. SEC publishes a daily
+ index only for business days, so every weekend and market holiday inside an
+ incremental walk lands on exactly this response (verified 2026-07-30:
+ ``form.20260725.idx``, a Saturday, 403s while the Friday and Monday files
+ return 200 on the same User-Agent).
+
+ A genuine fair-access rejection is distinguishable and must stay fatal: it is
+ SEC's WAF interstitial — ``text/html``, "Your Request Originates from an
+ Undeclared Automated Tool" — and it is returned for files that *do* exist,
+ on any path. Hence the narrow gate: the Archives prefix plus S3's own error
+ document. Nothing else may be downgraded to "missing"."""
+ try:
+ parsed = httpx.URL(url)
+ except (TypeError, ValueError): # pragma: no cover — url comes from us
+ return False
+ if parsed.host != "www.sec.gov" or not parsed.path.startswith("/Archives/"):
+ return False
+ if "xml" not in resp.headers.get("Content-Type", "").lower():
+ return False
+ try:
+ return "AccessDenied" in resp.text
+ except (UnicodeDecodeError, httpx.HTTPError): # pragma: no cover
+ return False
def _looks_like_contact_email(ua: str) -> bool:
@@ -155,6 +190,8 @@ class SecClient:
code = resp.status_code
if code == 403:
+ if _is_absent_archive_key(url, resp):
+ raise SecNotFoundError(f"SEC 403/AccessDenied (absent) for {url}")
raise SecForbiddenError(
f"SEC 403 for {url} — User-Agent/pattern rejected; set a real "
"sec_user_agent contact email"
@@ -252,7 +289,16 @@ class SecClient:
try:
text = await self.get_text(url)
except SecNotFoundError:
- logger.info("no daily index for %s (404)", day)
+ # Absent is normal on a weekend (SEC publishes business days only). On a
+ # weekday it is not: a SEC hiccup — or a rejection page misread as absent
+ # — would otherwise let the importer advance past real filings silently,
+ # so surface it at WARNING instead of hiding it in the info stream.
+ logger.log(
+ logging.INFO if day.weekday() >= 5 else logging.WARNING,
+ "no daily index published for %s (%s)",
+ day,
+ f"{day:%a}",
+ )
return [] # weekend/holiday/not-yet-published; other errors propagate
return _parse_form_index(text)
diff --git a/docs/dolt-sec-a3-design.md b/docs/dolt-sec-a3-design.md
index cd54949..072f944 100644
--- a/docs/dolt-sec-a3-design.md
+++ b/docs/dolt-sec-a3-design.md
@@ -226,7 +226,12 @@ primary period (safe — a filing's own context is correct for its current perio
Identifying `User-Agent` with contact email on every request; well under 10 req/s
with spacing; exponential backoff on 429; **403 → alert and stop, never
-retry-loop**. New config: `sec_user_agent`, `sec_request_spacing_seconds`,
+retry-loop** — with one carved-out exception: `www.sec.gov/Archives` is served
+from an S3 bucket without a `ListBucket` grant, so an **absent** file 403s with
+S3's `AccessDenied` XML rather than 404 (every weekend/holiday daily index does
+this). That one shape is read as "missing"; a real rejection is the WAF's
+`text/html` "Undeclared Automated Tool" page and still stops the run.
+New config: `sec_user_agent`, `sec_request_spacing_seconds`,
`sec_max_retries`. Keep only the last ~2 fetched artifacts on disk for debugging
(reproducibility is the normalized Postgres rows, per the plan).
diff --git a/tests/unit/test_sec_client.py b/tests/unit/test_sec_client.py
index f1317c7..31a82d7 100644
--- a/tests/unit/test_sec_client.py
+++ b/tests/unit/test_sec_client.py
@@ -207,6 +207,69 @@ async def test_only_404_is_treated_as_missing():
assert await c.latest_index_date(today=date(2026, 7, 22)) is None
+# The two shapes a real SEC 403 takes (captured live 2026-07-30). /Archives is
+# S3-backed with no ListBucket grant, so an ABSENT file comes back as S3's
+# AccessDenied XML; a genuine fair-access rejection is the WAF interstitial.
+S3_ACCESS_DENIED = (
+ ''
+ "AccessDeniedAccess Denied"
+ "5AWQBRAEX3NPAPHBMHFbU0a3k0ER"
+)
+WAF_HTML = (
+ "SEC.gov | Your Request Originates from "
+ "an Undeclared Automated Tool..."
+)
+
+
+def _forbidden(body: str, content_type: str) -> httpx.Response:
+ return httpx.Response(
+ 403, content=body.encode(), headers={"Content-Type": content_type}
+ )
+
+
+async def test_archives_access_denied_is_absent_not_forbidden():
+ # SEC publishes no daily index on weekends, and the bucket reports the absent
+ # key as 403/AccessDenied. Treating that as fatal wedged the importer on the
+ # first Saturday of an incremental walk (2026-07-25); it must read as "missing".
+ def handler(request: httpx.Request) -> httpx.Response:
+ url = str(request.url)
+ if url.endswith("QTR2/index.json"):
+ return httpx.Response(200, json={"directory": {"item": [{"name": "form.20260630.idx"}]}})
+ return _forbidden(S3_ACCESS_DENIED, "application/xml")
+
+ def client() -> SecClient:
+ return SecClient(
+ transport=httpx.MockTransport(handler), spacing_seconds=0, max_retries=0
+ )
+
+ async with client() as c:
+ assert await c.daily_index(date(2026, 7, 25)) == []
+ async with client() as c:
+ # QTR3 absent → the previous-quarter fallback now actually fires.
+ assert await c.latest_index_date(today=date(2026, 7, 22)) == date(2026, 6, 30)
+
+
+async def test_archives_waf_rejection_stays_forbidden():
+ # A real UA/pattern rejection is served for files that DO exist — never
+ # downgrade it, or a blocked run would look like an empty index.
+ def handler(request: httpx.Request) -> httpx.Response:
+ return _forbidden(WAF_HTML, "text/html")
+
+ async with SecClient(transport=httpx.MockTransport(handler), spacing_seconds=0) as c:
+ with pytest.raises(SecForbiddenError):
+ await c.daily_index(date(2026, 7, 21))
+
+
+async def test_access_denied_outside_archives_stays_forbidden():
+ # The downgrade is gated on the Archives prefix; data.sec.gov is not S3-backed.
+ def handler(request: httpx.Request) -> httpx.Response:
+ return _forbidden(S3_ACCESS_DENIED, "application/xml")
+
+ async with SecClient(transport=httpx.MockTransport(handler), spacing_seconds=0) as c:
+ with pytest.raises(SecForbiddenError):
+ await c.companyfacts(320193)
+
+
async def test_fair_access_validation_on_real_client():
# Placeholder email rejected.
with pytest.raises(SecError):