dennisthiessen/signal-platform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

research: single-source liquid mask; orphan +0.06 fip IC

Browse Source 
Harness and diagnostics share _filter_liquid_breadth_week_rich. Recompute
shows unconditional liquid fip IC -0.017 (mask binds 97%); mom-conditional
-0.088/t-4.58 stands. Document +0.0575 as orphaned.
This commit is contained in:
dennisthiessen
2026-07-19 00:06:04 +02:00
parent ceaaadc49f
commit 7d60e54f5a
4 changed files with 7280 additions and 526 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/services/backtest_service.py
+96 -11
View File
@@ -942,6 +942,8 @@ def _accumulate_signal_series(
records: list,
collected: dict,
benchmark_closes: dict[date, float] | None = None,
*,
symbol: str | None = None,
) -> None:
"""For each weekly as-of bar, emit (signal, forward-return) pairs keyed by ISO
week into ``collected[name][week_key]``. Forward return is close-to-close over
@@ -974,6 +976,7 @@ def _accumulate_signal_series(
"fwd": fwd,
"close": closes[i],
"median_dvol_63": dvol,
"symbol": symbol,
})
else:
collected[name][week_key].append((val, fwd))
@@ -1041,12 +1044,28 @@ def _filter_liquid_breadth_week(
Ranking is relative (IEX volume undercount is OK for order stats). Membership
is recomputed every week from as-of bars — never frozen from today's liquidity.
"""
ranked: list[tuple[float, float, float]] = [] # (-dvol, val, fwd)
kept = _filter_liquid_breadth_week_rich(
recs, top_n=top_n, min_price=min_price
)
return [(float(r["val"]), float(r["fwd"])) for r in kept]
def _filter_liquid_breadth_week_rich(
recs: list,
*,
top_n: int,
min_price: float,
) -> list[dict]:
"""Same mask as ``_filter_liquid_breadth_week``, returning rich rows.
Single source for harness IC and research diagnostics. Eligible pool =
dict observations with close ≥ min_price and median_dvol_63 > 0; then
keep top_n by dollar volume (highest first). Non-dict legacy tuples are
not eligible for the liquid mask (they have no dvol).
"""
eligible: list[tuple[float, dict]] = [] # (-dvol, row)
for rec in recs:
if not isinstance(rec, dict):
pair = _obs_val_fwd(rec)
if pair is not None:
ranked.append((0.0, pair[0], pair[1]))
continue
close = rec.get("close")
dvol = rec.get("median_dvol_63")
@@ -1057,10 +1076,50 @@ def _filter_liquid_breadth_week(
pair = _obs_val_fwd(rec)
if pair is None:
continue
ranked.append((-float(dvol), pair[0], pair[1]))
ranked.sort(key=lambda row: row[0])
kept = ranked[:top_n]
return [(val, fwd) for _, val, fwd in kept]
row = {
"val": pair[0],
"fwd": pair[1],
"close": float(close),
"median_dvol_63": float(dvol),
"symbol": rec.get("symbol"),
}
# Preserve optional research fields for mom-conditional diagnostics.
for key in ("mom_12_1", "mom_12_1_resid", "vol_6m", "fip_id"):
if key in rec and rec[key] is not None:
row[key] = rec[key]
eligible.append((-float(dvol), row))
eligible.sort(key=lambda item: item[0])
return [row for _, row in eligible[:top_n]]
def _liquid_breadth_week_stats(
recs: list,
*,
top_n: int,
min_price: float,
) -> dict[str, int | bool]:
"""Pre/post mask counts for reconciling avg_cross_section semantics."""
raw = len(recs)
eligible = 0
for rec in recs:
if not isinstance(rec, dict):
continue
close = rec.get("close")
dvol = rec.get("median_dvol_63")
if close is None or float(close) < min_price:
continue
if dvol is None or float(dvol) <= 0:
continue
if _obs_val_fwd(rec) is None:
continue
eligible += 1
post = min(eligible, top_n) if top_n > 0 else eligible
return {
"raw_pool": raw,
"eligible_pre_mask": eligible,
"post_mask": post,
"mask_binds": bool(top_n > 0 and eligible > top_n),
}
def _quintile_spread(pairs: list[tuple[float, float]]) -> float | None:
@@ -1126,9 +1185,18 @@ def _signal_evaluation(collected: dict) -> list[dict]:
ics: list[float] = []
spreads: list[float] = []
sizes: list[int] = []
raw_sizes: list[int] = []
eligible_sizes: list[int] = []
bind_flags: list[bool] = []
for wk in kept:
recs = weeks_map[wk]
if top_n > 0:
stats = _liquid_breadth_week_stats(
recs, top_n=top_n, min_price=min_price
)
raw_sizes.append(int(stats["raw_pool"]))
eligible_sizes.append(int(stats["eligible_pre_mask"]))
bind_flags.append(bool(stats["mask_binds"]))
pairs = _filter_liquid_breadth_week(
recs, top_n=top_n, min_price=min_price
)
@@ -1146,6 +1214,7 @@ def _signal_evaluation(collected: dict) -> list[dict]:
spread = _quintile_spread(pairs)
if spread is not None:
spreads.append(spread)
# avg_cross_section is ALWAYS post-mask pair count (the IC sample).
sizes.append(len(pairs))
if not ics:
continue
@@ -1168,16 +1237,32 @@ def _signal_evaluation(collected: dict) -> list[dict]:
if top_n > 0:
row["liquid_breadth_top_n"] = top_n
row["liquid_min_price"] = min_price
# Explicit pre/post mask diagnostics (reconcile "did top-N bind?").
if raw_sizes:
row["avg_raw_pool"] = round(sum(raw_sizes) / len(raw_sizes), 1)
if eligible_sizes:
row["avg_eligible_pre_mask"] = round(
sum(eligible_sizes) / len(eligible_sizes), 1
)
if bind_flags:
row["mask_binds_pct"] = round(
sum(1 for b in bind_flags if b) / len(bind_flags) * 100, 1
)
rows.append(row)
rows.sort(key=lambda r: r["mean_ic"], reverse=True)
return rows
def _signal_series(records: list, benchmark_closes: dict[date, float] | None = None) -> dict:
def _signal_series(
records: list,
benchmark_closes: dict[date, float] | None = None,
*,
symbol: str | None = None,
) -> dict:
"""Per-ticker signal/forward-return series as a PLAIN (picklable) nested dict
— no defaultdict/lambda — so it can cross a process boundary."""
tmp: dict = defaultdict(lambda: defaultdict(list))
_accumulate_signal_series(records, tmp, benchmark_closes)
_accumulate_signal_series(records, tmp, benchmark_closes, symbol=symbol)
return {name: dict(weeks) for name, weeks in tmp.items()}
@@ -1218,7 +1303,7 @@ def _replay_and_signals(
)
return (
candidates,
_signal_series(bars, benchmark_closes),
_signal_series(bars, benchmark_closes, symbol=symbol),
)
docs/research/fip-breadth-ic.md
+81 -87
View File
@@ -1,151 +1,145 @@
# Broad-universe fip_id IC research (Phase B)
**Status:** research complete enough for a platform decision on *unconditional* fip.
**Status:** unconditional fip closed; mom-conditional lead confirmed on single-sourced path.
**Production impact:** none. Display card remains context-only.
Generated: 2026-07-18 (breadth run + diagnostics same day).
## Scope
- **Research only** — production universe, gate, scanner, schedule unchanged.
- Price-only signal harness; no sentiment/fundamentals on the broad tier.
- Snapshot: `research.sqlite` — ~4,650 tickers (prod + nasdaq_all extend).
- IC mask: top **1,500** by point-in-time 63d median $vol, price ≥ **$5**, per week.
- Research only — production universe, gate, scanner, schedule unchanged.
- Snapshot: `research.sqlite` (~4,650 tickers = prod + nasdaq_all extend).
- Liquid mask: top **1,500** by point-in-time 63d median $vol, price ≥ **$5**/week.
## Caveats
- **Survivorship bias** — today's constituents, history backfilled (worse in small caps).
- **IEX volume undercount** — relative $vol rank only, not absolute floors.
- **Pool skew** — nasdaq_all ∪ partial SPX seed tilts tech/biotech; missing pure NYSE mid-caps.
- **Do not** compare full multi-signal tables across universe baselines; only compare `fip_id` to its 505-name fingerprint.
- Survivorship bias (today’s constituents, history backfilled).
- IEX volume undercount → relative $vol rank only.
- Pool skew: Nasdaq-heavy; missing pure NYSE mid-caps.
- Do not mix multi-signal tables across universe baselines.
---
## Fingerprint (505-name prod snapshot)
## Fingerprint (505-name prod)
| | Expected | Observed |
|---|---:|---:|
| mean IC | −0.045 | **−0.045** |
| t-stat | −2.9 | **−2.91** |
| weeks | ≥12 | 35 |
| avg N | ~500 | 497.7 |
| reliable | true | **true** |
| weeks / N / reliable | ≥12 / ~500 / true | 35 / 497.7 / true |
**Pass.** Pipeline and formula are trustworthy.
Artifacts: `reports/fip-breadth-20260718-211440-fingerprint.json`
**Pass.** Formula + pipeline trustworthy.
---
## First breadth harness run (pre-registered iron rule)
## Discrepancy (must not be papered over)
Unconditional `fip_id` on liquid top-1500 (runner `run_fip_breadth_research.py`):
| Source | fip IC (liquid ~1500) | t |
|---|---:|---:|
| Report `fip-breadth-20260718-211440-breadth.json` | **+0.0575** | **+5.12** |
| Single-sourced recompute (2026-07-19) | **−0.0168** | **−1.85** |
That is a **sign disagreement** on the same intended quantity. Method rule: the number you cannot reconcile is the number you cannot use.
### What we did
1. **Single-sourced the mask** — diagnostics call harness `_signal_series` + `_filter_liquid_breadth_week_rich` only (no parallel mask).
2. **Documented avg_cross_section semantics** — always **post-mask** IC sample size.
3. **Logged pre-mask stats** so “did top-N bind?” is answerable.
### Authoritative unconditional liquid fip (post-reconciliation)
| metric | value |
|---|---:|
| mean_ic | **+0.0575** |
| ic_t_stat | **+5.12** |
| ic_positive_pct | 88.6% |
| mean_ic | **−0.0168** |
| ic_t_stat | **−1.85** |
| weeks | 35 |
| avg_cross_section | 1471.2 |
| avg_cross_section (**post-mask**) | 1471.2 |
| avg_raw_pool | 3214.4 |
| avg_eligible_pre_mask | **2338.4** |
| mask_binds_pct | **97.1%** |
| reliable | true |
**Iron rule as written (need negative sign):** **not green.**
Honest call: no production change from that screen alone.
**Mask binds hard** (eligible ≫ 1500). The hypothesis that “1471 meant the mask never bound / unmasked +5σ” is **false**.
Artifact: `reports/fip-breadth-20260718-211440-breadth.json`
Harness `_signal_evaluation` vs manual IC through the same filter: **exact match** (−0.0168 / −1.85).
### Verdict on the orphan
The **+0.0575 / t +5.12** row is **orphaned**. Do not cite it. Root cause of that single run is not fully forensic-reconstructed (no dual dump from the original process remains), but every single-sourced recompute on this snapshot lands near **−0.017**, and the tier blend (≈800×−0.035 + ≈670×+0.014)/1471 ≈ **−0.013** is internally consistent with that number—not with +0.058.
**Iron rule unconditional:** still **not green** (|IC| 0.017 < 0.03), and now with the correct mild-negative sign.
Artifact: `reports/fip-reconcile-20260719-000520.json`
---
## Why “+IC on Nasdaq” is not a jumpiness-premium story
## Compositional story (supported)
`fip_id = sign(PRET) × (%neg − %pos)` **pools two opposite continuous populations:**
`fip_id = sign(PRET)×(%neg−%pos)` pools:
| Leg | Formation | Continuation intuition | IC contribution |
|---|---|---|---|
| **Continuous winners** | PRET>0, mostly up days (smooth climbers) | Paper: keep going up | **negative** |
| **Continuous losers / bleeders** | PRET<0, mostly down days (grind-down biotechs, SPACs, etc.) | Momentum: keep going down | **positive** |
- **Continuous winners** → want **negative** IC
- **Continuous bleeders** → want **positive** IC
Unconditional IC is a **tug-of-war weighted by universe composition**:
| check | IC | t | read |
|---|---:|---:|---|
| Prod-universe subset inside liquid | **−0.044** | **−2.88** | Matches fingerprint → compositional, not regime change |
| Tier 1–800 (senior) | **−0.035** | **−2.99** | Winner leg |
| Tier 801–1500 (junior) | **+0.014** | +1.25 | More bleeder / junk weight |
| Lagged membership (prior-week $vol) | −0.010 | −0.93 | Same sign as same-week; not a +5σ leak artifact |
- **S&P-like book** ≈ few steady bleeders → winner leg dominates → IC **−0.045**.
- **Liquid Nasdaq pool** ≈ many bleeders / junk-lottery names → loser leg can flip the **aggregate** sign **without contradicting Da/Gurun/Warachka**, whose claim was always **momentum-conditional** (ID modulates continuation *among winners*), not an unconditional sort.
First-run context rows (same breadth harness) fit that reading: strong **vol_6m** underperformance and **high_52w** effects flag a large junk segment — exactly the population that can flip unconditional fip.
**Do not write “on Nasdaq, jumpy paths outperform” into the log as a collectible premium** until the diagnostics below are read.
**Do not log “on Nasdaq, jumpy paths outperform.”** That would mythologize an orphaned +0.06.
---
## Follow-up diagnostics (same snapshot, independent panel)
## Platform-relevant test: momentum-conditional fip
Script: `scripts/run_fip_breadth_diagnostics.py`
Artifact: `reports/fip-breadth-diagnostics-20260718-213908.json`
Among liquid top-1500, keep **mom_12_1 ≥ P80** (~294 names/week):
| check | mean_ic | t | weeks | avg N | reliable |
|---|---:|---:|---:|---:|---|
| fip same-week liquid 1500 (panel) | −0.017 | −1.85 | 35 | 1471 | true |
| fip **lagged membership** (prior-week $vol) | −0.010 | −0.93 | 35 | 1471 | true |
| fip **tier 1–800** (senior liquid) | **−0.035** | **−2.99** | 35 | 791 | true |
| fip **tier 801–1500** (junior liquid) | **+0.014** | +1.25 | 35 | 700 | true |
| fip **prod-universe subset** inside liquid | **−0.044** | **−2.88** | 35 | 498 | true |
| fip **mom-conditional** (top 20% mom_12_1) | **−0.088** | **−4.58** | 35 | 294 | true |
| vol_6m liquid 1500 (panel) | −0.047 | −1.3 | 35 | 1471 | true |
| mom_12_1 liquid 1500 | +0.046 | +1.91 | 35 | 1471 | true |
| mom_12_1_resid liquid 1500 | +0.029 | +1.33 | 35 | 1471 | true |
| metric | value |
|---|---:|
| mean_ic | **−0.0879** |
| ic_t_stat | **−4.58** |
| ic_positive_pct | 22.9% |
| weeks | 35 |
| reliable | **true** |
### What the checks settle
Computed on the **same single-sourced path** as the authoritative −0.017. This is the paper’s claim and the only version a gate could consume.
1. **Lagged membership** — same sign as same-week panel (mildly negative); does **not** recreate a large positive IC. Not a clean “liquidity explosion leak manufactures +0.06” story for the panel path. (The first harness run’s **+0.0575** still does not match the independent panel’s −0.017 — treat the **+0.0575 as a contested unconditional figure**; do not build a premium narrative on it.)
2. **Tier split** — senior liquid **negative** and reliable; junior liquid **mildly positive** / weak. Bias and bleeder weight are stronger in the junior tier.
3. **Prod-universe subset** — IC **−0.044 / t −2.88**, ~498 names/week — matches the fingerprint. **Sign flip is compositional**, not “the whole market regime flipped.”
4. **Momentum-conditional fip (the platform test)** — IC **−0.088 / t −4.58**, reliable, ~294 winners/week. **Negative sign, |IC| ≳ 0.03.** This is the paper’s claim and the only version a gate could consume.
### Platform verdict
| Question | Answer |
| Decision | |
|---|---|
| Unconditional fip iron rule (negative on liquid-1500) | **Not green** (first harness +0.06 fails sign; panel mild neg fails magnitude) |
| Production change now? | **No** |
| Is fip “dead forever”? | **No** — **alive only as a momentum-conditional tilt candidate** on breadth |
| Next real step if pursued | Book-level experiment: among qualified residual-momentum names, tilt/filter by lower fip — **not** an unconditional fip sort |
| Display card | Stays; still the right home until a book test wins |
| Unconditional fip | **Closed** for production |
| Mom-conditional fip | **Alive as book-tilt candidate only** — book sim before any gate talk |
| Display card | Stays |
| Production change | **None** |
---
## Buried headline: vol tilt / residual mom on breadth
## Vol-tilt / residual-mom warning (any future breadth move)
Even with panel vs harness magnitude differences, the **direction** is clear:
| signal (liquid, single-sourced) | IC | t |
|---|---:|---:|
| vol_6m | −0.048 | −1.4 |
| mom_12_1 | +0.046 | +1.9 |
| mom_12_1_resid | +0.029 | +1.3 |
- **High vol underperforms** on this pool relative to a clean S&P-like book.
- Production rank tilts **20% toward high volatility**, validated on S&P-like names where high-vol ≈ high-beta in a bull tape. On broad Nasdaq liquid, high-vol often means **lottery junk**.
- **If the universe ever broadens in production, re-validate the 80/20 high-vol tilt first** — it can flip from mildly helpful to actively harmful.
- **Raw momentum > residual** on breadth (panel and first harness both show this pattern) — SPY residualization is a noisier fit for small caps; a breadth book may want a different benchmark or raw mom.
High-vol names tend to underperform on this pool relative to a clean S&P-like book. Production **80/20 high-vol tilt** was validated on S&P-like names. **If the universe ever broadens in production, re-validate that tilt first** — it can flip from mildly helpful to harmful. Raw momentum also looks stronger than SPY residualization here (noisier fit for small caps).
---
## How to re-run (research branch only)
```powershell
# Windows
.\.venv\Scripts\python.exe scripts\run_fip_breadth_diagnostics.py `
--research-snapshot backtest_snapshots\research.sqlite `
--prod-snapshot backtest_snapshots\prod.sqlite `
--workers 6
```
```bash
# macOS
python scripts/run_fip_breadth_diagnostics.py \
--research-snapshot backtest_snapshots/research.sqlite \
--prod-snapshot backtest_snapshots/prod.sqlite \
--workers 6
--workers 6 --allow-spawn
```
---
## Bottom line
- Formal first screen: **not green**, no production change, fingerprint **pass**.
- Deeper reading: unconditional sign is a **compositional tug-of-war**, not a new jumpiness premium.
- **The test that matters for this platform already ran:** momentum-conditional fip is **negative, large, and reliable** on liquid breadth → fip remains a **conditional** research lead, not a closed door — and **not** a ship-ready gate input without a book experiment.
1. Formal iron-rule screen: **not green** either before or after reconciliation.
2. **+0.0575 / +5.12 is orphaned** — authoritative unconditional liquid fip is **−0.017 / −1.9**; mask binds (~97%).
3. Compositional tug-of-war is the right story; jumpiness premium is not.
4. **Mom-conditional −0.088 / −4.6 stands on the single-sourced path** → optional next research step is a **book** A/B, not a gate wire-in.
5. Log any future reader who sees both numbers: trust the reconcile artifact, not the orphaned breadth headline.
reports/fip-reconcile-20260719-000520.json
+6693
View File
File diff suppressed because it is too large Load Diff
scripts/run_fip_breadth_diagnostics.py
+410 -428
View File
File diff suppressed because it is too large Load Diff