[codex] Land core platform migration slices

CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ## Unreleased
 
+- Fixed `mypy alphaforge` regressions across ref-period PIT query normalization, `DataContext` adapter signatures, PIT adapter wrappers, cache manifest aggregation, futures loaders, and optional-store handling so the existing type-check CI gate passes on this branch.
+- Declared explicit `pytz` runtime dependency and `types-pytz` development stub so fresh installs and docs-example CI no longer rely on pandas pulling timezone support transitively.
+- Clarified the PIT ref-query public contract: `RefSnapshotQuery` / `RefRevisionQuery` plus `snapshot_ref(...)` / `revisions_ref(...)` are the canonical surface, while `get_snapshot_ref(...)` and `get_revision_timeline_ref(...)` remain compatibility wrappers.
 - Added unified data layer with `SourceAdapter` protocol, `SourceAdapterBase` mixin, `FetchResult`/`CacheManifest` value types, and `CacheLayer` (DuckDB-backed PIT/market cache).
 - Added built-in source adapters: `TiingoAdapter` (market OHLCV), `FREDSourceAdapter` (macro PIT), `CFTCAdapter` (CoT positioning), `DTCCAdapter` (swap derivatives).
 - Added `alphaforge.source_adapters` entry-point group and `discover_adapters()` for plugin-style adapter registration.

alphaforge/__init__.py

@@ -10,6 +10,8 @@
     BCBSGSDataSource,
     BEADataSource,
     BLSDataSource,
+    CFTCCoTSource,
+    CFTCDisaggregatedCoTSource,
     CFTCWeeklySwapsSource,
     CMEProductSlateSource,
     DestatisGenesisDataSource,
@@ -24,7 +26,9 @@
     FRBTermStructureBenchmarkSource,
     IBGESidraDataSource,
     LCHCDSClearDailySource,
+    MOFJGBYieldCurveSource,
     PhiladelphiaSPFMeanLevelSource,
+    default_public_web_sources,
 )
 from .data.query import Query
 from .data.schema import TableSchema
@@ -37,6 +41,7 @@
 )
 from .data.universe import EntityMetadata, Universe
 from .features.frame import Artifact, FeatureFrame
+from .features.market import LagReturnsTemplate, RollingVolatilityTemplate
 from .features.ops import join_feature_frames, materialize
 from .features.realization import FeatureRealization, FitState
 from .features.template import FeatureTemplate, ParamSpec, SliceSpec
@@ -85,6 +90,12 @@
     PITPipelineStep,
     coerce_pipeline_spec,
 )
+from .pit.queries import (
+    RefRevisionQuery,
+    RefSnapshotQuery,
+    coerce_ref_revision_query,
+    coerce_ref_snapshot_query,
+)
 from .pit.ref_entity import make_ref_entity_id, parse_ref_entity_id
 from .pit.tasks import (
     build_snapshot_tape,
@@ -110,7 +121,25 @@
 from .time.align import AlignedPanel, AlignSpec, AvailabilityState, align_panel
 from .time.calendar import TradingCalendar
 from .time.grids import EventGrid, Grid, NativeGrid, SessionGrid
-from .time.ref_period import RefFreq, RefPeriod
+from .time.missingness import MissingnessReason, classify_missingness
+from .time.ref_period import (
+    ObsDateAnchor,
+    RefFreq,
+    RefPeriod,
+    coerce_ref_period,
+    normalize_obs_date_anchor,
+    normalize_ref_freq,
+)
+from .time.release_rules import (
+    CalendarDay,
+    CustomRule,
+    FixedLagMonths,
+    NthBusinessDay,
+    NthWeekday,
+    QuarterlyRelease,
+    ReleaseRule,
+    WeeklyRelease,
+)
 
 __all__ = [
     "DataContext",
@@ -132,6 +161,8 @@
     "load_first_rate_futures_metadata",
     "BLSDataSource",
     "BEADataSource",
+    "CFTCCoTSource",
+    "CFTCDisaggregatedCoTSource",
     "EIADataSource",
     "EurostatDataSource",
     "ECBSDMXDataSource",
@@ -149,7 +180,9 @@
     "LCHCDSClearDailySource",
     "EzoicAdRevenueDailySource",
     "FRBTermStructureBenchmarkSource",
+    "MOFJGBYieldCurveSource",
     "PhiladelphiaSPFMeanLevelSource",
+    "default_public_web_sources",
     "build_kim_orphanides_dataset",
     "build_policy_rule_dataset",
     "build_duan_weekly_dataset",
@@ -167,9 +200,25 @@
     "align_panel",
     "RefFreq",
     "RefPeriod",
+    "ObsDateAnchor",
+    "coerce_ref_period",
+    "normalize_ref_freq",
+    "normalize_obs_date_anchor",
+    "ReleaseRule",
+    "NthBusinessDay",
+    "NthWeekday",
+    "CalendarDay",
+    "FixedLagMonths",
+    "QuarterlyRelease",
+    "WeeklyRelease",
+    "CustomRule",
+    "MissingnessReason",
+    "classify_missingness",
     "FeatureFrame",
     "Artifact",
+    "LagReturnsTemplate",
     "ParamSpec",
+    "RollingVolatilityTemplate",
     "SliceSpec",
     "FeatureTemplate",
     "FeatureRealization",
@@ -193,6 +242,10 @@
     "PITPipelineSpec",
     "PITPipelineResult",
     "coerce_pipeline_spec",
+    "RefSnapshotQuery",
+    "RefRevisionQuery",
+    "coerce_ref_snapshot_query",
+    "coerce_ref_revision_query",
     "PITExpressionNode",
     "PITExpressionGraphSpec",
     "PITExpressionGraphResult",

alphaforge/data/cache_layer.py

@@ -113,25 +113,28 @@ def store(
 
         # Update manifest — query aggregate stats from the full table
         # Get total row count for this dataset+source in the table
-        total_rows = self._conn.execute(
+        total_rows_row = self._conn.execute(
             f"SELECT COUNT(*) FROM {table} WHERE dataset = ? AND source = ?",
             [dataset, source],
-        ).fetchone()[0]
+        ).fetchone()
+        total_rows = int(total_rows_row[0]) if total_rows_row is not None else 0
 
         # Get all entity keys for this dataset+source
         all_keys = self._conn.execute(
             f"SELECT DISTINCT series_key FROM {table} WHERE dataset = ? AND source = ? ORDER BY series_key",
             [dataset, source],
         ).fetchdf()["series_key"].tolist()
 
-        all_min = self._conn.execute(
+        all_min_row = self._conn.execute(
             f"SELECT MIN(obs_date) FROM {table} WHERE dataset = ? AND source = ?",
             [dataset, source],
-        ).fetchone()[0]
-        all_max = self._conn.execute(
+        ).fetchone()
+        all_min = all_min_row[0] if all_min_row is not None else None
+        all_max_row = self._conn.execute(
             f"SELECT MAX(obs_date) FROM {table} WHERE dataset = ? AND source = ?",
             [dataset, source],
-        ).fetchone()[0]
+        ).fetchone()
+        all_max = all_max_row[0] if all_max_row is not None else None
 
         entity_keys_str = ",".join(all_keys)
 

alphaforge/data/context.py

@@ -1,8 +1,8 @@
 from __future__ import annotations
 
 from dataclasses import dataclass, field
-from datetime import timedelta
-from typing import TYPE_CHECKING, Mapping, Optional
+from datetime import date, timedelta
+from typing import TYPE_CHECKING, Mapping, Optional, Sequence
 
 import pandas as pd
 
@@ -11,7 +11,7 @@
 from ..store.store import Store
 from ..time.calendar import TradingCalendar
 from .panel import PanelFrame
-from .query import Query
+from .query import Query, VintageMode
 from .source import DataSource
 from .universe import EntityMetadata, Universe
 
@@ -27,20 +27,23 @@ class DataContext:
     Parameters
     ----------
     sources : Mapping[str, DataSource]
-        Legacy data source mapping (backward compatibility).
+        Legacy data source mapping kept for backward compatibility and
+        raw-loader workflows.
     calendars : Mapping[str, TradingCalendar]
         Trading calendar lookup.
     store : Store
         Backing store for persistence.
     adapters : dict[str, SourceAdapter] | None
-        Unified source adapters keyed by source_name (e.g. ``"cftc"``).
+        Canonical public data-loading surface keyed by source_name
+        (e.g. ``"cftc"``).
     default_sources : dict[str, str] | None
-        Maps dataset → default source_name (e.g. ``{"cot.tff": "cftc"}``).
+        Maps dataset → default source_name for canonical adapter routing
+        (e.g. ``{"cot.tff": "cftc"}``).
     """
 
     sources: Mapping[str, DataSource]
     calendars: Mapping[str, TradingCalendar]
-    store: Store
+    store: Store | None
     universe: Optional[Universe] = None
     entity_meta: Optional[EntityMetadata] = None
     adapters: Optional[dict[str, "SourceAdapter"]] = field(default=None)
@@ -52,6 +55,47 @@ class DataContext:
         init=False, default_factory=dict, repr=False
     )
 
+    @classmethod
+    def from_adapters(
+        cls,
+        *adapters: "SourceAdapter",
+        calendars: Mapping[str, TradingCalendar] | None = None,
+        store: Store | None = None,
+        universe: Optional[Universe] = None,
+        entity_meta: Optional[EntityMetadata] = None,
+        default_sources: Optional[dict[str, str]] = None,
+    ) -> "DataContext":
+        """Build a DataContext from adapters without manual mapping boilerplate."""
+        adapter_map: dict[str, "SourceAdapter"] = {}
+        dataset_to_sources: dict[str, list[str]] = {}
+
+        for adapter in adapters:
+            if adapter.source_name in adapter_map:
+                raise ValueError(
+                    f"Duplicate adapter source_name: {adapter.source_name!r}"
+                )
+            adapter_map[adapter.source_name] = adapter
+            for dataset in adapter.datasets:
+                dataset_to_sources.setdefault(dataset, []).append(adapter.source_name)
+
+        derived_defaults = {
+            dataset: sources[0]
+            for dataset, sources in dataset_to_sources.items()
+            if len(sources) == 1
+        }
+        merged_defaults = dict(derived_defaults)
+        merged_defaults.update(default_sources or {})
+
+        return cls(
+            sources={},
+            calendars=dict(calendars or {}),
+            store=store,
+            universe=universe,
+            entity_meta=entity_meta,
+            adapters=adapter_map,
+            default_sources=merged_defaults or None,
+        )
+
     def __post_init__(self) -> None:
         if isinstance(self.store, DuckDBParquetStore):
             self.pit = PITAccessor(self.store.conn())
@@ -63,6 +107,7 @@ def __post_init__(self) -> None:
                     self._dataset_to_sources.setdefault(ds, []).append(source_name)
 
     def fetch_panel(self, source: str, q: Query) -> PanelFrame:
+        """Legacy panel-building path for DataSource-backed loaders."""
         df = self.sources[source].fetch(q)
 
         try:
@@ -145,12 +190,29 @@ def _resolve_source(
         # Use default_sources mapping
         if self.default_sources and dataset in self.default_sources:
             default_src = self.default_sources[dataset]
-            if default_src in self.adapters:
-                return self.adapters[default_src]
+            if default_src not in self.adapters:
+                raise KeyError(
+                    f"Default source '{default_src}' for dataset '{dataset}' is not "
+                    f"registered. Available: {sorted(self.adapters.keys())}"
+                )
+            adapter = self.adapters[default_src]
+            if dataset not in adapter.datasets:
+                raise KeyError(
+                    f"Default source '{default_src}' does not serve dataset "
+                    f"'{dataset}'. It serves: {sorted(adapter.datasets)}"
+                )
+            return adapter
 
         # Fallback: find any adapter that serves this dataset
         if dataset in self._dataset_to_sources:
-            src_name = self._dataset_to_sources[dataset][0]
+            source_names = self._dataset_to_sources[dataset]
+            if len(source_names) > 1:
+                raise KeyError(
+                    f"Multiple adapters serve dataset '{dataset}': "
+                    f"{sorted(source_names)}. Configure default_sources or "
+                    "pass source= explicitly."
+                )
+            src_name = source_names[0]
             return self.adapters[src_name]
 
         raise KeyError(
@@ -165,30 +227,86 @@ def fetch(
         source: Optional[str] = None,
         max_staleness: Optional[timedelta] = None,
     ) -> "FetchResult":
-        """Unified fetch: resolve adapter and delegate."""
+        """Canonical fetch path: resolve an adapter and delegate."""
         adapter = self._resolve_source(query.table, source)
         return adapter.fetch(query, max_staleness=max_staleness)
 
+    def load(
+        self,
+        dataset: str,
+        *,
+        columns: Sequence[str],
+        start: Optional[pd.Timestamp | str] = None,
+        end: Optional[pd.Timestamp | str] = None,
+        entities: Optional[Sequence[str]] = None,
+        asof: Optional[pd.Timestamp | str] = None,
+        vintage: VintageMode = "latest",
+        vintage_id: Optional[str] = None,
+        grid: Optional[str] = None,
+        source: Optional[str] = None,
+        max_staleness: Optional[timedelta] = None,
+    ) -> "FetchResult":
+        """Happy-path source load without explicit Query construction."""
+        return self.fetch(
+            Query(
+                table=dataset,
+                columns=list(columns),
+                start=start,
+                end=end,
+                entities=list(entities) if entities is not None else None,
+                asof=asof,
+                vintage=vintage,
+                vintage_id=vintage_id,
+                grid=grid,
+            ),
+            source=source,
+            max_staleness=max_staleness,
+        )
+
     def fetch_many(
         self,
         queries: list[Query],
         *,
         source: Optional[str] = None,
         max_staleness: Optional[timedelta] = None,
     ) -> list["FetchResult"]:
-        """Fetch multiple queries, routing each to the correct adapter."""
-        results = []
-        for q in queries:
-            adapter = self._resolve_source(q.table, source)
-            results.append(adapter.fetch(q, max_staleness=max_staleness))
-        return results
+        """Canonical batch fetch path, grouped by resolved adapter."""
+        if not queries:
+            return []
+
+        grouped_queries: dict[int, tuple["SourceAdapter", list[tuple[int, Query]]]] = {}
+        for idx, query in enumerate(queries):
+            adapter = self._resolve_source(query.table, source)
+            adapter_key = id(adapter)
+            if adapter_key not in grouped_queries:
+                grouped_queries[adapter_key] = (adapter, [])
+            grouped_queries[adapter_key][1].append((idx, query))
+
+        results: list[Optional["FetchResult"]] = [None] * len(queries)
+        for adapter, indexed_queries in grouped_queries.values():
+            batch_queries = [query for _, query in indexed_queries]
+            batch_results = adapter.fetch_many(
+                batch_queries,
+                max_staleness=max_staleness,
+            )
+            if len(batch_results) != len(batch_queries):
+                raise ValueError(
+                    f"Adapter '{adapter.source_name}' returned {len(batch_results)} "
+                    f"results for {len(batch_queries)} queries."
+                )
+            for (idx, _), result in zip(indexed_queries, batch_results):
+                results[idx] = result
+
+        if any(result is None for result in results):
+            raise ValueError("fetch_many() did not populate every requested result.")
+        return [result for result in results if result is not None]
 
     def prefetch(
         self,
         dataset: str,
         *,
         source: Optional[str] = None,
-        asof_range: tuple = None,
+        asof_range: tuple[date, date] | None = None,
     ) -> "CacheManifest":
         """Warm cache for a dataset via the resolved adapter."""
         adapter = self._resolve_source(dataset, source)