fix: lazy load includes child collections in on-demand sync mode (#1471)

* Unit tests to check that includes are loaded lazily on-demand

* Lazy load included rows in on-demand mode when index is available

* ci: apply automated fixes

* Fix type issue

* changeset

* fix: pass child where clauses to loadSubset in includes (#1472)

* Unit tests to check that where clauses of included collections are passed

* Unit tests to reproduce the problem with where clauses on included collections not being passed

* Pass child where clauses to loadSubset

* ci: apply automated fixes

* changeset for child where clauses fix

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: 3 people

.changeset/includes-child-where-clauses.md

@@ -0,0 +1,7 @@
+---
+'@tanstack/db': patch
+---
+
+fix: pass child where clauses to loadSubset in includes
+
+Pure-child WHERE clauses on includes subqueries (e.g., `.where(({ item }) => eq(item.status, 'active'))`) are now passed through to the child collection's `loadSubset`/`queryFn`, enabling server-side filtering. Previously only the correlation filter reached the sync layer; additional child filters were applied client-side only.

.changeset/includes-lazy-loading.md

@@ -0,0 +1,7 @@
+---
+'@tanstack/db': patch
+---
+
+fix: lazy load includes child collections in on-demand sync mode
+
+Includes child collections now use the same lazy loading mechanism as regular joins. When a query uses includes with a correlation WHERE clause (e.g., `.where(({ item }) => eq(item.rootId, r.id))`), only matching child rows are loaded on-demand via `requestSnapshot({ where: inArray(field, keys) })` instead of loading all data upfront. This ensures the sync layer's `queryFn` receives the correlation filter in `loadSubsetOptions`, enabling efficient server-side filtering.

packages/db/src/query/compiler/index.ts

@@ -4,6 +4,7 @@ import {
   join as joinOperator,
   map,
   reduce,
+  tap,
 } from '@tanstack/db-ivm'
 import { optimizeQuery } from '../optimizer.js'
 import {
@@ -22,6 +23,8 @@ import {
   Value as ValClass,
   getWhereExpression,
 } from '../ir.js'
+import { ensureIndexForField } from '../../indexes/auto-index.js'
+import { inArray } from '../builder/functions.js'
 import { compileExpression, toBooleanPredicate } from './evaluators.js'
 import { processJoins } from './joins.js'
 import { containsAggregate, processGroupBy } from './group-by.js'
@@ -379,6 +382,72 @@ export function compileQuery(
         ),
       )
 
+      // --- Includes lazy loading (mirrors join lazy loading in joins.ts) ---
+      // Resolve the child correlation field to its underlying collection + field path
+      // so we can set up an index and targeted requestSnapshot calls.
+      const childCorrelationAlias = subquery.childCorrelationField.path[0]!
+      const childFromCollection =
+        subquery.query.from.type === `collectionRef`
+          ? subquery.query.from.collection
+          : (null as unknown as Collection)
+      const followRefResult = followRef(
+        subquery.query,
+        subquery.childCorrelationField,
+        childFromCollection,
+      )
+
+      if (followRefResult) {
+        const followRefCollection = followRefResult.collection
+        const fieldPath = followRefResult.path
+        const fieldName = fieldPath[0]
+
+        // 1. Mark child source as lazy so CollectionSubscriber skips initial full load
+        lazySources.add(childCorrelationAlias)
+
+        // 2. Ensure an index on the correlation field for efficient lookups
+        if (fieldName) {
+          ensureIndexForField(fieldName, fieldPath, followRefCollection)
+        }
+
+        // 3. Tap parent keys to intercept correlation values and request
+        //    matching child rows on-demand via the child's subscription
+        parentKeys = parentKeys.pipe(
+          tap((data: any) => {
+            const resolvedAlias =
+              aliasRemapping[childCorrelationAlias] || childCorrelationAlias
+            const lazySourceSubscription = subscriptions[resolvedAlias]
+
+            if (!lazySourceSubscription) {
+              return
+            }
+
+            if (lazySourceSubscription.hasLoadedInitialState()) {
+              return
+            }
+
+            const joinKeys = [
+              ...new Set(
+                data
+                  .getInner()
+                  .map(
+                    ([[correlationValue]]: any) => correlationValue as unknown,
+                  )
+                  .filter((key: unknown) => key != null),
+              ),
+            ]
+
+            if (joinKeys.length === 0) {
+              return
+            }
+
+            const lazyJoinRef = new PropRef(fieldPath)
+            lazySourceSubscription.requestSnapshot({
+              where: inArray(lazyJoinRef, joinKeys),
+            })
+          }),
+        )
+      }
+
       // If parent filters exist, append them to the child query's WHERE
       const childQuery =
         subquery.parentFilters && subquery.parentFilters.length > 0
@@ -410,6 +479,9 @@ export function compileQuery(
       // Merge child's alias metadata into parent's
       Object.assign(aliasToCollectionId, childResult.aliasToCollectionId)
       Object.assign(aliasRemapping, childResult.aliasRemapping)
+      for (const [alias, whereClause] of childResult.sourceWhereClauses) {
+        sourceWhereClauses.set(alias, whereClause)
+      }
 
       includesResults.push({
         pipeline: childResult.pipeline,