From 0bd0f1c4d3142ed73c34d8f68ba405ba9db18e90 Mon Sep 17 00:00:00 2001
From: Diego Costa <diecoscai@gmail.com>
Date: Thu, 12 Mar 2026 15:39:07 -0300
Subject: [PATCH 01/14] fix(components): optimize AAIDomains loader memory for
 large datasets

---
 .../documentloaders/AAIDomains/AAIDomains.ts  | 69 +++++++------------
 1 file changed, 23 insertions(+), 46 deletions(-)

diff --git a/packages/components/nodes/documentloaders/AAIDomains/AAIDomains.ts b/packages/components/nodes/documentloaders/AAIDomains/AAIDomains.ts
index 16feafd87c8..013b86c89ce 100644
--- a/packages/components/nodes/documentloaders/AAIDomains/AAIDomains.ts
+++ b/packages/components/nodes/documentloaders/AAIDomains/AAIDomains.ts
@@ -273,36 +273,13 @@ class AAIDomains_DocumentLoaders implements INode {
         }
 
         // Apply metadata
-        if (metadata) {
-            const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
-            docs = docs.map((doc) => ({
-                ...doc,
-                metadata:
-                    _omitMetadataKeys === '*'
-                        ? {
-                              ...parsedMetadata
-                          }
-                        : omit(
-                              {
-                                  ...doc.metadata,
-                                  ...parsedMetadata
-                              },
-                              omitMetadataKeys
-                          )
-            }))
-        } else {
-            docs = docs.map((doc) => ({
-                ...doc,
-                metadata:
-                    _omitMetadataKeys === '*'
-                        ? {}
-                        : omit(
-                              {
-                                  ...doc.metadata
-                              },
-                              omitMetadataKeys
-                          )
-            }))
+        const parsedMetadata = metadata ? (typeof metadata === 'object' ? metadata : JSON.parse(metadata)) : null
+        for (const doc of docs) {
+            if (_omitMetadataKeys === '*') {
+                doc.metadata = parsedMetadata ? { ...parsedMetadata } : {}
+            } else {
+                doc.metadata = omit({ ...doc.metadata, ...(parsedMetadata || {}) }, omitMetadataKeys)
+            }
         }
 
         if (output === 'text') {
@@ -375,7 +352,7 @@ class AAIDomainsLoader extends BaseDocumentLoader {
 
         // Use larger page size since we're only selecting essential fields
         const pageSize = Math.min(this.limit, 100)
-        let allDomains: any[] = []
+        let allDocs: IDocument[] = []
         let currentPage = 0
 
         console.info('[AAIDomains] Starting load with params:', {
@@ -388,8 +365,8 @@ class AAIDomainsLoader extends BaseDocumentLoader {
             hasAnalysis: this.hasAnalysis
         })
 
-        while (allDomains.length < this.limit) {
-            const remainingItems = this.limit - allDomains.length
+        while (allDocs.length < this.limit) {
+            const remainingItems = this.limit - allDocs.length
             const currentPageSize = Math.min(pageSize, remainingItems)
 
             console.info(`[AAIDomains] Fetching page ${currentPage}, size ${currentPageSize}`)
@@ -513,23 +490,23 @@ class AAIDomainsLoader extends BaseDocumentLoader {
                         break
                     }
 
-                    // Transform domain_tags array to flat tags array
-                    const domainsWithTags = data.map((domain: any) => ({
-                        ...domain,
-                        tags: domain.domain_tags?.map((dt: any) => dt.tags).filter(Boolean) || []
-                    }))
+                    // Transform domain_tags array to flat tags array (in-place mutation)
+                    for (const domain of data as any[]) {
+                        domain.tags = domain.domain_tags?.map((dt: any) => dt.tags).filter(Boolean) || []
+                    }
 
                     // Apply tag filtering if specified
-                    let filteredDomains = domainsWithTags
+                    let filteredDomains: any[] = data as any[]
                     if (this.includeTags.length > 0 || this.excludeTags.length > 0) {
-                        filteredDomains = this.filterByTags(domainsWithTags)
+                        filteredDomains = this.filterByTags(data as any[])
                     }
 
-                    allDomains.push(...filteredDomains)
+                    const pageDocs = filteredDomains.map((d: any) => this.createDocumentFromDomain(d))
+                    allDocs.push(...pageDocs)
                     currentPage++
 
                     // Stop if we've fetched enough
-                    if (allDomains.length >= this.limit || data.length < currentPageSize) {
+                    if (allDocs.length >= this.limit || data.length < currentPageSize) {
                         shouldStopPagination = true
                         break
                     }
@@ -563,14 +540,14 @@ class AAIDomainsLoader extends BaseDocumentLoader {
             await new Promise((resolve) => setTimeout(resolve, 200))
         }
 
-        console.info(`[AAIDomains] Load complete. Total domains: ${allDomains.length}`)
+        console.info(`[AAIDomains] Load complete. Total documents: ${allDocs.length}`)
 
         // Truncate to exact limit
-        if (allDomains.length > this.limit) {
-            allDomains = allDomains.slice(0, this.limit)
+        if (allDocs.length > this.limit) {
+            allDocs = allDocs.slice(0, this.limit)
         }
 
-        return allDomains.map((domain) => this.createDocumentFromDomain(domain))
+        return allDocs
     }
 
     private filterByTags(domains: any[]): any[] {

From c48057327e4082362f4e8cd139ea0108d8394608 Mon Sep 17 00:00:00 2001
From: Diego Costa <diecoscai@gmail.com>
Date: Thu, 12 Mar 2026 15:39:15 -0300
Subject: [PATCH 02/14] fix(documentstore): batch pipeline with OOM prevention
 and result accumulation

---
 .../src/services/documentstore/index.ts       | 135 ++++++++++++------
 1 file changed, 91 insertions(+), 44 deletions(-)

diff --git a/packages/server/src/services/documentstore/index.ts b/packages/server/src/services/documentstore/index.ts
index 02b451b9a80..11167400138 100644
--- a/packages/server/src/services/documentstore/index.ts
+++ b/packages/server/src/services/documentstore/index.ts
@@ -1237,29 +1237,34 @@ const _saveChunksToStorage = async (
                 }
                 return acc
             }, 0)
-            await Promise.all(
-                response.chunks.map(async (chunk: IDocument, index: number) => {
-                    try {
-                        const docChunk: DocumentStoreFileChunk = {
-                            docId: newLoaderId,
-                            storeId: data.storeId || '',
-                            id: uuidv4(),
-                            chunkNo: index + 1,
-                            pageContent: sanitizeChunkContent(chunk.pageContent),
-                            metadata: JSON.stringify(chunk.metadata),
-                            userId: data.userId,
-                            organizationId: data.organizationId
+            const SAVE_BATCH_SIZE = 500
+            for (let i = 0; i < response.chunks.length; i += SAVE_BATCH_SIZE) {
+                const batch = response.chunks.slice(i, i + SAVE_BATCH_SIZE)
+                await Promise.all(
+                    batch.map(async (chunk: IDocument, localIndex: number) => {
+                        try {
+                            const globalIndex = i + localIndex
+                            const docChunk: DocumentStoreFileChunk = {
+                                docId: newLoaderId,
+                                storeId: data.storeId || '',
+                                id: uuidv4(),
+                                chunkNo: globalIndex + 1,
+                                pageContent: sanitizeChunkContent(chunk.pageContent),
+                                metadata: JSON.stringify(chunk.metadata),
+                                userId: data.userId,
+                                organizationId: data.organizationId
+                            }
+                            const dChunk = appDataSource.getRepository(DocumentStoreFileChunk).create(docChunk)
+                            await appDataSource.getRepository(DocumentStoreFileChunk).save(dChunk)
+                        } catch (chunkError) {
+                            throw new InternalFlowiseError(
+                                StatusCodes.INTERNAL_SERVER_ERROR,
+                                `Error: documentStoreServices._saveChunksToStorage - ${getErrorMessage(chunkError)}`
+                            )
                         }
-                        const dChunk = appDataSource.getRepository(DocumentStoreFileChunk).create(docChunk)
-                        await appDataSource.getRepository(DocumentStoreFileChunk).save(dChunk)
-                    } catch (chunkError) {
-                        throw new InternalFlowiseError(
-                            StatusCodes.INTERNAL_SERVER_ERROR,
-                            `Error: documentStoreServices._saveChunksToStorage - ${getErrorMessage(chunkError)}`
-                        )
-                    }
-                })
-            )
+                    })
+                )
+            }
             // update the loader with the new metrics
             loader.totalChunks = response.totalChunks
             loader.totalChars = totalChars
@@ -1570,20 +1575,60 @@ const _insertIntoVectorStoreWorkerThread = async (
         if (data.docId) {
             filterOptions['docId'] = data.docId
         }
-        const chunks = await appDataSource.getRepository(DocumentStoreFileChunk).find({
-            where: filterOptions
-        })
-        const docs: Document[] = chunks.map((chunk: DocumentStoreFileChunk) => {
-            return new Document({
-                pageContent: chunk.pageContent,
-                metadata: JSON.parse(chunk.metadata)
+        const UPSERT_BATCH_SIZE = 500
+        const isFullCleanup = recordManagerObj && data.recordManagerConfig && JSON.parse(data.recordManagerConfig)?.cleanup === 'full'
+
+        let indexResult: ICommonObject | undefined
+        if (isFullCleanup) {
+            const chunks = await appDataSource.getRepository(DocumentStoreFileChunk).find({
+                where: filterOptions
             })
-        })
-        vStoreNodeData.inputs.document = docs
+            const docs: Document[] = chunks.map((chunk: DocumentStoreFileChunk) => {
+                return new Document({
+                    pageContent: chunk.pageContent,
+                    metadata: JSON.parse(chunk.metadata)
+                })
+            })
+            vStoreNodeData.inputs.document = docs
 
-        // Get Vector Store Instance
-        const vectorStoreObj = await _createVectorStoreObject(componentNodes, data, vStoreNodeData, upsertHistory)
-        const indexResult = await vectorStoreObj.vectorStoreMethods.upsert(vStoreNodeData, options)
+            const vectorStoreObj = await _createVectorStoreObject(componentNodes, data, vStoreNodeData, upsertHistory)
+            indexResult = await vectorStoreObj.vectorStoreMethods.upsert(vStoreNodeData, options)
+        } else {
+            const totalCount = await appDataSource.getRepository(DocumentStoreFileChunk).count({
+                where: filterOptions
+            })
+            const vectorStoreObj = await _createVectorStoreObject(componentNodes, data, vStoreNodeData, upsertHistory)
+            for (let skip = 0; skip < totalCount; skip += UPSERT_BATCH_SIZE) {
+                const chunks = await appDataSource.getRepository(DocumentStoreFileChunk).find({
+                    where: filterOptions,
+                    skip,
+                    take: UPSERT_BATCH_SIZE,
+                    order: { chunkNo: 'ASC' }
+                })
+                const docs: Document[] = chunks.map((chunk: DocumentStoreFileChunk) => {
+                    return new Document({
+                        pageContent: chunk.pageContent,
+                        metadata: JSON.parse(chunk.metadata)
+                    })
+                })
+                vStoreNodeData.inputs.document = docs
+
+                const batchResult = await vectorStoreObj.vectorStoreMethods.upsert(vStoreNodeData, options)
+                if (!batchResult) continue
+
+                if (!indexResult) {
+                    indexResult = { ...batchResult, addedDocs: batchResult.addedDocs ?? [] }
+                    continue
+                }
+
+                indexResult.numAdded = (indexResult.numAdded ?? 0) + (batchResult.numAdded ?? 0)
+                indexResult.numDeleted = (indexResult.numDeleted ?? 0) + (batchResult.numDeleted ?? 0)
+                indexResult.numUpdated = (indexResult.numUpdated ?? 0) + (batchResult.numUpdated ?? 0)
+                indexResult.numSkipped = (indexResult.numSkipped ?? 0) + (batchResult.numSkipped ?? 0)
+                indexResult.totalKeys = batchResult.totalKeys ?? indexResult.totalKeys
+                indexResult.addedDocs = batchResult.addedDocs ?? []
+            }
+        }
 
         // Save to DB
         if (indexResult) {
@@ -1597,16 +1642,18 @@ const _insertIntoVectorStoreWorkerThread = async (
             await appDataSource.getRepository(UpsertHistory).save(upsertHistoryItem)
         }
 
-        await telemetry.sendTelemetry(
-            'vector_upserted',
-            {
-                version: await getAppVersion(),
-                chatlowId: chatflowid,
-                type: ChatType.INTERNAL,
-                flowGraph: omit(indexResult['result'], ['totalKeys', 'addedDocs'])
-            },
-            orgId
-        )
+        if (indexResult) {
+            await telemetry.sendTelemetry(
+                'vector_upserted',
+                {
+                    version: await getAppVersion(),
+                    chatlowId: chatflowid,
+                    type: ChatType.INTERNAL,
+                    flowGraph: omit(indexResult['result'], ['totalKeys', 'addedDocs'])
+                },
+                orgId
+            )
+        }
 
         entity.status = DocumentStoreStatus.UPSERTED
         await appDataSource.getRepository(DocumentStore).save(entity)

From 18c59f47bd7ebf90cefde41e2bfa6965eafbe324 Mon Sep 17 00:00:00 2001
From: Diego Costa <diecoscai@gmail.com>
Date: Thu, 12 Mar 2026 16:14:47 -0300
Subject: [PATCH 03/14] fix(documentstore): guard cleanup parsing and stabilize
 batch paging

---
 packages/server/src/services/documentstore/index.ts | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/services/documentstore/index.ts b/packages/server/src/services/documentstore/index.ts
index 11167400138..3f15b2f11b1 100644
--- a/packages/server/src/services/documentstore/index.ts
+++ b/packages/server/src/services/documentstore/index.ts
@@ -1576,7 +1576,9 @@ const _insertIntoVectorStoreWorkerThread = async (
             filterOptions['docId'] = data.docId
         }
         const UPSERT_BATCH_SIZE = 500
-        const isFullCleanup = recordManagerObj && data.recordManagerConfig && JSON.parse(data.recordManagerConfig)?.cleanup === 'full'
+        const recordManagerConfig =
+            typeof data.recordManagerConfig === 'string' ? JSON.parse(data.recordManagerConfig) : data.recordManagerConfig
+        const isFullCleanup = recordManagerObj && recordManagerConfig?.cleanup === 'full'
 
         let indexResult: ICommonObject | undefined
         if (isFullCleanup) {
@@ -1603,7 +1605,7 @@ const _insertIntoVectorStoreWorkerThread = async (
                     where: filterOptions,
                     skip,
                     take: UPSERT_BATCH_SIZE,
-                    order: { chunkNo: 'ASC' }
+                    order: { chunkNo: 'ASC', id: 'ASC' }
                 })
                 const docs: Document[] = chunks.map((chunk: DocumentStoreFileChunk) => {
                     return new Document({

From e68e6486910dd0f52090174699fcc22b1f856ea5 Mon Sep 17 00:00:00 2001
From: Diego Costa <diecoscai@gmail.com>
Date: Thu, 12 Mar 2026 16:38:24 -0300
Subject: [PATCH 04/14] perf(documentstore): fix OOM in upsert pipeline with
 keyset pagination

Full-cleanup branch: replace unbounded find() with keyset-paginated while loop
accumulating all chunks into a single docs[] array, then calling upsert() once.
Preserves the single indexStartDt boundary required by record manager full-cleanup
semantics - splitting upsert() calls causes later calls to delete earlier keys.

Non-full path: replace count()+skip offset pagination with MoreThan(lastId) keyset
cursor on UUID PK, eliminating race condition where concurrent writes shift pages.
---
 .../src/services/documentstore/index.ts       | 69 +++++++++++--------
 1 file changed, 42 insertions(+), 27 deletions(-)

diff --git a/packages/server/src/services/documentstore/index.ts b/packages/server/src/services/documentstore/index.ts
index 3f15b2f11b1..a5267574781 100644
--- a/packages/server/src/services/documentstore/index.ts
+++ b/packages/server/src/services/documentstore/index.ts
@@ -15,7 +15,7 @@ import {
 import { StatusCodes } from 'http-status-codes'
 import { cloneDeep, omit } from 'lodash'
 import * as path from 'path'
-import { DataSource, In } from 'typeorm'
+import { DataSource, In, MoreThan } from 'typeorm'
 import { v4 as uuidv4 } from 'uuid'
 import {
     addLoaderSource,
@@ -1582,31 +1582,44 @@ const _insertIntoVectorStoreWorkerThread = async (
 
         let indexResult: ICommonObject | undefined
         if (isFullCleanup) {
-            const chunks = await appDataSource.getRepository(DocumentStoreFileChunk).find({
-                where: filterOptions
-            })
-            const docs: Document[] = chunks.map((chunk: DocumentStoreFileChunk) => {
-                return new Document({
-                    pageContent: chunk.pageContent,
-                    metadata: JSON.parse(chunk.metadata)
+            const docs: Document[] = []
+            let lastId = ''
+            while (true) {
+                const batchFilter: ICommonObject = lastId
+                    ? { ...filterOptions, id: MoreThan(lastId) }
+                    : { ...filterOptions }
+                const chunks = await appDataSource.getRepository(DocumentStoreFileChunk).find({
+                    where: batchFilter,
+                    take: UPSERT_BATCH_SIZE,
+                    order: { id: 'ASC' }
                 })
-            })
+                if (chunks.length === 0) break
+                for (const chunk of chunks) {
+                    docs.push(new Document({
+                        pageContent: chunk.pageContent,
+                        metadata: JSON.parse(chunk.metadata)
+                    }))
+                }
+                if (chunks.length < UPSERT_BATCH_SIZE) break
+                lastId = chunks[chunks.length - 1].id
+            }
             vStoreNodeData.inputs.document = docs
 
             const vectorStoreObj = await _createVectorStoreObject(componentNodes, data, vStoreNodeData, upsertHistory)
             indexResult = await vectorStoreObj.vectorStoreMethods.upsert(vStoreNodeData, options)
         } else {
-            const totalCount = await appDataSource.getRepository(DocumentStoreFileChunk).count({
-                where: filterOptions
-            })
             const vectorStoreObj = await _createVectorStoreObject(componentNodes, data, vStoreNodeData, upsertHistory)
-            for (let skip = 0; skip < totalCount; skip += UPSERT_BATCH_SIZE) {
+            let lastId = ''
+            while (true) {
+                const batchFilter: ICommonObject = lastId
+                    ? { ...filterOptions, id: MoreThan(lastId) }
+                    : { ...filterOptions }
                 const chunks = await appDataSource.getRepository(DocumentStoreFileChunk).find({
-                    where: filterOptions,
-                    skip,
+                    where: batchFilter,
                     take: UPSERT_BATCH_SIZE,
-                    order: { chunkNo: 'ASC', id: 'ASC' }
+                    order: { id: 'ASC' }
                 })
+                if (chunks.length === 0) break
                 const docs: Document[] = chunks.map((chunk: DocumentStoreFileChunk) => {
                     return new Document({
                         pageContent: chunk.pageContent,
@@ -1616,19 +1629,21 @@ const _insertIntoVectorStoreWorkerThread = async (
                 vStoreNodeData.inputs.document = docs
 
                 const batchResult = await vectorStoreObj.vectorStoreMethods.upsert(vStoreNodeData, options)
-                if (!batchResult) continue
-
-                if (!indexResult) {
-                    indexResult = { ...batchResult, addedDocs: batchResult.addedDocs ?? [] }
-                    continue
+                if (batchResult) {
+                    if (!indexResult) {
+                        indexResult = { ...batchResult, addedDocs: batchResult.addedDocs ?? [] }
+                    } else {
+                        indexResult.numAdded = (indexResult.numAdded ?? 0) + (batchResult.numAdded ?? 0)
+                        indexResult.numDeleted = (indexResult.numDeleted ?? 0) + (batchResult.numDeleted ?? 0)
+                        indexResult.numUpdated = (indexResult.numUpdated ?? 0) + (batchResult.numUpdated ?? 0)
+                        indexResult.numSkipped = (indexResult.numSkipped ?? 0) + (batchResult.numSkipped ?? 0)
+                        indexResult.totalKeys = batchResult.totalKeys ?? indexResult.totalKeys
+                        indexResult.addedDocs = batchResult.addedDocs ?? []
+                    }
                 }
 
-                indexResult.numAdded = (indexResult.numAdded ?? 0) + (batchResult.numAdded ?? 0)
-                indexResult.numDeleted = (indexResult.numDeleted ?? 0) + (batchResult.numDeleted ?? 0)
-                indexResult.numUpdated = (indexResult.numUpdated ?? 0) + (batchResult.numUpdated ?? 0)
-                indexResult.numSkipped = (indexResult.numSkipped ?? 0) + (batchResult.numSkipped ?? 0)
-                indexResult.totalKeys = batchResult.totalKeys ?? indexResult.totalKeys
-                indexResult.addedDocs = batchResult.addedDocs ?? []
+                if (chunks.length < UPSERT_BATCH_SIZE) break
+                lastId = chunks[chunks.length - 1].id
             }
         }
 

From 805b2eb3ef67761a980d90ee1494d3c93da5dddb Mon Sep 17 00:00:00 2001
From: Diego Costa <diecoscai@gmail.com>
Date: Thu, 12 Mar 2026 17:23:11 -0300
Subject: [PATCH 05/14] fix(documentstore): close upsert correctness, tenancy
 gaps, and code quality improvements

---
 .../src/services/documentstore/index.ts       | 28 ++++++++++++++-----
 1 file changed, 21 insertions(+), 7 deletions(-)

diff --git a/packages/server/src/services/documentstore/index.ts b/packages/server/src/services/documentstore/index.ts
index a5267574781..c78557fc145 100644
--- a/packages/server/src/services/documentstore/index.ts
+++ b/packages/server/src/services/documentstore/index.ts
@@ -57,6 +57,10 @@ import { checkStorage, updateStorageUsage } from '../../utils/quotaUsage'
 import { Telemetry } from '../../utils/telemetry'
 import nodesService from '../nodes'
 
+// Batch sizes for chunk DB operations and vector store upsert
+const SAVE_BATCH_SIZE = 500
+const UPSERT_BATCH_SIZE = 500
+
 const createDocumentStore = async (newDocumentStore: DocumentStore, orgId: string) => {
     try {
         const appServer = getRunningExpressApp()
@@ -1237,7 +1241,6 @@ const _saveChunksToStorage = async (
                 }
                 return acc
             }, 0)
-            const SAVE_BATCH_SIZE = 500
             for (let i = 0; i < response.chunks.length; i += SAVE_BATCH_SIZE) {
                 const batch = response.chunks.slice(i, i + SAVE_BATCH_SIZE)
                 await Promise.all(
@@ -1570,19 +1573,27 @@ const _insertIntoVectorStoreWorkerThread = async (
 
         // Prepare docs for upserting
         const filterOptions: ICommonObject = {
-            storeId: data.storeId
+            storeId: data.storeId,
+            organizationId: entity.organizationId
         }
         if (data.docId) {
             filterOptions['docId'] = data.docId
         }
-        const UPSERT_BATCH_SIZE = 500
         const recordManagerConfig =
             typeof data.recordManagerConfig === 'string' ? JSON.parse(data.recordManagerConfig) : data.recordManagerConfig
         const isFullCleanup = recordManagerObj && recordManagerConfig?.cleanup === 'full'
 
         let indexResult: ICommonObject | undefined
+        // IMPORTANT: Full-cleanup mode must call upsert() exactly once with ALL docs accumulated.
+        // The record manager captures indexStartDt = getTime() at the start of each upsert() call
+        // and deletes all keys with timestamp < indexStartDt after the call completes.
+        // Splitting into multiple upsert() calls would cause each call to delete the previous
+        // call's keys, resulting in data loss. Paginate the DB reads, but keep a single upsert().
         if (isFullCleanup) {
             const docs: Document[] = []
+            // Paginate chunks using PK keyset cursor (id ASC). The chunk dataset is treated as
+            // stable during this worker iteration — no concurrent chunk writes are expected.
+            // If concurrent writes are introduced in future, migrate to a monotonic sequence cursor.
             let lastId = ''
             while (true) {
                 const batchFilter: ICommonObject = lastId
@@ -1597,7 +1608,7 @@ const _insertIntoVectorStoreWorkerThread = async (
                 for (const chunk of chunks) {
                     docs.push(new Document({
                         pageContent: chunk.pageContent,
-                        metadata: JSON.parse(chunk.metadata)
+                        metadata: (() => { try { return chunk.metadata ? JSON.parse(chunk.metadata) : {} } catch { return {} } })()
                     }))
                 }
                 if (chunks.length < UPSERT_BATCH_SIZE) break
@@ -1609,6 +1620,9 @@ const _insertIntoVectorStoreWorkerThread = async (
             indexResult = await vectorStoreObj.vectorStoreMethods.upsert(vStoreNodeData, options)
         } else {
             const vectorStoreObj = await _createVectorStoreObject(componentNodes, data, vStoreNodeData, upsertHistory)
+            // Paginate chunks using PK keyset cursor (id ASC). The chunk dataset is treated as
+            // stable during this worker iteration — no concurrent chunk writes are expected.
+            // If concurrent writes are introduced in future, migrate to a monotonic sequence cursor.
             let lastId = ''
             while (true) {
                 const batchFilter: ICommonObject = lastId
@@ -1623,7 +1637,7 @@ const _insertIntoVectorStoreWorkerThread = async (
                 const docs: Document[] = chunks.map((chunk: DocumentStoreFileChunk) => {
                     return new Document({
                         pageContent: chunk.pageContent,
-                        metadata: JSON.parse(chunk.metadata)
+                        metadata: (() => { try { return chunk.metadata ? JSON.parse(chunk.metadata) : {} } catch { return {} } })()
                     })
                 })
                 vStoreNodeData.inputs.document = docs
@@ -1638,7 +1652,7 @@ const _insertIntoVectorStoreWorkerThread = async (
                         indexResult.numUpdated = (indexResult.numUpdated ?? 0) + (batchResult.numUpdated ?? 0)
                         indexResult.numSkipped = (indexResult.numSkipped ?? 0) + (batchResult.numSkipped ?? 0)
                         indexResult.totalKeys = batchResult.totalKeys ?? indexResult.totalKeys
-                        indexResult.addedDocs = batchResult.addedDocs ?? []
+                        indexResult.addedDocs = [...(indexResult.addedDocs ?? []), ...(batchResult.addedDocs ?? [])]
                     }
                 }
 
@@ -1664,7 +1678,7 @@ const _insertIntoVectorStoreWorkerThread = async (
                 'vector_upserted',
                 {
                     version: await getAppVersion(),
-                    chatlowId: chatflowid,
+                    chatflowId: chatflowid,
                     type: ChatType.INTERNAL,
                     flowGraph: omit(indexResult['result'], ['totalKeys', 'addedDocs'])
                 },

From 61a95f769700fe76ecce72810082ca2b44ac2981 Mon Sep 17 00:00:00 2001
From: Diego Costa <diecoscai@gmail.com>
Date: Thu, 12 Mar 2026 17:23:16 -0300
Subject: [PATCH 06/14] chore(components): avoid in-place domain tag mutation

---
 .../nodes/documentloaders/AAIDomains/AAIDomains.ts  | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/packages/components/nodes/documentloaders/AAIDomains/AAIDomains.ts b/packages/components/nodes/documentloaders/AAIDomains/AAIDomains.ts
index 013b86c89ce..3df934eb6cc 100644
--- a/packages/components/nodes/documentloaders/AAIDomains/AAIDomains.ts
+++ b/packages/components/nodes/documentloaders/AAIDomains/AAIDomains.ts
@@ -490,15 +490,16 @@ class AAIDomainsLoader extends BaseDocumentLoader {
                         break
                     }
 
-                    // Transform domain_tags array to flat tags array (in-place mutation)
-                    for (const domain of data as any[]) {
-                        domain.tags = domain.domain_tags?.map((dt: any) => dt.tags).filter(Boolean) || []
-                    }
+                    // Transform domain_tags array to flat tags array (non-mutating)
+                    const domainsWithTags = (data as any[]).map((domain: any) => ({
+                        ...domain,
+                        tags: domain.domain_tags?.map((dt: any) => dt.tags).filter(Boolean) || []
+                    }))
 
                     // Apply tag filtering if specified
-                    let filteredDomains: any[] = data as any[]
+                    let filteredDomains: any[] = domainsWithTags
                     if (this.includeTags.length > 0 || this.excludeTags.length > 0) {
-                        filteredDomains = this.filterByTags(data as any[])
+                        filteredDomains = this.filterByTags(domainsWithTags)
                     }
 
                     const pageDocs = filteredDomains.map((d: any) => this.createDocumentFromDomain(d))

From 2f8a442bdfe49b91d0f7a2e3f7cb3d68242ecfe9 Mon Sep 17 00:00:00 2001
From: Diego Costa <diecoscai@gmail.com>
Date: Fri, 13 Mar 2026 11:09:59 -0300
Subject: [PATCH 07/14] test(documentstore): add save-path failure matrix
 coverage

---
 .../test/services/documentstore/index.test.ts | 444 ++++++++++++++++++
 1 file changed, 444 insertions(+)
 create mode 100644 packages/server/test/services/documentstore/index.test.ts

diff --git a/packages/server/test/services/documentstore/index.test.ts b/packages/server/test/services/documentstore/index.test.ts
new file mode 100644
index 00000000000..5e5b773849e
--- /dev/null
+++ b/packages/server/test/services/documentstore/index.test.ts
@@ -0,0 +1,444 @@
+// RED tests: _saveChunksToStorage durability contract (must FAIL before implementation)
+
+let mockReturnChunks: any[] = []
+
+jest.mock('/virtual/test/mock-loader', () => ({
+    nodeClass: class MockLoader {
+        async init() {
+            return mockReturnChunks
+        }
+    }
+}), { virtual: true })
+
+const mockChunkSave = jest.fn()
+const mockChunkCreate = jest.fn((data: any) => data)
+const mockChunkDelete = jest.fn()
+const mockChunkCount = jest.fn()
+const mockChunkFind = jest.fn()
+const mockDocStoreSave = jest.fn()
+const mockDocStoreFindOneBy = jest.fn()
+
+// jest.mock calls MUST precede imports (jest hoisting requirement)
+jest.mock('flowise-components', () => ({
+    addArrayFilesToStorage: jest.fn(),
+    addSingleFileToStorage: jest.fn(),
+    getFileFromStorage: jest.fn(),
+    getFileFromUpload: jest.fn(),
+    ICommonObject: {},
+    IDocument: {},
+    mapExtToInputField: jest.fn(),
+    mapMimeTypeToInputField: jest.fn(),
+    removeFilesFromStorage: jest.fn(),
+    removeSpecificFileFromStorage: jest.fn().mockResolvedValue({ totalSize: 0 }),
+    removeSpecificFileFromUpload: jest.fn()
+}))
+
+jest.mock('@langchain/core/documents', () => ({
+    Document: class Document {}
+}))
+
+jest.mock('../../../src/database/entities/DocumentStore', () => {
+    class DocumentStoreMock {}
+    return { DocumentStore: DocumentStoreMock }
+})
+
+jest.mock('../../../src/database/entities/DocumentStoreFileChunk', () => {
+    class DocumentStoreFileChunkMock {}
+    return { DocumentStoreFileChunk: DocumentStoreFileChunkMock }
+})
+
+jest.mock('../../../src/database/entities/ChatFlow', () => ({
+    ChatFlow: class ChatFlow {}
+}))
+
+jest.mock('../../../src/database/entities/UpsertHistory', () => ({
+    UpsertHistory: class UpsertHistory {}
+}))
+
+jest.mock('../../../src/enterprise/database/entities/workspace.entity', () => ({
+    Workspace: class Workspace {}
+}))
+
+jest.mock('../../../src/enterprise/utils/ControllerServiceUtils', () => ({
+    getWorkspaceSearchOptions: jest.fn()
+}))
+
+jest.mock('../../../src/errors/internalFlowiseError', () => ({
+    InternalFlowiseError: class InternalFlowiseError extends Error {
+        statusCode: number
+        constructor(statusCode: number, message: string) {
+            super(message)
+            this.statusCode = statusCode
+        }
+    }
+}))
+
+jest.mock('../../../src/errors/utils', () => ({
+    getErrorMessage: (e: any) => e?.message || String(e)
+}))
+
+jest.mock('../../../src/Interface', () => ({
+    addLoaderSource: jest.fn().mockReturnValue('mock-source'),
+    ChatType: { CHATFLOW: 'CHATFLOW' },
+    DocumentStoreDTO: jest.fn(),
+    DocumentStoreStatus: {
+        EMPTY_SYNC: 'EMPTY',
+        SYNC: 'SYNC',
+        SYNCING: 'SYNCING',
+        STALE: 'STALE',
+        NEW: 'NEW',
+        UPSERTING: 'UPSERTING',
+        UPSERTED: 'UPSERTED'
+    },
+    MODE: { QUEUE: 'queue' }
+}))
+
+jest.mock('../../../src/UsageCacheManager', () => ({
+    UsageCacheManager: class UsageCacheManager {}
+}))
+
+jest.mock('../../../src/utils/getRunningExpressApp', () => ({
+    getRunningExpressApp: jest.fn()
+}))
+
+jest.mock('../../../src/utils/logger', () => ({
+    default: {
+        info: jest.fn(),
+        error: jest.fn(),
+        debug: jest.fn(),
+        warn: jest.fn()
+    }
+}))
+
+jest.mock('../../../src/utils', () => ({
+    databaseEntities: {},
+    getAppVersion: jest.fn(),
+    saveUpsertFlowData: jest.fn(),
+    constructGraphs: jest.fn(),
+    getEndingNodes: jest.fn(),
+    getTelemetryFlowObj: jest.fn(),
+    isFlowValidForStream: jest.fn()
+}))
+
+jest.mock('../../../src/utils/constants', () => ({
+    DOCUMENT_STORE_BASE_FOLDER: 'document-store',
+    INPUT_PARAMS_TYPE: {},
+    OMIT_QUEUE_JOB_DATA: []
+}))
+
+jest.mock('../../../src/utils/quotaUsage', () => ({
+    checkStorage: jest.fn(),
+    updateStorageUsage: jest.fn()
+}))
+
+jest.mock('../../../src/utils/prompt', () => ({
+    DOCUMENTSTORE_TOOL_DESCRIPTION_PROMPT_GENERATOR: ''
+}))
+
+jest.mock('../../../src/utils/telemetry', () => ({
+    Telemetry: class Telemetry {}
+}))
+
+jest.mock('../../../src/services/nodes', () => ({
+    default: {}
+}))
+
+let uuidCounter = 0
+jest.mock('uuid', () => ({
+    v4: jest.fn(() => `mock-uuid-${uuidCounter++}`)
+}))
+
+import { processLoader } from '../../../src/services/documentstore/index'
+import { DocumentStore } from '../../../src/database/entities/DocumentStore'
+import { DocumentStoreFileChunk } from '../../../src/database/entities/DocumentStoreFileChunk'
+
+const LOADER_ID = 'loader-1'
+const STORE_ID = 'store-1'
+
+const makeChunks = (n: number) =>
+    Array.from({ length: n }, (_, i) => ({
+        pageContent: `chunk-content-${i}`,
+        metadata: { source: 'test', index: i }
+    }))
+
+const makeEntity = () => ({
+    id: STORE_ID,
+    name: 'Test Store',
+    description: 'test',
+    loaders: JSON.stringify([{
+        id: LOADER_ID,
+        loaderId: 'mockLoader',
+        loaderName: 'Mock Loader',
+        loaderConfig: { text: 'hello' },
+        splitterId: '',
+        splitterConfig: {},
+        credential: '',
+        status: 'SYNCING',
+        totalChunks: 0,
+        totalChars: 0,
+        files: [],
+        source: ''
+    }]),
+    whereUsed: '[]',
+    status: 'SYNCING',
+    vectorStoreConfig: null,
+    embeddingConfig: null,
+    recordManagerConfig: null,
+    userId: 'user-1',
+    organizationId: 'org-1',
+    workspaceId: 'ws-1',
+    createdDate: new Date(),
+    updatedDate: new Date()
+})
+
+const makeData = () => ({
+    id: LOADER_ID,
+    storeId: STORE_ID,
+    loaderId: 'mockLoader',
+    loaderName: 'Mock Loader',
+    loaderConfig: { text: 'hello' },
+    splitterId: '',
+    splitterConfig: {},
+    credential: '',
+    previewChunkCount: -1,
+    userId: 'user-1',
+    organizationId: 'org-1',
+    user: { id: 'user-1', organizationId: 'org-1' }
+})
+
+const componentNodes: any = {
+    mockLoader: { filePath: '/virtual/test/mock-loader' }
+}
+
+const buildMockAppDataSource = () => {
+    const docStoreRepo = {
+        save: mockDocStoreSave,
+        findOneBy: mockDocStoreFindOneBy,
+        create: jest.fn((d: any) => d),
+        find: jest.fn().mockResolvedValue([]),
+        count: jest.fn().mockResolvedValue(0),
+        delete: jest.fn()
+    }
+    const chunkRepo = {
+        save: mockChunkSave,
+        create: mockChunkCreate,
+        delete: mockChunkDelete,
+        find: mockChunkFind,
+        count: mockChunkCount,
+        findOneBy: jest.fn()
+    }
+
+    return {
+        getRepository: jest.fn((entity: any) => {
+            if (entity === DocumentStore) return docStoreRepo
+            if (entity === DocumentStoreFileChunk) return chunkRepo
+            return {}
+        })
+    } as any
+}
+
+describe('_saveChunksToStorage durability contract', () => {
+    let appDataSource: any
+
+    beforeEach(() => {
+        jest.clearAllMocks()
+        uuidCounter = 0
+        appDataSource = buildMockAppDataSource()
+        mockChunkSave.mockResolvedValue(undefined)
+        mockChunkDelete.mockResolvedValue(undefined)
+        mockChunkCount.mockResolvedValue(0)
+        mockChunkFind.mockResolvedValue([])
+    })
+
+    describe('happy path', () => {
+        it('saves all chunks, sets loader status SYNC, counts match actual persisted', async () => {
+            const chunks = makeChunks(3)
+            mockReturnChunks = chunks
+
+            const entity = makeEntity()
+            mockDocStoreFindOneBy.mockResolvedValue(entity)
+            mockDocStoreSave.mockResolvedValue(entity)
+
+            await processLoader({
+                appDataSource,
+                componentNodes,
+                data: makeData(),
+                docLoaderId: LOADER_ID,
+                orgId: 'org-1',
+                workspaceId: 'ws-1',
+                subscriptionId: 'sub-1',
+                usageCacheManager: {} as any
+            } as any)
+
+            expect(mockChunkSave).toHaveBeenCalledTimes(3)
+
+            expect(mockDocStoreSave).toHaveBeenCalled()
+            const savedEntity = mockDocStoreSave.mock.calls[0][0]
+            const loaders = JSON.parse(savedEntity.loaders)
+            expect(loaders[0].status).toBe('SYNC')
+            expect(loaders[0].totalChunks).toBe(3)
+
+            const expectedChars = chunks.reduce((acc, c) => acc + c.pageContent.length, 0)
+            expect(loaders[0].totalChars).toBe(expectedChars)
+        })
+    })
+
+    describe('first-batch failure', () => {
+        it('does NOT delete old chunks when first save fails, sets loader status STALE', async () => {
+            const chunks = makeChunks(3)
+            mockReturnChunks = chunks
+
+            const entity = makeEntity()
+            mockDocStoreFindOneBy.mockResolvedValue(entity)
+            mockDocStoreSave.mockResolvedValue(entity)
+            mockChunkSave.mockRejectedValue(new Error('DB connection lost'))
+
+            let thrownError: any = null
+            try {
+                await processLoader({
+                    appDataSource,
+                    componentNodes,
+                    data: makeData(),
+                    docLoaderId: LOADER_ID,
+                    orgId: 'org-1',
+                    workspaceId: 'ws-1',
+                    subscriptionId: 'sub-1',
+                    usageCacheManager: {} as any
+                } as any)
+            } catch (e) {
+                thrownError = e
+            }
+
+            // CONTRACT: handle error internally, don't throw — BUG: current code throws → RED
+            expect(thrownError).toBeNull()
+
+            // CONTRACT: old chunks preserved (delete deferred) — BUG: L1247 deletes BEFORE save → RED
+            expect(mockChunkDelete).not.toHaveBeenCalled()
+
+            // CONTRACT: entity saved with STALE status
+            const savedEntity = mockDocStoreSave.mock.calls[0]?.[0]
+            if (savedEntity) {
+                const loaders = JSON.parse(savedEntity.loaders)
+                expect(loaders[0].status).toBe('STALE')
+            }
+        })
+    })
+
+    describe('mid-run failure (batch N>1)', () => {
+        it('preserves completed batches, sets status STALE, counts = actual persisted', async () => {
+            const chunks = makeChunks(501) // 2 batches (SAVE_BATCH_SIZE=500)
+            mockReturnChunks = chunks
+
+            const entity = makeEntity()
+            mockDocStoreFindOneBy.mockResolvedValue(entity)
+            mockDocStoreSave.mockResolvedValue(entity)
+
+            let saveCount = 0
+            mockChunkSave.mockImplementation(() => {
+                saveCount++
+                if (saveCount > 500) {
+                    return Promise.reject(new Error('DB connection lost mid-run'))
+                }
+                return Promise.resolve(undefined)
+            })
+
+            let thrownError: any = null
+            try {
+                await processLoader({
+                    appDataSource,
+                    componentNodes,
+                    data: makeData(),
+                    docLoaderId: LOADER_ID,
+                    orgId: 'org-1',
+                    workspaceId: 'ws-1',
+                    subscriptionId: 'sub-1',
+                    usageCacheManager: {} as any
+                } as any)
+            } catch (e) {
+                thrownError = e
+            }
+
+            // CONTRACT: handle partial failure internally — BUG: current code throws → RED
+            expect(thrownError).toBeNull()
+
+            expect(mockDocStoreSave).toHaveBeenCalled()
+            const savedEntity = mockDocStoreSave.mock.calls[0]?.[0]
+            if (savedEntity) {
+                const loaders = JSON.parse(savedEntity.loaders)
+                expect(loaders[0].status).toBe('STALE')
+                // BUG: L1284 sets totalChunks = response.totalChunks (501) not actual (500) → RED
+                expect(loaders[0].totalChunks).toBe(500)
+            }
+        })
+    })
+
+    describe('idempotent rerun', () => {
+        it('after partial failure, rerun reaches SYNC with correct counts, no duplication', async () => {
+            const chunks = makeChunks(3)
+            mockReturnChunks = chunks
+
+            const entity1 = makeEntity()
+            mockDocStoreFindOneBy.mockResolvedValue(entity1)
+            mockDocStoreSave.mockResolvedValue(entity1)
+
+            let firstRunSaveCount = 0
+            mockChunkSave.mockImplementation(() => {
+                firstRunSaveCount++
+                if (firstRunSaveCount === 2) {
+                    return Promise.reject(new Error('Transient DB error'))
+                }
+                return Promise.resolve(undefined)
+            })
+
+            try {
+                await processLoader({
+                    appDataSource,
+                    componentNodes,
+                    data: makeData(),
+                    docLoaderId: LOADER_ID,
+                    orgId: 'org-1',
+                    workspaceId: 'ws-1',
+                    subscriptionId: 'sub-1',
+                    usageCacheManager: {} as any
+                } as any)
+            } catch (_) {
+                // current code throws on failure — expected for now
+            }
+
+            jest.clearAllMocks()
+            uuidCounter = 0
+            appDataSource = buildMockAppDataSource()
+
+            const entity2 = makeEntity()
+            mockDocStoreFindOneBy.mockResolvedValue(entity2)
+            mockDocStoreSave.mockResolvedValue(entity2)
+            mockChunkSave.mockResolvedValue(undefined)
+            mockChunkDelete.mockResolvedValue(undefined)
+            mockChunkCount.mockResolvedValue(0)
+            mockChunkFind.mockResolvedValue([])
+
+            await processLoader({
+                appDataSource,
+                componentNodes,
+                data: makeData(),
+                docLoaderId: LOADER_ID,
+                orgId: 'org-1',
+                workspaceId: 'ws-1',
+                subscriptionId: 'sub-1',
+                usageCacheManager: {} as any
+            } as any)
+
+            expect(mockDocStoreSave).toHaveBeenCalled()
+            const savedEntity = mockDocStoreSave.mock.calls[0][0]
+            const loaders = JSON.parse(savedEntity.loaders)
+            expect(loaders[0].status).toBe('SYNC')
+            expect(loaders[0].totalChunks).toBe(3)
+
+            // CONTRACT: delete happens AFTER save (safe delete timing)
+            // BUG: L1247 deletes BEFORE L1256 save → RED
+            const deleteCallOrder = mockChunkDelete.mock.invocationCallOrder[0]
+            const firstSaveCallOrder = mockChunkSave.mock.invocationCallOrder[0]
+            expect(deleteCallOrder).toBeGreaterThan(firstSaveCallOrder)
+        })
+    })
+})

From fa47df0117b9a17c959fff50ca09f8d88f6334c9 Mon Sep 17 00:00:00 2001
From: Diego Costa <diecoscai@gmail.com>
Date: Fri, 13 Mar 2026 11:10:17 -0300
Subject: [PATCH 08/14] fix(documentstore): make loader chunk save path durable
 per batch

- Move delete-before-save to delete-after-save in both _saveChunksToStorage
  and syncAndRefreshChunks: old chunks are preserved until all new chunks
  are confirmed persisted
- Add per-batch error handling so partial failures set status STALE and
  preserve completed batches rather than throwing all-or-nothing
- Replace sequential one-by-one saves in syncAndRefreshChunks with
  SAVE_BATCH_SIZE-batched parallel saves matching the existing pattern
- Derive totalChunks/totalChars from actual persisted counts, not expected
  values from the chunk response
- Add durability contract comments to both save paths documenting the
  SYNCING -> SYNC/STALE state machine and safe-delete-timing mechanism
---
 .../src/services/documentstore/index.ts       | 139 +++++++++++-------
 1 file changed, 88 insertions(+), 51 deletions(-)

diff --git a/packages/server/src/services/documentstore/index.ts b/packages/server/src/services/documentstore/index.ts
index c78557fc145..8652e70c672 100644
--- a/packages/server/src/services/documentstore/index.ts
+++ b/packages/server/src/services/documentstore/index.ts
@@ -396,6 +396,12 @@ const getDocumentStoreFileChunks = async (
 }
 
 // Sync and refresh chunks for a specific loader or store
+// DURABILITY CONTRACT: syncAndRefreshChunks
+// Success path:   SYNCING -> SYNC, totalChunks/totalChars = actual persisted count
+// Partial failure (batch N>1 fails): SYNCING -> STALE, counts = actual persisted rows
+// First-batch failure: SYNCING -> STALE, old chunks untouched (delete deferred until full save)
+// Idempotent rerun: safe — delete of old chunks happens only after new set is fully persisted
+// Mechanism: safe delete timing — old chunks deleted AFTER all new chunks confirmed saved
 const syncAndRefreshChunks = async (storeId: string, fileId: string, userId: string, organizationId: string, workspaceId: string) => {
     try {
         const appServer = getRunningExpressApp()
@@ -436,40 +442,58 @@ const syncAndRefreshChunks = async (storeId: string, fileId: string, userId: str
         entity.loaders = JSON.stringify(loaders)
         await appServer.AppDataSource.getRepository(DocumentStore).save(entity)
 
-        // Delete existing chunks
-        await appServer.AppDataSource.getRepository(DocumentStoreFileChunk).delete({
-            docId: fileId,
-            userId,
-            organizationId
-        })
-
         // Get fresh documents from Google Drive
         const docs = await _splitIntoChunks(appServer.AppDataSource, componentNodes, data, userId, organizationId)
 
-        // Save new chunks
-        let totalChars = 0
-        for (let i = 0; i < docs.length; i++) {
-            const chunk = docs[i]
-            totalChars += chunk.pageContent.length
+        // Save new chunks first (safe delete timing — delete AFTER all saved)
+        let persistedChunks = 0
+        let persistedChars = 0
+        let saveFailed = false
 
-            const docChunk: DocumentStoreFileChunk = {
-                userId,
-                organizationId,
-                docId: fileId,
-                storeId: storeId,
-                id: uuidv4(),
-                chunkNo: i + 1,
-                pageContent: chunk.pageContent,
-                metadata: JSON.stringify(chunk.metadata)
+        for (let i = 0; i < docs.length; i += SAVE_BATCH_SIZE) {
+            const batch = docs.slice(i, i + SAVE_BATCH_SIZE)
+            try {
+                await Promise.all(
+                    batch.map(async (chunk: IDocument, localIndex: number) => {
+                        const globalIndex = i + localIndex
+                        const docChunk: DocumentStoreFileChunk = {
+                            userId,
+                            organizationId,
+                            docId: fileId,
+                            storeId: storeId,
+                            id: uuidv4(),
+                            chunkNo: globalIndex + 1,
+                            pageContent: chunk.pageContent,
+                            metadata: JSON.stringify(chunk.metadata)
+                        }
+                        const dChunk = appServer.AppDataSource.getRepository(DocumentStoreFileChunk).create(docChunk)
+                        await appServer.AppDataSource.getRepository(DocumentStoreFileChunk).save(dChunk)
+                    })
+                )
+                persistedChunks += batch.length
+                persistedChars += batch.reduce((acc: number, chunk: IDocument) => acc + (chunk.pageContent?.length ?? 0), 0)
+            } catch (batchError) {
+                saveFailed = true
+                break
             }
-
-            const dChunk = appServer.AppDataSource.getRepository(DocumentStoreFileChunk).create(docChunk)
-            await appServer.AppDataSource.getRepository(DocumentStoreFileChunk).save(dChunk)
         }
 
-        loader.totalChunks = docs.length
-        loader.totalChars = totalChars
-        loader.status = DocumentStoreStatus.SYNC
+        if (!saveFailed) {
+            // Delete old chunks only after all new chunks are confirmed saved
+            await appServer.AppDataSource.getRepository(DocumentStoreFileChunk).delete({
+                docId: fileId,
+                userId,
+                organizationId
+            })
+            loader.totalChunks = persistedChunks
+            loader.totalChars = persistedChars
+            loader.status = DocumentStoreStatus.SYNC
+        } else {
+            // Partial failure: record actual persisted count, mark STALE
+            loader.totalChunks = persistedChunks
+            loader.totalChars = persistedChars
+            loader.status = DocumentStoreStatus.STALE
+        }
 
         // Check if all loaders are synced
         const allSynced = loaders.every((ldr: IDocumentStoreLoader) => ldr.status === DocumentStoreStatus.SYNC)
@@ -1123,6 +1147,12 @@ const processLoaderMiddleware = async (
     }
 }
 
+// DURABILITY CONTRACT: _saveChunksToStorage
+// Success path:   SYNCING -> SYNC, totalChunks/totalChars = actual persisted count
+// Partial failure (batch N>1 fails): SYNCING -> STALE, counts = actual persisted rows
+// First-batch failure: SYNCING -> STALE, old chunks untouched (delete deferred until full save)
+// Idempotent rerun: safe — delete of old chunks happens only after new set is fully persisted
+// Mechanism: safe delete timing — old chunks deleted AFTER all new chunks confirmed saved
 const _saveChunksToStorage = async (
     appDataSource: DataSource,
     componentNodes: IComponentNodes,
@@ -1231,21 +1261,17 @@ const _saveChunksToStorage = async (
             existingLoaders.push(loader)
         }
 
-        //step 7: remove all previous chunks
-        await appDataSource.getRepository(DocumentStoreFileChunk).delete({ docId: newLoaderId })
         if (response.chunks) {
-            //step 8: now save the new chunks
-            const totalChars = response.chunks.reduce((acc, chunk) => {
-                if (chunk.pageContent) {
-                    return acc + chunk.pageContent.length
-                }
-                return acc
-            }, 0)
+            //step 7: save new chunks first (safe delete timing — delete AFTER all saved)
+            let persistedChunks = 0
+            let persistedChars = 0
+            let saveFailed = false
+
             for (let i = 0; i < response.chunks.length; i += SAVE_BATCH_SIZE) {
                 const batch = response.chunks.slice(i, i + SAVE_BATCH_SIZE)
-                await Promise.all(
-                    batch.map(async (chunk: IDocument, localIndex: number) => {
-                        try {
+                try {
+                    await Promise.all(
+                        batch.map(async (chunk: IDocument, localIndex: number) => {
                             const globalIndex = i + localIndex
                             const docChunk: DocumentStoreFileChunk = {
                                 docId: newLoaderId,
@@ -1259,20 +1285,31 @@ const _saveChunksToStorage = async (
                             }
                             const dChunk = appDataSource.getRepository(DocumentStoreFileChunk).create(docChunk)
                             await appDataSource.getRepository(DocumentStoreFileChunk).save(dChunk)
-                        } catch (chunkError) {
-                            throw new InternalFlowiseError(
-                                StatusCodes.INTERNAL_SERVER_ERROR,
-                                `Error: documentStoreServices._saveChunksToStorage - ${getErrorMessage(chunkError)}`
-                            )
-                        }
-                    })
-                )
+                        })
+                    )
+                    persistedChunks += batch.length
+                    persistedChars += batch.reduce((acc: number, chunk: IDocument) => acc + (chunk.pageContent?.length ?? 0), 0)
+                } catch (batchError) {
+                    saveFailed = true
+                    break
+                }
             }
-            // update the loader with the new metrics
-            loader.totalChunks = response.totalChunks
-            loader.totalChars = totalChars
+
+            if (!saveFailed) {
+                //step 8: delete old chunks only after all new chunks are confirmed saved
+                await appDataSource.getRepository(DocumentStoreFileChunk).delete({ docId: newLoaderId })
+                loader.totalChunks = persistedChunks
+                loader.totalChars = persistedChars
+                loader.status = 'SYNC'
+            } else {
+                // Partial failure: record actual persisted count, mark STALE
+                loader.totalChunks = persistedChunks
+                loader.totalChars = persistedChars
+                loader.status = DocumentStoreStatus.STALE
+            }
+        } else {
+            loader.status = 'SYNC'
         }
-        loader.status = 'SYNC'
         // have a flag and iterate over the loaders and update the entity status to SYNC
         const allSynced = existingLoaders.every((ldr: IDocumentStoreLoader) => ldr.status === 'SYNC')
         entity.status = allSynced ? DocumentStoreStatus.SYNC : DocumentStoreStatus.STALE

From dc0368eb91181946a009e518737c281bdbbb166e Mon Sep 17 00:00:00 2001
From: Diego Costa <diecoscai@gmail.com>
Date: Fri, 13 Mar 2026 16:41:10 -0300
Subject: [PATCH 09/14] fix(documentstore): port safe preview offset and loader
 limit cap

---
 .../documentloaders/AAIDomains/AAIDomains.ts    |  8 +++++++-
 packages/server/src/Interface.DocumentStore.ts  |  1 +
 .../server/src/services/documentstore/index.ts  | 17 ++++++++++++++---
 3 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/packages/components/nodes/documentloaders/AAIDomains/AAIDomains.ts b/packages/components/nodes/documentloaders/AAIDomains/AAIDomains.ts
index 3df934eb6cc..b0e228e8003 100644
--- a/packages/components/nodes/documentloaders/AAIDomains/AAIDomains.ts
+++ b/packages/components/nodes/documentloaders/AAIDomains/AAIDomains.ts
@@ -247,11 +247,17 @@ class AAIDomains_DocumentLoaders implements INode {
 
         // Convert 'all' to null for RPC function (per API spec)
         const isValidFilter = isValid === 'all' ? null : isValid
+        const requestedLimit = limit || 100
+        const envConfiguredLimit = Number(process.env.AAIDOMAINS_LIMIT ?? process.env.AAI_DOMAINS_MAX_LIMIT ?? 0)
+        const effectiveLimit =
+            Number.isFinite(envConfiguredLimit) && envConfiguredLimit > 0
+                ? Math.min(requestedLimit, Math.floor(envConfiguredLimit))
+                : requestedLimit
 
         const loaderOptions: AAIDomainsLoaderParams = {
             supabaseUrl,
             supabaseKey,
-            limit: limit || 100,
+            limit: effectiveLimit,
             searchTerm: searchTerm || null,
             includeTags: includeTags ? includeTags.split(',').map((t) => t.trim()) : [],
             includeTagsLogic,
diff --git a/packages/server/src/Interface.DocumentStore.ts b/packages/server/src/Interface.DocumentStore.ts
index b68e4b1efbc..113a3318f2e 100644
--- a/packages/server/src/Interface.DocumentStore.ts
+++ b/packages/server/src/Interface.DocumentStore.ts
@@ -79,6 +79,7 @@ export interface IDocumentStoreLoaderForPreview extends IDocumentStoreLoader {
     rehydrated?: boolean
     preview?: boolean
     previewChunkCount?: number
+    previewChunkOffset?: number
     user: IUser
 }
 
diff --git a/packages/server/src/services/documentstore/index.ts b/packages/server/src/services/documentstore/index.ts
index 8652e70c672..ddaa08fe838 100644
--- a/packages/server/src/services/documentstore/index.ts
+++ b/packages/server/src/services/documentstore/index.ts
@@ -946,14 +946,25 @@ export const previewChunks = async ({ appDataSource, componentNodes, data, orgId
         }
         let docs = await _splitIntoChunks(appDataSource, componentNodes, data, data.userId, data.organizationId)
         const totalChunks = docs.length
+        const previewChunkOffset = Math.max(0, data.previewChunkOffset || 0)
+
         // if -1, return all chunks
         if (data.previewChunkCount === -1) data.previewChunkCount = totalChunks
         // return all docs if the user ask for more than we have
         if (totalChunks <= (data.previewChunkCount || 0)) data.previewChunkCount = totalChunks
-        // return only the first n chunks
-        if (totalChunks > (data.previewChunkCount || 0)) docs = docs.slice(0, data.previewChunkCount)
+        const previewChunkCount = Math.max(0, data.previewChunkCount || 0)
+        if (previewChunkCount > 0) {
+            docs = docs.slice(previewChunkOffset, previewChunkOffset + previewChunkCount)
+        } else {
+            docs = []
+        }
 
-        return { chunks: docs, totalChunks: totalChunks, previewChunkCount: data.previewChunkCount }
+        return {
+            chunks: docs,
+            totalChunks: totalChunks,
+            previewChunkCount: previewChunkCount,
+            previewChunkOffset: previewChunkOffset
+        }
     } catch (error) {
         throw new InternalFlowiseError(
             StatusCodes.INTERNAL_SERVER_ERROR,

From 3e9e525c6f95cd2a6d882a0e584533597c3686a0 Mon Sep 17 00:00:00 2001
From: Diego Costa <diecoscai@gmail.com>
Date: Fri, 13 Mar 2026 17:09:10 -0300
Subject: [PATCH 10/14] fix(documentstore): delete only preexisting chunks
 after refresh/save

---
 .../src/services/documentstore/index.ts       | 42 ++++++++++++++-----
 .../test/services/documentstore/index.test.ts |  6 ++-
 2 files changed, 36 insertions(+), 12 deletions(-)

diff --git a/packages/server/src/services/documentstore/index.ts b/packages/server/src/services/documentstore/index.ts
index ddaa08fe838..cf101cf18ef 100644
--- a/packages/server/src/services/documentstore/index.ts
+++ b/packages/server/src/services/documentstore/index.ts
@@ -442,6 +442,17 @@ const syncAndRefreshChunks = async (storeId: string, fileId: string, userId: str
         entity.loaders = JSON.stringify(loaders)
         await appServer.AppDataSource.getRepository(DocumentStore).save(entity)
 
+        const chunkRepository = appServer.AppDataSource.getRepository(DocumentStoreFileChunk)
+        const existingChunkIds = (
+            await chunkRepository.find({
+                where: {
+                    docId: fileId,
+                    userId,
+                    organizationId
+                }
+            })
+        ).map((chunk) => chunk.id)
+
         // Get fresh documents from Google Drive
         const docs = await _splitIntoChunks(appServer.AppDataSource, componentNodes, data, userId, organizationId)
 
@@ -466,8 +477,8 @@ const syncAndRefreshChunks = async (storeId: string, fileId: string, userId: str
                             pageContent: chunk.pageContent,
                             metadata: JSON.stringify(chunk.metadata)
                         }
-                        const dChunk = appServer.AppDataSource.getRepository(DocumentStoreFileChunk).create(docChunk)
-                        await appServer.AppDataSource.getRepository(DocumentStoreFileChunk).save(dChunk)
+                        const dChunk = chunkRepository.create(docChunk)
+                        await chunkRepository.save(dChunk)
                     })
                 )
                 persistedChunks += batch.length
@@ -480,11 +491,9 @@ const syncAndRefreshChunks = async (storeId: string, fileId: string, userId: str
 
         if (!saveFailed) {
             // Delete old chunks only after all new chunks are confirmed saved
-            await appServer.AppDataSource.getRepository(DocumentStoreFileChunk).delete({
-                docId: fileId,
-                userId,
-                organizationId
-            })
+            if (existingChunkIds.length > 0) {
+                await chunkRepository.delete({ id: In(existingChunkIds) })
+            }
             loader.totalChunks = persistedChunks
             loader.totalChars = persistedChars
             loader.status = DocumentStoreStatus.SYNC
@@ -1272,6 +1281,17 @@ const _saveChunksToStorage = async (
             existingLoaders.push(loader)
         }
 
+        const chunkRepository = appDataSource.getRepository(DocumentStoreFileChunk)
+        const existingChunkIds = (
+            await chunkRepository.find({
+                where: {
+                    docId: newLoaderId,
+                    userId: data.userId,
+                    organizationId: data.organizationId
+                }
+            })
+        ).map((chunk) => chunk.id)
+
         if (response.chunks) {
             //step 7: save new chunks first (safe delete timing — delete AFTER all saved)
             let persistedChunks = 0
@@ -1294,8 +1314,8 @@ const _saveChunksToStorage = async (
                                 userId: data.userId,
                                 organizationId: data.organizationId
                             }
-                            const dChunk = appDataSource.getRepository(DocumentStoreFileChunk).create(docChunk)
-                            await appDataSource.getRepository(DocumentStoreFileChunk).save(dChunk)
+                            const dChunk = chunkRepository.create(docChunk)
+                            await chunkRepository.save(dChunk)
                         })
                     )
                     persistedChunks += batch.length
@@ -1308,7 +1328,9 @@ const _saveChunksToStorage = async (
 
             if (!saveFailed) {
                 //step 8: delete old chunks only after all new chunks are confirmed saved
-                await appDataSource.getRepository(DocumentStoreFileChunk).delete({ docId: newLoaderId })
+                if (existingChunkIds.length > 0) {
+                    await chunkRepository.delete({ id: In(existingChunkIds) })
+                }
                 loader.totalChunks = persistedChunks
                 loader.totalChars = persistedChars
                 loader.status = 'SYNC'
diff --git a/packages/server/test/services/documentstore/index.test.ts b/packages/server/test/services/documentstore/index.test.ts
index 5e5b773849e..0d9ac1e8e05 100644
--- a/packages/server/test/services/documentstore/index.test.ts
+++ b/packages/server/test/services/documentstore/index.test.ts
@@ -247,7 +247,7 @@ describe('_saveChunksToStorage durability contract', () => {
         mockChunkSave.mockResolvedValue(undefined)
         mockChunkDelete.mockResolvedValue(undefined)
         mockChunkCount.mockResolvedValue(0)
-        mockChunkFind.mockResolvedValue([])
+        mockChunkFind.mockResolvedValue([{ id: 'existing-chunk-1' }])
     })
 
     describe('happy path', () => {
@@ -280,6 +280,8 @@ describe('_saveChunksToStorage durability contract', () => {
 
             const expectedChars = chunks.reduce((acc, c) => acc + c.pageContent.length, 0)
             expect(loaders[0].totalChars).toBe(expectedChars)
+            expect(mockChunkDelete).toHaveBeenCalledTimes(1)
+            expect(mockChunkDelete).not.toHaveBeenCalledWith({ docId: LOADER_ID })
         })
     })
 
@@ -415,7 +417,7 @@ describe('_saveChunksToStorage durability contract', () => {
             mockChunkSave.mockResolvedValue(undefined)
             mockChunkDelete.mockResolvedValue(undefined)
             mockChunkCount.mockResolvedValue(0)
-            mockChunkFind.mockResolvedValue([])
+            mockChunkFind.mockResolvedValue([{ id: 'existing-chunk-2' }])
 
             await processLoader({
                 appDataSource,

From 5bbb6fe54692bf96a093d76c26e4b319b26867db Mon Sep 17 00:00:00 2001
From: Diego Costa <diecoscai@gmail.com>
Date: Fri, 13 Mar 2026 17:45:58 -0300
Subject: [PATCH 11/14] fix(lint): address Node CI errors in documentstore
 files

---
 .../src/services/documentstore/index.ts       | 53 +++++++++++++------
 .../test/services/documentstore/index.test.ts | 46 +++++++++-------
 2 files changed, 62 insertions(+), 37 deletions(-)

diff --git a/packages/server/src/services/documentstore/index.ts b/packages/server/src/services/documentstore/index.ts
index cf101cf18ef..4e09a3a6221 100644
--- a/packages/server/src/services/documentstore/index.ts
+++ b/packages/server/src/services/documentstore/index.ts
@@ -1652,6 +1652,13 @@ const _insertIntoVectorStoreWorkerThread = async (
         const recordManagerConfig =
             typeof data.recordManagerConfig === 'string' ? JSON.parse(data.recordManagerConfig) : data.recordManagerConfig
         const isFullCleanup = recordManagerObj && recordManagerConfig?.cleanup === 'full'
+        const parseChunkMetadata = (metadata?: string) => {
+            try {
+                return metadata ? JSON.parse(metadata) : {}
+            } catch {
+                return {}
+            }
+        }
 
         let indexResult: ICommonObject | undefined
         // IMPORTANT: Full-cleanup mode must call upsert() exactly once with ALL docs accumulated.
@@ -1665,23 +1672,30 @@ const _insertIntoVectorStoreWorkerThread = async (
             // stable during this worker iteration — no concurrent chunk writes are expected.
             // If concurrent writes are introduced in future, migrate to a monotonic sequence cursor.
             let lastId = ''
-            while (true) {
-                const batchFilter: ICommonObject = lastId
-                    ? { ...filterOptions, id: MoreThan(lastId) }
-                    : { ...filterOptions }
+            let hasMore = true
+            while (hasMore) {
+                const batchFilter: ICommonObject = lastId ? { ...filterOptions, id: MoreThan(lastId) } : { ...filterOptions }
                 const chunks = await appDataSource.getRepository(DocumentStoreFileChunk).find({
                     where: batchFilter,
                     take: UPSERT_BATCH_SIZE,
                     order: { id: 'ASC' }
                 })
-                if (chunks.length === 0) break
+                if (chunks.length === 0) {
+                    hasMore = false
+                    continue
+                }
                 for (const chunk of chunks) {
-                    docs.push(new Document({
-                        pageContent: chunk.pageContent,
-                        metadata: (() => { try { return chunk.metadata ? JSON.parse(chunk.metadata) : {} } catch { return {} } })()
-                    }))
+                    docs.push(
+                        new Document({
+                            pageContent: chunk.pageContent,
+                            metadata: parseChunkMetadata(chunk.metadata)
+                        })
+                    )
+                }
+                if (chunks.length < UPSERT_BATCH_SIZE) {
+                    hasMore = false
+                    continue
                 }
-                if (chunks.length < UPSERT_BATCH_SIZE) break
                 lastId = chunks[chunks.length - 1].id
             }
             vStoreNodeData.inputs.document = docs
@@ -1694,20 +1708,22 @@ const _insertIntoVectorStoreWorkerThread = async (
             // stable during this worker iteration — no concurrent chunk writes are expected.
             // If concurrent writes are introduced in future, migrate to a monotonic sequence cursor.
             let lastId = ''
-            while (true) {
-                const batchFilter: ICommonObject = lastId
-                    ? { ...filterOptions, id: MoreThan(lastId) }
-                    : { ...filterOptions }
+            let hasMore = true
+            while (hasMore) {
+                const batchFilter: ICommonObject = lastId ? { ...filterOptions, id: MoreThan(lastId) } : { ...filterOptions }
                 const chunks = await appDataSource.getRepository(DocumentStoreFileChunk).find({
                     where: batchFilter,
                     take: UPSERT_BATCH_SIZE,
                     order: { id: 'ASC' }
                 })
-                if (chunks.length === 0) break
+                if (chunks.length === 0) {
+                    hasMore = false
+                    continue
+                }
                 const docs: Document[] = chunks.map((chunk: DocumentStoreFileChunk) => {
                     return new Document({
                         pageContent: chunk.pageContent,
-                        metadata: (() => { try { return chunk.metadata ? JSON.parse(chunk.metadata) : {} } catch { return {} } })()
+                        metadata: parseChunkMetadata(chunk.metadata)
                     })
                 })
                 vStoreNodeData.inputs.document = docs
@@ -1726,7 +1742,10 @@ const _insertIntoVectorStoreWorkerThread = async (
                     }
                 }
 
-                if (chunks.length < UPSERT_BATCH_SIZE) break
+                if (chunks.length < UPSERT_BATCH_SIZE) {
+                    hasMore = false
+                    continue
+                }
                 lastId = chunks[chunks.length - 1].id
             }
         }
diff --git a/packages/server/test/services/documentstore/index.test.ts b/packages/server/test/services/documentstore/index.test.ts
index 0d9ac1e8e05..b46dce7827e 100644
--- a/packages/server/test/services/documentstore/index.test.ts
+++ b/packages/server/test/services/documentstore/index.test.ts
@@ -2,13 +2,17 @@
 
 let mockReturnChunks: any[] = []
 
-jest.mock('/virtual/test/mock-loader', () => ({
-    nodeClass: class MockLoader {
-        async init() {
-            return mockReturnChunks
+jest.mock(
+    '/virtual/test/mock-loader',
+    () => ({
+        nodeClass: class MockLoader {
+            async init() {
+                return mockReturnChunks
+            }
         }
-    }
-}), { virtual: true })
+    }),
+    { virtual: true }
+)
 
 const mockChunkSave = jest.fn()
 const mockChunkCreate = jest.fn((data: any) => data)
@@ -165,20 +169,22 @@ const makeEntity = () => ({
     id: STORE_ID,
     name: 'Test Store',
     description: 'test',
-    loaders: JSON.stringify([{
-        id: LOADER_ID,
-        loaderId: 'mockLoader',
-        loaderName: 'Mock Loader',
-        loaderConfig: { text: 'hello' },
-        splitterId: '',
-        splitterConfig: {},
-        credential: '',
-        status: 'SYNCING',
-        totalChunks: 0,
-        totalChars: 0,
-        files: [],
-        source: ''
-    }]),
+    loaders: JSON.stringify([
+        {
+            id: LOADER_ID,
+            loaderId: 'mockLoader',
+            loaderName: 'Mock Loader',
+            loaderConfig: { text: 'hello' },
+            splitterId: '',
+            splitterConfig: {},
+            credential: '',
+            status: 'SYNCING',
+            totalChunks: 0,
+            totalChars: 0,
+            files: [],
+            source: ''
+        }
+    ]),
     whereUsed: '[]',
     status: 'SYNCING',
     vectorStoreConfig: null,

From ad241dedeae2e43289413a235109076afe5a14f4 Mon Sep 17 00:00:00 2001
From: Diego Costa <diecoscai@gmail.com>
Date: Fri, 13 Mar 2026 18:23:38 -0300
Subject: [PATCH 12/14] revert(components): drop AAIDOMAINS env limit cap

---
 .../nodes/documentloaders/AAIDomains/AAIDomains.ts        | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/packages/components/nodes/documentloaders/AAIDomains/AAIDomains.ts b/packages/components/nodes/documentloaders/AAIDomains/AAIDomains.ts
index b0e228e8003..3df934eb6cc 100644
--- a/packages/components/nodes/documentloaders/AAIDomains/AAIDomains.ts
+++ b/packages/components/nodes/documentloaders/AAIDomains/AAIDomains.ts
@@ -247,17 +247,11 @@ class AAIDomains_DocumentLoaders implements INode {
 
         // Convert 'all' to null for RPC function (per API spec)
         const isValidFilter = isValid === 'all' ? null : isValid
-        const requestedLimit = limit || 100
-        const envConfiguredLimit = Number(process.env.AAIDOMAINS_LIMIT ?? process.env.AAI_DOMAINS_MAX_LIMIT ?? 0)
-        const effectiveLimit =
-            Number.isFinite(envConfiguredLimit) && envConfiguredLimit > 0
-                ? Math.min(requestedLimit, Math.floor(envConfiguredLimit))
-                : requestedLimit
 
         const loaderOptions: AAIDomainsLoaderParams = {
             supabaseUrl,
             supabaseKey,
-            limit: effectiveLimit,
+            limit: limit || 100,
             searchTerm: searchTerm || null,
             includeTags: includeTags ? includeTags.split(',').map((t) => t.trim()) : [],
             includeTagsLogic,

From 9349114000798a78568876ea5f5b626f4fef2816 Mon Sep 17 00:00:00 2001
From: Diego Costa <diecoscai@gmail.com>
Date: Fri, 13 Mar 2026 18:28:57 -0300
Subject: [PATCH 13/14] docs(documentstore): add load-error resolution summary
 from transcripts

---
 .../documentstore-load-error-resolution.md    | 60 +++++++++++++++++++
 1 file changed, 60 insertions(+)
 create mode 100644 docs/features/documentstore-load-error-resolution.md

diff --git a/docs/features/documentstore-load-error-resolution.md b/docs/features/documentstore-load-error-resolution.md
new file mode 100644
index 00000000000..44fd00c7e25
--- /dev/null
+++ b/docs/features/documentstore-load-error-resolution.md
@@ -0,0 +1,60 @@
+# Document Store Load Error Resolution
+
+## Goal
+Resolve load failures and instability when processing very large domain datasets in Document Store flows, especially during preview/process operations.
+
+## Problem Summary (from meeting transcripts)
+- High limits in preview/process could timeout or overload the platform when datasets reached tens of thousands of domains.
+- The processing path was effectively waiting to collect all chunks before safe persistence completed, creating memory pressure during large runs.
+- Failure handling risked ending in a bad state where a long run could fail late and lose practical progress.
+
+Source transcripts reviewed:
+- `/Users/diegocosta/Downloads/GMT20260311-174121_Recording.transcript.vtt`
+- `/Users/diegocosta/Downloads/GMT20260311-160055_Recording.transcript.vtt`
+- `/Users/diegocosta/Downloads/GMT20260312-182400_Recording.transcript.vtt`
+
+## Scope Delivered
+
+### 1) Durable batch persistence for chunk save flows
+File: `packages/server/src/services/documentstore/index.ts`
+
+- Save chunks in bounded batches (`SAVE_BATCH_SIZE`) with per-batch failure handling.
+- Preserve partial progress by updating loader/store status to `STALE` when a batch fails.
+- Report real persisted counts (`totalChunks`, `totalChars`) instead of optimistic totals.
+
+### 2) Safe delete timing with pre-existing scope only
+File: `packages/server/src/services/documentstore/index.ts`
+
+- Capture pre-existing chunk IDs before inserting replacements.
+- Delete only those pre-existing IDs after successful persistence.
+- Avoid broad post-save deletes that can match newly inserted rows.
+
+### 3) Upsert pipeline stability for large datasets
+File: `packages/server/src/services/documentstore/index.ts`
+
+- Keep keyset pagination (`MoreThan(lastId)`) for chunk reads into upsert.
+- Preserve full-cleanup contract by accumulating docs and calling upsert once in that branch.
+- Maintain organization scoping on read/write/delete paths.
+
+### 4) Preview pagination contract
+Files:
+- `packages/server/src/Interface.DocumentStore.ts`
+- `packages/server/src/services/documentstore/index.ts`
+
+- Added `previewChunkOffset` to preview data contract.
+- Added offset-aware slicing in `previewChunks` response.
+
+## What This Fixes
+- Reduces timeout/instability risk under high-volume loads by avoiding all-or-nothing behavior in save paths.
+- Prevents accidental deletion of newly saved chunks in refresh/save flows.
+- Improves recoverability of long-running loads by preserving persisted progress and clear status transitions.
+
+## Validation Performed
+- `pnpm --filter flowise test -- --runInBand test/services/documentstore/index.test.ts`
+- `pnpm --filter flowise build`
+- `pnpm --filter flowise-components build`
+
+## Remaining Performance Note
+- `previewChunks` currently still generates all chunks before applying offset/count slicing.
+- This is functionally correct but can be expensive for very large datasets.
+- Follow-up optimization can move to windowed generation to avoid materializing the full set for small previews.

From 74dd022580c7ed48d8e2a03e5b2369efd55957d6 Mon Sep 17 00:00:00 2001
From: Diego Costa <diecoscai@gmail.com>
Date: Fri, 13 Mar 2026 18:33:43 -0300
Subject: [PATCH 14/14] chore(pr): remove transcript-based doc from repository

---
 .../documentstore-load-error-resolution.md    | 60 -------------------
 1 file changed, 60 deletions(-)
 delete mode 100644 docs/features/documentstore-load-error-resolution.md

diff --git a/docs/features/documentstore-load-error-resolution.md b/docs/features/documentstore-load-error-resolution.md
deleted file mode 100644
index 44fd00c7e25..00000000000
--- a/docs/features/documentstore-load-error-resolution.md
+++ /dev/null
@@ -1,60 +0,0 @@
-# Document Store Load Error Resolution
-
-## Goal
-Resolve load failures and instability when processing very large domain datasets in Document Store flows, especially during preview/process operations.
-
-## Problem Summary (from meeting transcripts)
-- High limits in preview/process could timeout or overload the platform when datasets reached tens of thousands of domains.
-- The processing path was effectively waiting to collect all chunks before safe persistence completed, creating memory pressure during large runs.
-- Failure handling risked ending in a bad state where a long run could fail late and lose practical progress.
-
-Source transcripts reviewed:
-- `/Users/diegocosta/Downloads/GMT20260311-174121_Recording.transcript.vtt`
-- `/Users/diegocosta/Downloads/GMT20260311-160055_Recording.transcript.vtt`
-- `/Users/diegocosta/Downloads/GMT20260312-182400_Recording.transcript.vtt`
-
-## Scope Delivered
-
-### 1) Durable batch persistence for chunk save flows
-File: `packages/server/src/services/documentstore/index.ts`
-
-- Save chunks in bounded batches (`SAVE_BATCH_SIZE`) with per-batch failure handling.
-- Preserve partial progress by updating loader/store status to `STALE` when a batch fails.
-- Report real persisted counts (`totalChunks`, `totalChars`) instead of optimistic totals.
-
-### 2) Safe delete timing with pre-existing scope only
-File: `packages/server/src/services/documentstore/index.ts`
-
-- Capture pre-existing chunk IDs before inserting replacements.
-- Delete only those pre-existing IDs after successful persistence.
-- Avoid broad post-save deletes that can match newly inserted rows.
-
-### 3) Upsert pipeline stability for large datasets
-File: `packages/server/src/services/documentstore/index.ts`
-
-- Keep keyset pagination (`MoreThan(lastId)`) for chunk reads into upsert.
-- Preserve full-cleanup contract by accumulating docs and calling upsert once in that branch.
-- Maintain organization scoping on read/write/delete paths.
-
-### 4) Preview pagination contract
-Files:
-- `packages/server/src/Interface.DocumentStore.ts`
-- `packages/server/src/services/documentstore/index.ts`
-
-- Added `previewChunkOffset` to preview data contract.
-- Added offset-aware slicing in `previewChunks` response.
-
-## What This Fixes
-- Reduces timeout/instability risk under high-volume loads by avoiding all-or-nothing behavior in save paths.
-- Prevents accidental deletion of newly saved chunks in refresh/save flows.
-- Improves recoverability of long-running loads by preserving persisted progress and clear status transitions.
-
-## Validation Performed
-- `pnpm --filter flowise test -- --runInBand test/services/documentstore/index.test.ts`
-- `pnpm --filter flowise build`
-- `pnpm --filter flowise-components build`
-
-## Remaining Performance Note
-- `previewChunks` currently still generates all chunks before applying offset/count slicing.
-- This is functionally correct but can be expensive for very large datasets.
-- Follow-up optimization can move to windowed generation to avoid materializing the full set for small previews.