diff --git a/packages/components/nodes/documentloaders/AAIDomains/AAIDomains.ts b/packages/components/nodes/documentloaders/AAIDomains/AAIDomains.ts new file mode 100644 index 00000000000..a7ccc8e31bc --- /dev/null +++ b/packages/components/nodes/documentloaders/AAIDomains/AAIDomains.ts @@ -0,0 +1,708 @@ +import { omit } from 'lodash' +import { Document } from '@langchain/core/documents' +import { TextSplitter } from 'langchain/text_splitter' +import { BaseDocumentLoader } from 'langchain/document_loaders/base' +import { IDocument, ICommonObject, INode, INodeData, INodeParams, INodeOutputsValue } from '../../../src/Interface' +import { handleEscapeCharacters } from '../../../src' +import { createClient, SupabaseClient } from '@supabase/supabase-js' + +class AAIDomains_DocumentLoaders implements INode { + label: string + name: string + version: number + description: string + type: string + icon: string + category: string + tags: string[] + baseClasses: string[] + inputs: INodeParams[] + outputs: INodeOutputsValue[] + + constructor() { + this.label = 'AnswerAgentAI Domains' + this.name = 'aaiDomains' + this.version = 1.0 + this.type = 'Document' + this.icon = 'answerai-square-black.png' + this.category = 'Document Loaders' + this.description = 'Load domain information from AnswerAgentAI Datastore' + this.tags = ['AAI'] + this.baseClasses = [this.type] + this.inputs = [ + { + label: 'Text Splitter', + name: 'textSplitter', + type: 'TextSplitter', + optional: true + }, + { + label: 'Limit', + name: 'limit', + type: 'number', + default: 100, + description: 'Maximum number of domains to load', + optional: true + }, + { + label: 'Search Term', + name: 'searchTerm', + type: 'string', + description: 'Search in domain name, meta title, and description', + optional: true, + additionalParams: true + }, + { + label: 'Include Tags', + name: 'includeTags', + type: 'string', + placeholder: 'ecommerce,saas', + description: 'Comma-separated list of tag slugs to include', + optional: true, + additionalParams: true + }, + { + label: 'Include Tags Logic', + name: 'includeTagsLogic', + type: 'options', + options: [ + { + label: 'OR (any tag matches)', + name: 'OR' + }, + { + label: 'AND (all tags must match)', + name: 'AND' + } + ], + default: 'OR', + optional: true, + additionalParams: true + }, + { + label: 'Exclude Tags', + name: 'excludeTags', + type: 'string', + placeholder: 'spam,parked', + description: 'Comma-separated list of tag slugs to exclude', + optional: true, + additionalParams: true + }, + { + label: 'Is Valid', + name: 'isValid', + type: 'options', + options: [ + { + label: 'All', + name: 'all' + }, + { + label: 'Valid Only', + name: 'true' + }, + { + label: 'Invalid Only', + name: 'false' + }, + { + label: 'Unknown', + name: 'null' + } + ], + default: 'all', + optional: true, + additionalParams: true + }, + { + label: 'Has Analysis', + name: 'hasAnalysis', + type: 'options', + options: [ + { + label: 'All', + name: 'all' + }, + { + label: 'Analyzed Only', + name: 'analyzed' + }, + { + label: 'Not Analyzed', + name: 'not_analyzed' + } + ], + default: 'all', + optional: true, + additionalParams: true + }, + { + label: 'Include AI Analysis in Content', + name: 'includeAiAnalysis', + type: 'boolean', + default: true, + description: 'Include full AI analysis markdown in page content', + optional: true, + additionalParams: true + }, + { + label: 'Additional Metadata', + name: 'metadata', + type: 'json', + description: 'Additional metadata to be added to the extracted documents', + optional: true, + additionalParams: true + }, + { + label: 'Omit Metadata Keys', + name: 'omitMetadataKeys', + type: 'string', + rows: 4, + description: + 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field', + placeholder: 'key1, key2, key3.nestedKey1', + optional: true, + additionalParams: true + } + ] + this.outputs = [ + { + label: 'Document', + name: 'document', + description: 'Array of document objects containing metadata and pageContent', + baseClasses: [...this.baseClasses, 'json'] + }, + { + label: 'Text', + name: 'text', + description: 'Concatenated string from pageContent of documents', + baseClasses: ['string', 'json'] + } + ] + } + + async init(nodeData: INodeData): Promise { + const textSplitter = nodeData.inputs?.textSplitter as TextSplitter + const limit = nodeData.inputs?.limit as number + const searchTerm = nodeData.inputs?.searchTerm as string + const includeTags = nodeData.inputs?.includeTags as string + const includeTagsLogic = (nodeData.inputs?.includeTagsLogic as string) || 'OR' + const excludeTags = nodeData.inputs?.excludeTags as string + const isValid = (nodeData.inputs?.isValid as string) || 'all' + const hasAnalysis = (nodeData.inputs?.hasAnalysis as string) || 'all' + const includeAiAnalysis = nodeData.inputs?.includeAiAnalysis !== false + const metadata = nodeData.inputs?.metadata + const _omitMetadataKeys = nodeData.inputs?.omitMetadataKeys as string + const output = nodeData.outputs?.output as string + + let omitMetadataKeys: string[] = [] + if (_omitMetadataKeys) { + omitMetadataKeys = _omitMetadataKeys.split(',').map((key) => key.trim()) + } + + // Validate inputs + if (limit !== undefined && limit !== null) { + if (limit <= 0) { + throw new Error('Limit must be a positive number') + } + if (!Number.isInteger(limit)) { + throw new Error('Limit must be an integer') + } + } + + if (includeTagsLogic && !['OR', 'AND'].includes(includeTagsLogic)) { + throw new Error('Include Tags Logic must be either "OR" or "AND"') + } + + if (isValid && !['all', 'true', 'false', 'null'].includes(isValid)) { + throw new Error('Is Valid must be one of: "all", "true", "false", "null"') + } + + if (hasAnalysis && !['all', 'analyzed', 'not_analyzed'].includes(hasAnalysis)) { + throw new Error('Has Analysis must be one of: "all", "analyzed", "not_analyzed"') + } + + // Get environment variables + const supabaseUrl = process.env.AAI_DATASTORE_SUPABASE_URL + const supabaseKey = process.env.AAI_DATASTORE_SUPABASE_SERVICE_ROLE_KEY + + if (!supabaseUrl || !supabaseKey) { + throw new Error( + 'AAI Datastore configuration missing. Please set AAI_DATASTORE_SUPABASE_URL and AAI_DATASTORE_SUPABASE_SERVICE_ROLE_KEY environment variables.' + ) + } + + // Convert 'all' to null for RPC function (per API spec) + const isValidFilter = isValid === 'all' ? null : isValid + + const loaderOptions: AAIDomainsLoaderParams = { + supabaseUrl, + supabaseKey, + limit: limit || 100, + searchTerm: searchTerm || null, + includeTags: includeTags ? includeTags.split(',').map((t) => t.trim()) : [], + includeTagsLogic, + excludeTags: excludeTags ? excludeTags.split(',').map((t) => t.trim()) : [], + isValid: isValidFilter, + hasAnalysis, + includeAiAnalysis + } + + const loader = new AAIDomainsLoader(loaderOptions) + + let docs: IDocument[] = [] + + if (textSplitter) { + docs = await loader.load() + docs = await textSplitter.splitDocuments(docs) + } else { + docs = await loader.load() + } + + // Apply metadata + if (metadata) { + const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) + docs = docs.map((doc) => ({ + ...doc, + metadata: + _omitMetadataKeys === '*' + ? { + ...parsedMetadata + } + : omit( + { + ...doc.metadata, + ...parsedMetadata + }, + omitMetadataKeys + ) + })) + } else { + docs = docs.map((doc) => ({ + ...doc, + metadata: + _omitMetadataKeys === '*' + ? {} + : omit( + { + ...doc.metadata + }, + omitMetadataKeys + ) + })) + } + + if (output === 'text') { + let finalText = '' + for (const doc of docs) { + finalText += `${doc.pageContent}\n` + } + return handleEscapeCharacters(finalText, false) + } + + return docs + } +} + +interface AAIDomainsLoaderParams { + supabaseUrl: string + supabaseKey: string + limit: number + searchTerm: string | null + includeTags: string[] + includeTagsLogic: string + excludeTags: string[] + isValid: string | null + hasAnalysis: string + includeAiAnalysis: boolean +} + +class AAIDomainsLoader extends BaseDocumentLoader { + private supabaseUrl: string + private supabaseKey: string + private limit: number + private searchTerm: string | null + private includeTags: string[] + private includeTagsLogic: string + private excludeTags: string[] + private isValid: string | null + private hasAnalysis: string + private includeAiAnalysis: boolean + + constructor(params: AAIDomainsLoaderParams) { + super() + this.supabaseUrl = params.supabaseUrl + this.supabaseKey = params.supabaseKey + this.limit = params.limit + this.searchTerm = params.searchTerm + this.includeTags = params.includeTags + this.includeTagsLogic = params.includeTagsLogic + this.excludeTags = params.excludeTags + this.isValid = params.isValid + this.hasAnalysis = params.hasAnalysis + this.includeAiAnalysis = params.includeAiAnalysis + } + + public async load(): Promise { + const supabase: SupabaseClient = createClient(this.supabaseUrl, this.supabaseKey, { + global: { + fetch: (...args) => { + const [resource, config] = args + // Set 60 second timeout for RPC calls + const controller = new AbortController() + const timeoutId = setTimeout(() => controller.abort(), 60000) + + return fetch(resource, { + ...config, + signal: controller.signal + }).finally(() => clearTimeout(timeoutId)) + } + } + }) + + // Use larger page size since we're only selecting essential fields + const pageSize = Math.min(this.limit, 100) + let allDomains: any[] = [] + let currentPage = 0 + + console.log('[AAIDomains] Starting load with params:', { + limit: this.limit, + searchTerm: this.searchTerm, + includeTags: this.includeTags, + includeTagsLogic: this.includeTagsLogic, + excludeTags: this.excludeTags, + isValid: this.isValid, + hasAnalysis: this.hasAnalysis + }) + + while (allDomains.length < this.limit) { + const remainingItems = this.limit - allDomains.length + const currentPageSize = Math.min(pageSize, remainingItems) + + console.log(`[AAIDomains] Fetching page ${currentPage}, size ${currentPageSize}`) + + // Retry logic with exponential backoff + let retryCount = 0 + const maxRetries = 3 + let lastError: any = null + let shouldStopPagination = false + + while (retryCount < maxRetries) { + try { + // Build query with only essential fields (exclude heavy JSONB history fields) + // NOTE: We explicitly exclude: ai_analysis_overrides, ai_analysis_history, + // override_metadata, source_metadata which can be multi-MB per record + let query = supabase + .from('domains') + .select( + ` + id, + domain_name, + is_valid, + meta_title, + meta_description, + robots_txt, + sitemaps, + custom_data, + ai_analysis, + created_at, + updated_at, + last_analyzed_at, + domain_tags( + tag_id, + tags(id, slug, label, color, parent_id) + ) + ` + ) + .order('updated_at', { ascending: false }) + .range(currentPage * pageSize, (currentPage + 1) * pageSize - 1) + + // Apply filters + if (this.searchTerm) { + query = query.or( + `domain_name.ilike.%${this.searchTerm}%,meta_title.ilike.%${this.searchTerm}%,meta_description.ilike.%${this.searchTerm}%` + ) + } + + if (this.isValid !== null) { + if (this.isValid === 'true') { + query = query.eq('is_valid', true) + } else if (this.isValid === 'false') { + query = query.eq('is_valid', false) + } else if (this.isValid === 'null') { + query = query.is('is_valid', null) + } + } + + if (this.hasAnalysis === 'analyzed') { + query = query.not('ai_analysis', 'is', null) + } else if (this.hasAnalysis === 'not_analyzed') { + query = query.is('ai_analysis', null) + } + + const { data, error } = await query + + if (error) { + console.error('[AAIDomains] Query error:', error) + throw new Error(`Failed to fetch domains from AAI Datastore: ${error.message}`) + } + + console.log(`[AAIDomains] Page ${currentPage} response:`, { + hasData: !!data, + domainsCount: data?.length || 0 + }) + + if (!data || data.length === 0) { + console.log('[AAIDomains] No more data, stopping pagination') + shouldStopPagination = true + break + } + + // Transform domain_tags array to flat tags array + const domainsWithTags = data.map((domain: any) => ({ + ...domain, + tags: domain.domain_tags?.map((dt: any) => dt.tags).filter(Boolean) || [] + })) + + // Apply tag filtering if specified + let filteredDomains = domainsWithTags + if (this.includeTags.length > 0 || this.excludeTags.length > 0) { + filteredDomains = this.filterByTags(domainsWithTags) + } + + allDomains.push(...filteredDomains) + currentPage++ + + // Stop if we've fetched enough + if (allDomains.length >= this.limit || data.length < currentPageSize) { + shouldStopPagination = true + break + } + + // Success - break out of retry loop + break + } catch (error: any) { + lastError = error + retryCount++ + + if (retryCount < maxRetries) { + const delayMs = Math.min(1000 * Math.pow(2, retryCount), 10000) + console.warn( + `[AAIDomains] Request failed (attempt ${retryCount}/${maxRetries}), retrying in ${delayMs}ms...`, + error.message + ) + await new Promise((resolve) => setTimeout(resolve, delayMs)) + } else { + console.error(`[AAIDomains] All ${maxRetries} retry attempts failed`) + throw lastError + } + } + } + + // If we got no data, hit the limit, or all retries failed, break out of pagination loop + if (shouldStopPagination || retryCount >= maxRetries) { + break + } + + // Add small delay between successful pages to avoid overwhelming the server + await new Promise((resolve) => setTimeout(resolve, 200)) + } + + console.log(`[AAIDomains] Load complete. Total domains: ${allDomains.length}`) + + // Truncate to exact limit + if (allDomains.length > this.limit) { + allDomains = allDomains.slice(0, this.limit) + } + + return allDomains.map((domain) => this.createDocumentFromDomain(domain)) + } + + private filterByTags(domains: any[]): any[] { + return domains.filter((domain) => { + const domainTagSlugs = domain.tags?.map((t: any) => t.slug) || [] + + // Apply include tags filter + if (this.includeTags.length > 0) { + if (this.includeTagsLogic === 'AND') { + // Domain must have ALL include tags + const hasAllTags = this.includeTags.every((tag) => domainTagSlugs.includes(tag)) + if (!hasAllTags) return false + } else { + // Domain must have AT LEAST ONE include tag (OR logic) + const hasAnyTag = this.includeTags.some((tag) => domainTagSlugs.includes(tag)) + if (!hasAnyTag) return false + } + } + + // Apply exclude tags filter + if (this.excludeTags.length > 0) { + // Exclude if domain has ANY of the exclude tags (OR logic for exclusion) + const hasExcludedTag = this.excludeTags.some((tag) => domainTagSlugs.includes(tag)) + if (hasExcludedTag) return false + } + + return true + }) + } + + private formatAiAnalysisAsMarkdown(aiAnalysis: any): string { + if (!aiAnalysis || Object.keys(aiAnalysis).length === 0) { + return '' + } + + const sections: string[] = ['## AI Analysis', ''] + + // Summary + if (aiAnalysis.summary) { + sections.push('### Summary', aiAnalysis.summary, '') + } + + // Category/Industry + if (aiAnalysis.category || aiAnalysis.industry) { + sections.push('### Classification', '') + if (aiAnalysis.category) { + sections.push(`**Category:** ${aiAnalysis.category}`) + } + if (aiAnalysis.industry) { + sections.push(`**Industry:** ${aiAnalysis.industry}`) + } + sections.push('') + } + + // Key Insights + if (aiAnalysis.key_insights) { + sections.push('### Key Insights', '') + if (Array.isArray(aiAnalysis.key_insights)) { + aiAnalysis.key_insights.forEach((insight: string) => { + sections.push(`- ${insight}`) + }) + } else if (typeof aiAnalysis.key_insights === 'string') { + sections.push(aiAnalysis.key_insights) + } + sections.push('') + } + + // Technologies/Features + if (aiAnalysis.technologies || aiAnalysis.features) { + sections.push('### Technical Details', '') + if (aiAnalysis.technologies) { + const techs = Array.isArray(aiAnalysis.technologies) ? aiAnalysis.technologies.join(', ') : aiAnalysis.technologies + sections.push(`**Technologies:** ${techs}`) + } + if (aiAnalysis.features) { + const feats = Array.isArray(aiAnalysis.features) ? aiAnalysis.features.join(', ') : aiAnalysis.features + sections.push(`**Features:** ${feats}`) + } + sections.push('') + } + + // Confidence Score + if (aiAnalysis.confidence_score) { + sections.push(`**Confidence Score:** ${aiAnalysis.confidence_score}`, '') + } + + // Additional fields (catch-all for any other properties) + const knownFields = [ + 'summary', + 'category', + 'industry', + 'key_insights', + 'technologies', + 'features', + 'confidence_score', + 'analysis_version' + ] + const additionalFields = Object.keys(aiAnalysis).filter((key) => !knownFields.includes(key)) + + if (additionalFields.length > 0) { + sections.push('### Additional Information', '') + additionalFields.forEach((key) => { + const value = aiAnalysis[key] + if (value && typeof value !== 'object') { + sections.push(`**${key.replace(/_/g, ' ').replace(/\b\w/g, (l) => l.toUpperCase())}:** ${value}`) + } else if (value && typeof value === 'object') { + sections.push(`**${key.replace(/_/g, ' ').replace(/\b\w/g, (l) => l.toUpperCase())}:**`) + sections.push('```json') + sections.push(JSON.stringify(value, null, 2)) + sections.push('```') + } + }) + } + + return sections.filter(Boolean).join('\n') + } + + private createDocumentFromDomain(domain: any): Document { + // Extract custom data and AI analysis + const customData = domain.custom_data || {} + const aiAnalysis = domain.ai_analysis || {} + + // Format AI analysis as markdown if requested + const aiAnalysisMarkdown = this.includeAiAnalysis ? this.formatAiAnalysisAsMarkdown(aiAnalysis) : '' + + // Create page content from meta description or domain analysis + const pageContent = [ + `# ${domain.domain_name}`, + '', + domain.meta_title ? `## ${domain.meta_title}` : '', + domain.meta_description ? `${domain.meta_description}` : '', + '', + aiAnalysisMarkdown + ] + .filter(Boolean) + .join('\n') + + // Build comprehensive metadata + const metadata: ICommonObject = { + // Core fields + id: domain.id, + domain_name: domain.domain_name, + is_valid: domain.is_valid, + + // Custom data (spread all custom fields) + ...customData, + + // AI analysis + ai_summary: aiAnalysis.summary, + ai_category: aiAnalysis.category, + ai_industry: aiAnalysis.industry, + ai_confidence: aiAnalysis.confidence_score, + + // Tags + tags: domain.tags || [], + tag_slugs: (domain.tags || []).map((t: any) => t.slug).filter(Boolean), + tag_labels: (domain.tags || []).map((t: any) => t.label).filter(Boolean), + + // Timestamps (ISO strings) + created_at: domain.created_at, + updated_at: domain.updated_at, + last_analyzed_at: domain.last_analyzed_at, + + // Timestamps (Unix timestamps in seconds for Pinecone numeric filters) + created_at_timestamp: domain.created_at ? Math.floor(new Date(domain.created_at).getTime() / 1000) : null, + updated_at_timestamp: domain.updated_at ? Math.floor(new Date(domain.updated_at).getTime() / 1000) : null, + last_analyzed_at_timestamp: domain.last_analyzed_at ? Math.floor(new Date(domain.last_analyzed_at).getTime() / 1000) : null, + + // Source + source: 'aai_datastore', + source_type: 'domain' + } + + // Remove undefined/null values + Object.keys(metadata).forEach((key) => { + if (metadata[key] === undefined || metadata[key] === null) { + delete metadata[key] + } + }) + + return new Document({ + pageContent, + metadata + }) + } +} + +module.exports = { + nodeClass: AAIDomains_DocumentLoaders +} diff --git a/packages/components/nodes/documentloaders/AAIDomains/answerai-square-black.png b/packages/components/nodes/documentloaders/AAIDomains/answerai-square-black.png new file mode 100644 index 00000000000..24dfd99f98c Binary files /dev/null and b/packages/components/nodes/documentloaders/AAIDomains/answerai-square-black.png differ diff --git a/packages/components/nodes/documentloaders/AAITags/AAITags.ts b/packages/components/nodes/documentloaders/AAITags/AAITags.ts new file mode 100644 index 00000000000..6a09d5c1fc1 --- /dev/null +++ b/packages/components/nodes/documentloaders/AAITags/AAITags.ts @@ -0,0 +1,368 @@ +import { omit } from 'lodash' +import { Document } from '@langchain/core/documents' +import { TextSplitter } from 'langchain/text_splitter' +import { BaseDocumentLoader } from 'langchain/document_loaders/base' +import { IDocument, ICommonObject, INode, INodeData, INodeParams, INodeOutputsValue } from '../../../src/Interface' +import { handleEscapeCharacters } from '../../../src' +import { createClient, SupabaseClient } from '@supabase/supabase-js' + +class AAITags_DocumentLoaders implements INode { + label: string + name: string + version: number + description: string + type: string + icon: string + category: string + tags: string[] + baseClasses: string[] + inputs: INodeParams[] + outputs: INodeOutputsValue[] + + constructor() { + this.label = 'AnswerAgentAI Tags' + this.name = 'aaiTags' + this.version = 1.0 + this.type = 'Document' + this.icon = 'answerai-square-black.png' + this.category = 'Document Loaders' + this.description = 'Load tags from AnswerAgentAI Datastore' + this.tags = ['AAI'] + this.baseClasses = [this.type] + this.inputs = [ + { + label: 'Text Splitter', + name: 'textSplitter', + type: 'TextSplitter', + optional: true + }, + { + label: 'Limit', + name: 'limit', + type: 'number', + default: 1000, + description: 'Maximum number of tags to load', + optional: true + }, + { + label: 'Search Term', + name: 'searchTerm', + type: 'string', + description: 'Search in tag slug, label, and description', + optional: true, + additionalParams: true + }, + { + label: 'Parent Tags Only', + name: 'parentTagsOnly', + type: 'boolean', + default: false, + description: 'Load only parent (top-level) tags', + optional: true, + additionalParams: true + }, + { + label: 'Child Tags Only', + name: 'childTagsOnly', + type: 'boolean', + default: false, + description: 'Load only child tags (tags with a parent)', + optional: true, + additionalParams: true + }, + { + label: 'Include Parent Info', + name: 'includeParentInfo', + type: 'boolean', + default: true, + description: 'Include parent tag information in child tags', + optional: true, + additionalParams: true + }, + { + label: 'Additional Metadata', + name: 'metadata', + type: 'json', + description: 'Additional metadata to be added to the extracted documents', + optional: true, + additionalParams: true + }, + { + label: 'Omit Metadata Keys', + name: 'omitMetadataKeys', + type: 'string', + rows: 4, + description: + 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field', + placeholder: 'key1, key2, key3.nestedKey1', + optional: true, + additionalParams: true + } + ] + this.outputs = [ + { + label: 'Document', + name: 'document', + description: 'Array of document objects containing metadata and pageContent', + baseClasses: [...this.baseClasses, 'json'] + }, + { + label: 'Text', + name: 'text', + description: 'Concatenated string from pageContent of documents', + baseClasses: ['string', 'json'] + } + ] + } + + async init(nodeData: INodeData): Promise { + const textSplitter = nodeData.inputs?.textSplitter as TextSplitter + const limit = nodeData.inputs?.limit as number + const searchTerm = nodeData.inputs?.searchTerm as string + const parentTagsOnly = nodeData.inputs?.parentTagsOnly as boolean + const childTagsOnly = nodeData.inputs?.childTagsOnly as boolean + const includeParentInfo = nodeData.inputs?.includeParentInfo as boolean + const metadata = nodeData.inputs?.metadata + const _omitMetadataKeys = nodeData.inputs?.omitMetadataKeys as string + const output = nodeData.outputs?.output as string + + let omitMetadataKeys: string[] = [] + if (_omitMetadataKeys) { + omitMetadataKeys = _omitMetadataKeys.split(',').map((key) => key.trim()) + } + + // Validate inputs + if (limit !== undefined && limit !== null) { + if (limit <= 0) { + throw new Error('Limit must be a positive number') + } + if (!Number.isInteger(limit)) { + throw new Error('Limit must be an integer') + } + } + + if (parentTagsOnly && childTagsOnly) { + throw new Error('Cannot set both Parent Tags Only and Child Tags Only to true - they are mutually exclusive') + } + + // Get environment variables + const supabaseUrl = process.env.AAI_DATASTORE_SUPABASE_URL + const supabaseKey = process.env.AAI_DATASTORE_SUPABASE_SERVICE_ROLE_KEY + + if (!supabaseUrl || !supabaseKey) { + throw new Error( + 'AAI Datastore configuration missing. Please set AAI_DATASTORE_SUPABASE_URL and AAI_DATASTORE_SUPABASE_SERVICE_ROLE_KEY environment variables.' + ) + } + + const loaderOptions: AAITagsLoaderParams = { + supabaseUrl, + supabaseKey, + limit: limit || 1000, + searchTerm: searchTerm || null, + parentTagsOnly: parentTagsOnly || false, + childTagsOnly: childTagsOnly || false, + includeParentInfo: includeParentInfo !== false // Default true + } + + const loader = new AAITagsLoader(loaderOptions) + + let docs: IDocument[] = [] + + if (textSplitter) { + docs = await loader.load() + docs = await textSplitter.splitDocuments(docs) + } else { + docs = await loader.load() + } + + // Apply metadata + if (metadata) { + const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) + docs = docs.map((doc) => ({ + ...doc, + metadata: + _omitMetadataKeys === '*' + ? { + ...parsedMetadata + } + : omit( + { + ...doc.metadata, + ...parsedMetadata + }, + omitMetadataKeys + ) + })) + } else { + docs = docs.map((doc) => ({ + ...doc, + metadata: + _omitMetadataKeys === '*' + ? {} + : omit( + { + ...doc.metadata + }, + omitMetadataKeys + ) + })) + } + + if (output === 'text') { + let finalText = '' + for (const doc of docs) { + finalText += `${doc.pageContent}\n` + } + return handleEscapeCharacters(finalText, false) + } + + return docs + } +} + +interface AAITagsLoaderParams { + supabaseUrl: string + supabaseKey: string + limit: number + searchTerm: string | null + parentTagsOnly: boolean + childTagsOnly: boolean + includeParentInfo: boolean +} + +class AAITagsLoader extends BaseDocumentLoader { + private supabaseUrl: string + private supabaseKey: string + private limit: number + private searchTerm: string | null + private parentTagsOnly: boolean + private childTagsOnly: boolean + private includeParentInfo: boolean + + constructor(params: AAITagsLoaderParams) { + super() + this.supabaseUrl = params.supabaseUrl + this.supabaseKey = params.supabaseKey + this.limit = params.limit + this.searchTerm = params.searchTerm + this.parentTagsOnly = params.parentTagsOnly + this.childTagsOnly = params.childTagsOnly + this.includeParentInfo = params.includeParentInfo + } + + public async load(): Promise { + const supabase: SupabaseClient = createClient(this.supabaseUrl, this.supabaseKey) + + // Build query + let query = supabase.from('tags').select( + this.includeParentInfo + ? ` + id, + slug, + label, + description, + color, + shade, + parent_id, + created_at, + updated_at, + parent:parent_id ( + id, + slug, + label, + description, + color, + shade + ) + ` + : '*' + ) + + // Apply filters + if (this.parentTagsOnly) { + query = query.is('parent_id', null) + } + + if (this.childTagsOnly) { + query = query.not('parent_id', 'is', null) + } + + if (this.searchTerm) { + query = query.or(`slug.ilike.%${this.searchTerm}%,label.ilike.%${this.searchTerm}%,description.ilike.%${this.searchTerm}%`) + } + + // Apply limit and order + query = query.order('label', { ascending: true }).limit(this.limit) + + const { data: tags, error } = await query + + if (error) { + throw new Error(`Failed to fetch tags from AAI Datastore: ${error.message}`) + } + + if (!tags || tags.length === 0) { + return [] + } + + return tags.map((tag) => this.createDocumentFromTag(tag)) + } + + private createDocumentFromTag(tag: any): Document { + // Create page content from tag information + const pageContent = [ + `Tag: ${tag.label} (${tag.slug})`, + tag.description ? `Description: ${tag.description}` : '', + tag.parent ? `Parent: ${tag.parent.label} (${tag.parent.slug})` : '' + ] + .filter(Boolean) + .join('\n') + + // Build comprehensive metadata + const metadata: ICommonObject = { + // Core fields + id: tag.id, + slug: tag.slug, + label: tag.label, + description: tag.description, + parent_id: tag.parent_id, + + // Parent information (if available) + parent_slug: tag.parent?.slug, + parent_label: tag.parent?.label, + parent_description: tag.parent?.description, + + // Hierarchy information + is_parent_tag: tag.parent_id === null, + is_child_tag: tag.parent_id !== null, + + // Timestamps (ISO strings) + created_at: tag.created_at, + updated_at: tag.updated_at, + + // Timestamps (Unix timestamps in seconds for Pinecone numeric filters) + created_at_timestamp: tag.created_at ? Math.floor(new Date(tag.created_at).getTime() / 1000) : null, + updated_at_timestamp: tag.updated_at ? Math.floor(new Date(tag.updated_at).getTime() / 1000) : null, + + // Source + source: 'aai_datastore', + source_type: 'tag' + } + + // Remove undefined/null values + Object.keys(metadata).forEach((key) => { + if (metadata[key] === undefined || metadata[key] === null) { + delete metadata[key] + } + }) + + return new Document({ + pageContent, + metadata + }) + } +} + +module.exports = { + nodeClass: AAITags_DocumentLoaders +} diff --git a/packages/components/nodes/documentloaders/AAITags/answerai-square-black.png b/packages/components/nodes/documentloaders/AAITags/answerai-square-black.png new file mode 100644 index 00000000000..24dfd99f98c Binary files /dev/null and b/packages/components/nodes/documentloaders/AAITags/answerai-square-black.png differ diff --git a/packages/components/nodes/documentloaders/AAITranscripts/AAITranscripts.ts b/packages/components/nodes/documentloaders/AAITranscripts/AAITranscripts.ts new file mode 100644 index 00000000000..ed53c7891d7 --- /dev/null +++ b/packages/components/nodes/documentloaders/AAITranscripts/AAITranscripts.ts @@ -0,0 +1,533 @@ +import { omit } from 'lodash' +import { Document } from '@langchain/core/documents' +import { TextSplitter } from 'langchain/text_splitter' +import { BaseDocumentLoader } from 'langchain/document_loaders/base' +import { IDocument, ICommonObject, INode, INodeData, INodeParams, INodeOutputsValue } from '../../../src/Interface' +import { handleEscapeCharacters } from '../../../src' +import { createClient, SupabaseClient } from '@supabase/supabase-js' + +class AAITranscripts_DocumentLoaders implements INode { + label: string + name: string + version: number + description: string + type: string + icon: string + category: string + tags: string[] + baseClasses: string[] + inputs: INodeParams[] + outputs: INodeOutputsValue[] + + constructor() { + this.label = 'AnswerAgentAI Transcripts' + this.name = 'aaiTranscripts' + this.version = 1.0 + this.type = 'Document' + this.icon = 'answerai-square-black.png' + this.category = 'Document Loaders' + this.description = 'Load call transcripts from AnswerAgentAI Datastore' + this.tags = ['AAI'] + this.baseClasses = [this.type] + this.inputs = [ + { + label: 'Text Splitter', + name: 'textSplitter', + type: 'TextSplitter', + optional: true + }, + { + label: 'Limit', + name: 'limit', + type: 'number', + default: 100, + description: 'Maximum number of transcripts to load', + optional: true + }, + { + label: 'Search Term', + name: 'searchTerm', + type: 'string', + description: 'Search in transcript and summary', + optional: true, + additionalParams: true + }, + { + label: 'Date From', + name: 'dateFrom', + type: 'string', + placeholder: '2025-01-01', + description: 'Filter calls from this date (ISO format: YYYY-MM-DD)', + optional: true, + additionalParams: true + }, + { + label: 'Date To', + name: 'dateTo', + type: 'string', + placeholder: '2025-12-31', + description: 'Filter calls to this date (ISO format: YYYY-MM-DD)', + optional: true, + additionalParams: true + }, + { + label: 'Include Tags', + name: 'includeTags', + type: 'string', + placeholder: 'positive_sentiment,product_inquiry', + description: 'Comma-separated list of tag slugs to include', + optional: true, + additionalParams: true + }, + { + label: 'Include Tags Logic', + name: 'includeTagsLogic', + type: 'options', + options: [ + { + label: 'OR (any tag matches)', + name: 'OR' + }, + { + label: 'AND (all tags must match)', + name: 'AND' + } + ], + default: 'OR', + optional: true, + additionalParams: true + }, + { + label: 'Exclude Tags', + name: 'excludeTags', + type: 'string', + placeholder: 'spam,test_call', + description: 'Comma-separated list of tag slugs to exclude', + optional: true, + additionalParams: true + }, + { + label: 'Has Analysis', + name: 'hasAnalysis', + type: 'options', + options: [ + { + label: 'All', + name: 'all' + }, + { + label: 'Analyzed Only', + name: 'analyzed' + }, + { + label: 'Not Analyzed', + name: 'not_analyzed' + } + ], + default: 'all', + optional: true, + additionalParams: true + }, + { + label: 'Sentiment Min', + name: 'sentimentMin', + type: 'number', + placeholder: '1', + description: 'Minimum sentiment score (1-10)', + optional: true, + additionalParams: true + }, + { + label: 'Sentiment Max', + name: 'sentimentMax', + type: 'number', + placeholder: '10', + description: 'Maximum sentiment score (1-10)', + optional: true, + additionalParams: true + }, + { + label: 'Additional Metadata', + name: 'metadata', + type: 'json', + description: 'Additional metadata to be added to the extracted documents', + optional: true, + additionalParams: true + }, + { + label: 'Omit Metadata Keys', + name: 'omitMetadataKeys', + type: 'string', + rows: 4, + description: + 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field', + placeholder: 'key1, key2, key3.nestedKey1', + optional: true, + additionalParams: true + } + ] + this.outputs = [ + { + label: 'Document', + name: 'document', + description: 'Array of document objects containing metadata and pageContent', + baseClasses: [...this.baseClasses, 'json'] + }, + { + label: 'Text', + name: 'text', + description: 'Concatenated string from pageContent of documents', + baseClasses: ['string', 'json'] + } + ] + } + + async init(nodeData: INodeData): Promise { + const textSplitter = nodeData.inputs?.textSplitter as TextSplitter + const limit = nodeData.inputs?.limit as number + const searchTerm = nodeData.inputs?.searchTerm as string + const dateFrom = nodeData.inputs?.dateFrom as string + const dateTo = nodeData.inputs?.dateTo as string + const includeTags = nodeData.inputs?.includeTags as string + const includeTagsLogic = (nodeData.inputs?.includeTagsLogic as string) || 'OR' + const excludeTags = nodeData.inputs?.excludeTags as string + const hasAnalysis = (nodeData.inputs?.hasAnalysis as string) || 'all' + const sentimentMin = nodeData.inputs?.sentimentMin as number + const sentimentMax = nodeData.inputs?.sentimentMax as number + const metadata = nodeData.inputs?.metadata + const _omitMetadataKeys = nodeData.inputs?.omitMetadataKeys as string + const output = nodeData.outputs?.output as string + + let omitMetadataKeys: string[] = [] + if (_omitMetadataKeys) { + omitMetadataKeys = _omitMetadataKeys.split(',').map((key) => key.trim()) + } + + // Validate inputs + if (limit !== undefined && limit !== null) { + if (limit <= 0) { + throw new Error('Limit must be a positive number') + } + if (!Number.isInteger(limit)) { + throw new Error('Limit must be an integer') + } + } + + if (includeTagsLogic && !['OR', 'AND'].includes(includeTagsLogic)) { + throw new Error('Include Tags Logic must be either "OR" or "AND"') + } + + if (dateFrom) { + const datePattern = /^\d{4}-\d{2}-\d{2}$/ + if (!datePattern.test(dateFrom)) { + throw new Error('Date From must be in ISO format (YYYY-MM-DD)') + } + const parsedDate = new Date(dateFrom) + if (isNaN(parsedDate.getTime())) { + throw new Error('Date From is not a valid date') + } + } + + if (dateTo) { + const datePattern = /^\d{4}-\d{2}-\d{2}$/ + if (!datePattern.test(dateTo)) { + throw new Error('Date To must be in ISO format (YYYY-MM-DD)') + } + const parsedDate = new Date(dateTo) + if (isNaN(parsedDate.getTime())) { + throw new Error('Date To is not a valid date') + } + } + + if (dateFrom && dateTo) { + const fromDate = new Date(dateFrom) + const toDate = new Date(dateTo) + if (fromDate > toDate) { + throw new Error('Date From must be before or equal to Date To') + } + } + + if (sentimentMin !== undefined && sentimentMin !== null) { + if (typeof sentimentMin !== 'number' || isNaN(sentimentMin)) { + throw new Error('Sentiment Min must be a number') + } + if (sentimentMin < 1 || sentimentMin > 10) { + throw new Error('Sentiment Min must be between 1 and 10') + } + } + + if (sentimentMax !== undefined && sentimentMax !== null) { + if (typeof sentimentMax !== 'number' || isNaN(sentimentMax)) { + throw new Error('Sentiment Max must be a number') + } + if (sentimentMax < 1 || sentimentMax > 10) { + throw new Error('Sentiment Max must be between 1 and 10') + } + } + + if ( + sentimentMin !== undefined && + sentimentMin !== null && + sentimentMax !== undefined && + sentimentMax !== null && + sentimentMin > sentimentMax + ) { + throw new Error('Sentiment Min cannot be greater than Sentiment Max') + } + + if (hasAnalysis && !['all', 'analyzed', 'not_analyzed'].includes(hasAnalysis)) { + throw new Error('Has Analysis must be one of: "all", "analyzed", "not_analyzed"') + } + + // Get environment variables + const supabaseUrl = process.env.AAI_DATASTORE_SUPABASE_URL + const supabaseKey = process.env.AAI_DATASTORE_SUPABASE_SERVICE_ROLE_KEY + + if (!supabaseUrl || !supabaseKey) { + throw new Error( + 'AAI Datastore configuration missing. Please set AAI_DATASTORE_SUPABASE_URL and AAI_DATASTORE_SUPABASE_SERVICE_ROLE_KEY environment variables.' + ) + } + + const loaderOptions: AAITranscriptsLoaderParams = { + supabaseUrl, + supabaseKey, + limit: limit || 100, + searchTerm: searchTerm || null, + dateFrom: dateFrom || null, + dateTo: dateTo || null, + includeTags: includeTags ? includeTags.split(',').map((t) => t.trim()) : [], + includeTagsLogic, + excludeTags: excludeTags ? excludeTags.split(',').map((t) => t.trim()) : [], + hasAnalysis, + sentimentMin: sentimentMin || null, + sentimentMax: sentimentMax || null + } + + const loader = new AAITranscriptsLoader(loaderOptions) + + let docs: IDocument[] = [] + + if (textSplitter) { + docs = await loader.load() + docs = await textSplitter.splitDocuments(docs) + } else { + docs = await loader.load() + } + + // Apply metadata + if (metadata) { + const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) + docs = docs.map((doc) => ({ + ...doc, + metadata: + _omitMetadataKeys === '*' + ? { + ...parsedMetadata + } + : omit( + { + ...doc.metadata, + ...parsedMetadata + }, + omitMetadataKeys + ) + })) + } else { + docs = docs.map((doc) => ({ + ...doc, + metadata: + _omitMetadataKeys === '*' + ? {} + : omit( + { + ...doc.metadata + }, + omitMetadataKeys + ) + })) + } + + if (output === 'text') { + let finalText = '' + for (const doc of docs) { + finalText += `${doc.pageContent}\n` + } + return handleEscapeCharacters(finalText, false) + } + + return docs + } +} + +interface AAITranscriptsLoaderParams { + supabaseUrl: string + supabaseKey: string + limit: number + searchTerm: string | null + dateFrom: string | null + dateTo: string | null + includeTags: string[] + includeTagsLogic: string + excludeTags: string[] + hasAnalysis: string + sentimentMin: number | null + sentimentMax: number | null +} + +class AAITranscriptsLoader extends BaseDocumentLoader { + private supabaseUrl: string + private supabaseKey: string + private limit: number + private searchTerm: string | null + private dateFrom: string | null + private dateTo: string | null + private includeTags: string[] + private includeTagsLogic: string + private excludeTags: string[] + private hasAnalysis: string + private sentimentMin: number | null + private sentimentMax: number | null + + constructor(params: AAITranscriptsLoaderParams) { + super() + this.supabaseUrl = params.supabaseUrl + this.supabaseKey = params.supabaseKey + this.limit = params.limit + this.searchTerm = params.searchTerm + this.dateFrom = params.dateFrom + this.dateTo = params.dateTo + this.includeTags = params.includeTags + this.includeTagsLogic = params.includeTagsLogic + this.excludeTags = params.excludeTags + this.hasAnalysis = params.hasAnalysis + this.sentimentMin = params.sentimentMin + this.sentimentMax = params.sentimentMax + } + + public async load(): Promise { + const supabase: SupabaseClient = createClient(this.supabaseUrl, this.supabaseKey) + + // Calculate pagination + const pageSize = Math.min(this.limit, 1000) // Max 1000 per request + const totalPages = Math.ceil(this.limit / pageSize) + + let allCalls: any[] = [] + + for (let page = 0; page < totalPages; page++) { + const remainingItems = this.limit - allCalls.length + const currentPageSize = Math.min(pageSize, remainingItems) + + const { data, error } = await supabase.rpc('filter_calls_v1', { + page, + page_size: currentPageSize, + search_term: this.searchTerm, + include_tags: { + tags: this.includeTags, + logic: this.includeTagsLogic + }, + exclude_tags: { + tags: this.excludeTags, + logic: 'OR' + }, + date_from: this.dateFrom, + date_to: this.dateTo, + has_analysis: this.hasAnalysis, + sentiment_min: this.sentimentMin, + sentiment_max: this.sentimentMax, + custom_filters: {} + }) + + if (error) { + throw new Error(`Failed to fetch transcripts from AAI Datastore: ${error.message}`) + } + + if (!data || !data.calls) { + break + } + + allCalls.push(...data.calls) + + // Stop if we've fetched enough or no more results + if (allCalls.length >= this.limit || data.calls.length < currentPageSize) { + break + } + } + + // Truncate to exact limit + if (allCalls.length > this.limit) { + allCalls = allCalls.slice(0, this.limit) + } + + return allCalls.map((call) => this.createDocumentFromCall(call)) + } + + private createDocumentFromCall(call: any): Document { + // Extract custom data fields + const customData = call.custom_data || {} + const aiAnalysis = call.ai_analysis || {} + + // Create page content from transcript or summary + const pageContent = call.transcript || call.summary || '' + + // Build comprehensive metadata + const metadata: ICommonObject = { + // Core fields + id: call.id, + recording_url: call.recording_url, + duration: call.duration, + call_datetime: call.call_datetime, + sentiment_score: call.sentiment_score, + + // Custom data fields + employee_name: customData.employee_name, + employee_id: customData.employee_id, + caller_name: customData.caller_name, + call_number: customData.call_number, + call_type: customData.call_type, + answered_by: customData.answered_by, + escalated: customData.escalated, + resolution_status: customData.resolution_status, + + // AI analysis fields + confidence_score: aiAnalysis.confidence_score, + analysis_version: aiAnalysis.analysis_version, + + // Tags + tags: call.tags || [], + tag_slugs: (call.tags || []).map((t: any) => t.slug).filter(Boolean), + tag_labels: (call.tags || []).map((t: any) => t.label).filter(Boolean), + + // Timestamps (ISO strings) + created_at: call.created_at, + updated_at: call.updated_at, + last_analyzed_at: call.last_analyzed_at, + + // Timestamps (Unix timestamps in seconds for Pinecone numeric filters) + call_datetime_timestamp: call.call_datetime ? Math.floor(new Date(call.call_datetime).getTime() / 1000) : null, + created_at_timestamp: call.created_at ? Math.floor(new Date(call.created_at).getTime() / 1000) : null, + updated_at_timestamp: call.updated_at ? Math.floor(new Date(call.updated_at).getTime() / 1000) : null, + last_analyzed_at_timestamp: call.last_analyzed_at ? Math.floor(new Date(call.last_analyzed_at).getTime() / 1000) : null, + + // Source + source: 'aai_datastore', + source_type: 'transcript' + } + + // Remove undefined/null values + Object.keys(metadata).forEach((key) => { + if (metadata[key] === undefined || metadata[key] === null) { + delete metadata[key] + } + }) + + return new Document({ + pageContent, + metadata + }) + } +} + +module.exports = { + nodeClass: AAITranscripts_DocumentLoaders +} diff --git a/packages/components/nodes/documentloaders/AAITranscripts/answerai-square-black.png b/packages/components/nodes/documentloaders/AAITranscripts/answerai-square-black.png new file mode 100644 index 00000000000..24dfd99f98c Binary files /dev/null and b/packages/components/nodes/documentloaders/AAITranscripts/answerai-square-black.png differ diff --git a/packages/components/nodes/documentloaders/AAIUrls/AAIUrls.ts b/packages/components/nodes/documentloaders/AAIUrls/AAIUrls.ts new file mode 100644 index 00000000000..80f31a7db08 --- /dev/null +++ b/packages/components/nodes/documentloaders/AAIUrls/AAIUrls.ts @@ -0,0 +1,739 @@ +import { omit } from 'lodash' +import { Document } from '@langchain/core/documents' +import { TextSplitter } from 'langchain/text_splitter' +import { BaseDocumentLoader } from 'langchain/document_loaders/base' +import { IDocument, ICommonObject, INode, INodeData, INodeParams, INodeOutputsValue } from '../../../src/Interface' +import { handleEscapeCharacters } from '../../../src' +import { createClient, SupabaseClient } from '@supabase/supabase-js' + +class AAIUrls_DocumentLoaders implements INode { + label: string + name: string + version: number + description: string + type: string + icon: string + category: string + tags: string[] + baseClasses: string[] + inputs: INodeParams[] + outputs: INodeOutputsValue[] + + constructor() { + this.label = 'AnswerAgentAI URLs' + this.name = 'aaiUrls' + this.version = 1.0 + this.type = 'Document' + this.icon = 'answerai-square-black.png' + this.category = 'Document Loaders' + this.description = 'Load URL/page information from AnswerAgentAI Datastore' + this.tags = ['AAI'] + this.baseClasses = [this.type] + this.inputs = [ + { + label: 'Text Splitter', + name: 'textSplitter', + type: 'TextSplitter', + optional: true + }, + { + label: 'Limit', + name: 'limit', + type: 'number', + default: 100, + description: 'Maximum number of URLs to load', + optional: true + }, + { + label: 'Search Term', + name: 'searchTerm', + type: 'string', + description: 'Search in URL, domain name, page title, and description', + optional: true, + additionalParams: true + }, + { + label: 'Include Tags', + name: 'includeTags', + type: 'string', + placeholder: 'blog,product_page', + description: 'Comma-separated list of tag slugs to include', + optional: true, + additionalParams: true + }, + { + label: 'Include Tags Logic', + name: 'includeTagsLogic', + type: 'options', + options: [ + { + label: 'OR (any tag matches)', + name: 'OR' + }, + { + label: 'AND (all tags must match)', + name: 'AND' + } + ], + default: 'OR', + optional: true, + additionalParams: true + }, + { + label: 'Exclude Tags', + name: 'excludeTags', + type: 'string', + placeholder: 'error_page,404', + description: 'Comma-separated list of tag slugs to exclude', + optional: true, + additionalParams: true + }, + { + label: 'HTTP Status Filter', + name: 'statusFilter', + type: 'string', + placeholder: 'success,redirect', + description: + 'Comma-separated list of status categories (success, redirect, client_error, server_error, unknown) or specific codes (200, 404, etc.)', + optional: true, + additionalParams: true + }, + { + label: 'Has Analysis', + name: 'hasAnalysis', + type: 'options', + options: [ + { + label: 'All', + name: 'all' + }, + { + label: 'Analyzed Only', + name: 'analyzed' + }, + { + label: 'Not Analyzed', + name: 'not_analyzed' + } + ], + default: 'all', + optional: true, + additionalParams: true + }, + { + label: 'Include AI Analysis in Content', + name: 'includeAiAnalysis', + type: 'boolean', + default: true, + description: 'Include full AI analysis markdown in page content', + optional: true, + additionalParams: true + }, + { + label: 'Additional Metadata', + name: 'metadata', + type: 'json', + description: 'Additional metadata to be added to the extracted documents', + optional: true, + additionalParams: true + }, + { + label: 'Omit Metadata Keys', + name: 'omitMetadataKeys', + type: 'string', + rows: 4, + description: + 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field', + placeholder: 'key1, key2, key3.nestedKey1', + optional: true, + additionalParams: true + } + ] + this.outputs = [ + { + label: 'Document', + name: 'document', + description: 'Array of document objects containing metadata and pageContent', + baseClasses: [...this.baseClasses, 'json'] + }, + { + label: 'Text', + name: 'text', + description: 'Concatenated string from pageContent of documents', + baseClasses: ['string', 'json'] + } + ] + } + + async init(nodeData: INodeData): Promise { + const textSplitter = nodeData.inputs?.textSplitter as TextSplitter + const limit = nodeData.inputs?.limit as number + const searchTerm = nodeData.inputs?.searchTerm as string + const includeTags = nodeData.inputs?.includeTags as string + const includeTagsLogic = (nodeData.inputs?.includeTagsLogic as string) || 'OR' + const excludeTags = nodeData.inputs?.excludeTags as string + const statusFilter = nodeData.inputs?.statusFilter as string + const hasAnalysis = (nodeData.inputs?.hasAnalysis as string) || 'all' + const includeAiAnalysis = nodeData.inputs?.includeAiAnalysis !== false + const metadata = nodeData.inputs?.metadata + const _omitMetadataKeys = nodeData.inputs?.omitMetadataKeys as string + const output = nodeData.outputs?.output as string + + let omitMetadataKeys: string[] = [] + if (_omitMetadataKeys) { + omitMetadataKeys = _omitMetadataKeys.split(',').map((key) => key.trim()) + } + + // Validate inputs + if (limit !== undefined && limit !== null) { + if (limit <= 0) { + throw new Error('Limit must be a positive number') + } + if (!Number.isInteger(limit)) { + throw new Error('Limit must be an integer') + } + } + + if (includeTagsLogic && !['OR', 'AND'].includes(includeTagsLogic)) { + throw new Error('Include Tags Logic must be either "OR" or "AND"') + } + + if (statusFilter) { + const validCategories = ['success', 'redirect', 'client_error', 'server_error', 'informational', 'unknown'] + const statusFilters = statusFilter.split(',').map((s) => s.trim()) + for (const filter of statusFilters) { + // Check if it's a valid category or a valid HTTP status code (100-599) + const isCategory = validCategories.includes(filter) + const isNumeric = /^\d+$/.test(filter) + const statusCode = isNumeric ? parseInt(filter, 10) : null + + if (!isCategory && (!isNumeric || statusCode === null || statusCode < 100 || statusCode > 599)) { + throw new Error( + `Invalid status filter "${filter}". Must be one of: ${validCategories.join(', ')}, or a valid HTTP status code (100-599)` + ) + } + } + } + + if (hasAnalysis && !['all', 'analyzed', 'not_analyzed'].includes(hasAnalysis)) { + throw new Error('Has Analysis must be one of: "all", "analyzed", "not_analyzed"') + } + + // Get environment variables + const supabaseUrl = process.env.AAI_DATASTORE_SUPABASE_URL + const supabaseKey = process.env.AAI_DATASTORE_SUPABASE_SERVICE_ROLE_KEY + + if (!supabaseUrl || !supabaseKey) { + throw new Error( + 'AAI Datastore configuration missing. Please set AAI_DATASTORE_SUPABASE_URL and AAI_DATASTORE_SUPABASE_SERVICE_ROLE_KEY environment variables.' + ) + } + + const loaderOptions: AAIUrlsLoaderParams = { + supabaseUrl, + supabaseKey, + limit: limit || 100, + searchTerm: searchTerm || null, + includeTags: includeTags ? includeTags.split(',').map((t) => t.trim()) : [], + includeTagsLogic, + excludeTags: excludeTags ? excludeTags.split(',').map((t) => t.trim()) : [], + statusFilter: statusFilter ? statusFilter.split(',').map((s) => s.trim()) : [], + hasAnalysis, + includeAiAnalysis + } + + const loader = new AAIUrlsLoader(loaderOptions) + + let docs: IDocument[] = [] + + if (textSplitter) { + docs = await loader.load() + docs = await textSplitter.splitDocuments(docs) + } else { + docs = await loader.load() + } + + // Apply metadata + if (metadata) { + const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) + docs = docs.map((doc) => ({ + ...doc, + metadata: + _omitMetadataKeys === '*' + ? { + ...parsedMetadata + } + : omit( + { + ...doc.metadata, + ...parsedMetadata + }, + omitMetadataKeys + ) + })) + } else { + docs = docs.map((doc) => ({ + ...doc, + metadata: + _omitMetadataKeys === '*' + ? {} + : omit( + { + ...doc.metadata + }, + omitMetadataKeys + ) + })) + } + + if (output === 'text') { + let finalText = '' + for (const doc of docs) { + finalText += `${doc.pageContent}\n` + } + return handleEscapeCharacters(finalText, false) + } + + return docs + } +} + +interface AAIUrlsLoaderParams { + supabaseUrl: string + supabaseKey: string + limit: number + searchTerm: string | null + includeTags: string[] + includeTagsLogic: string + excludeTags: string[] + statusFilter: string[] + hasAnalysis: string + includeAiAnalysis: boolean +} + +class AAIUrlsLoader extends BaseDocumentLoader { + private supabaseUrl: string + private supabaseKey: string + private limit: number + private searchTerm: string | null + private includeTags: string[] + private includeTagsLogic: string + private excludeTags: string[] + private statusFilter: string[] + private hasAnalysis: string + private includeAiAnalysis: boolean + + constructor(params: AAIUrlsLoaderParams) { + super() + this.supabaseUrl = params.supabaseUrl + this.supabaseKey = params.supabaseKey + this.limit = params.limit + this.searchTerm = params.searchTerm + this.includeTags = params.includeTags + this.includeTagsLogic = params.includeTagsLogic + this.excludeTags = params.excludeTags + this.statusFilter = params.statusFilter + this.hasAnalysis = params.hasAnalysis + this.includeAiAnalysis = params.includeAiAnalysis + } + + public async load(): Promise { + const supabase: SupabaseClient = createClient(this.supabaseUrl, this.supabaseKey, { + global: { + fetch: (...args) => { + const [resource, config] = args + const controller = new AbortController() + const timeoutId = setTimeout(() => controller.abort(), 60000) + + return fetch(resource, { + ...config, + signal: controller.signal + }).finally(() => clearTimeout(timeoutId)) + } + } + }) + + // Use larger page size since we're only selecting essential fields + const pageSize = Math.min(this.limit, 100) + let allUrls: any[] = [] + let currentPage = 0 + + console.log('[AAIUrls] Starting load with params:', { + limit: this.limit, + searchTerm: this.searchTerm, + includeTags: this.includeTags, + statusFilter: this.statusFilter, + hasAnalysis: this.hasAnalysis + }) + + while (allUrls.length < this.limit) { + const remainingItems = this.limit - allUrls.length + const currentPageSize = Math.min(pageSize, remainingItems) + + console.log(`[AAIUrls] Fetching page ${currentPage}, size ${currentPageSize}`) + + // Retry logic with exponential backoff + let retryCount = 0 + const maxRetries = 3 + let lastError: any = null + let shouldStopPagination = false + + while (retryCount < maxRetries) { + try { + // Build query with only essential fields (exclude heavy JSONB history fields) + // NOTE: We explicitly exclude: ai_analysis_overrides, ai_analysis_history, + // override_metadata, source_metadata which can be multi-MB per record + let query = supabase + .from('urls') + .select( + ` + id, + url, + domain_name, + domain_id, + http_status, + status_text, + content_type, + page_title, + meta_description, + canonical_url, + custom_data, + ai_analysis, + created_at, + updated_at, + processed_at, + first_seen_at, + last_crawled_at, + last_analyzed_at, + analysis_version, + url_tags( + tag_id, + tags(id, slug, label, color, parent_id) + ) + ` + ) + .order('updated_at', { ascending: false }) + .range(currentPage * pageSize, (currentPage + 1) * pageSize - 1) + + // Apply filters + if (this.searchTerm) { + query = query.or( + `url.ilike.%${this.searchTerm}%,domain_name.ilike.%${this.searchTerm}%,page_title.ilike.%${this.searchTerm}%,meta_description.ilike.%${this.searchTerm}%` + ) + } + + if (this.hasAnalysis === 'analyzed') { + query = query.not('ai_analysis', 'is', null) + } else if (this.hasAnalysis === 'not_analyzed') { + query = query.is('ai_analysis', null) + } + + // Note: HTTP status filtering is applied in-memory after fetching (see filterByHttpStatus) + + const { data, error } = await query + + if (error) { + console.error('[AAIUrls] Query error:', error) + throw new Error(`Failed to fetch URLs from AAI Datastore: ${error.message}`) + } + + console.log(`[AAIUrls] Page ${currentPage} response:`, { + hasData: !!data, + urlsCount: data?.length || 0 + }) + + if (!data || data.length === 0) { + console.log('[AAIUrls] No more data, stopping pagination') + shouldStopPagination = true + break + } + + // Transform url_tags array to flat tags array + const urlsWithTags = data.map((url: any) => ({ + ...url, + tags: url.url_tags?.map((ut: any) => ut.tags).filter(Boolean) || [] + })) + + // Apply tag filtering if specified + let filteredUrls = urlsWithTags + if (this.includeTags.length > 0 || this.excludeTags.length > 0) { + filteredUrls = this.filterByTags(filteredUrls) + } + + // Apply HTTP status filtering if specified + if (this.statusFilter.length > 0) { + filteredUrls = this.filterByHttpStatus(filteredUrls) + } + + allUrls.push(...filteredUrls) + currentPage++ + + // Stop if we've fetched enough + if (allUrls.length >= this.limit || data.length < currentPageSize) { + shouldStopPagination = true + break + } + + // Success - break out of retry loop + break + } catch (error: any) { + lastError = error + retryCount++ + + if (retryCount < maxRetries) { + const delayMs = Math.min(1000 * Math.pow(2, retryCount), 10000) + console.warn( + `[AAIUrls] Request failed (attempt ${retryCount}/${maxRetries}), retrying in ${delayMs}ms...`, + error.message + ) + await new Promise((resolve) => setTimeout(resolve, delayMs)) + } else { + console.error(`[AAIUrls] All ${maxRetries} retry attempts failed`) + throw lastError + } + } + } + + // If we got no data, hit the limit, or all retries failed, break out of pagination loop + if (shouldStopPagination || retryCount >= maxRetries) { + break + } + + // Add small delay between successful pages to avoid overwhelming the server + await new Promise((resolve) => setTimeout(resolve, 200)) + } + + console.log(`[AAIUrls] Load complete. Total URLs: ${allUrls.length}`) + + // Truncate to exact limit + if (allUrls.length > this.limit) { + allUrls = allUrls.slice(0, this.limit) + } + + return allUrls.map((url) => this.createDocumentFromUrl(url)) + } + + private filterByTags(urls: any[]): any[] { + return urls.filter((url) => { + const urlTagSlugs = url.tags?.map((t: any) => t.slug) || [] + + // Apply include tags filter + if (this.includeTags.length > 0) { + if (this.includeTagsLogic === 'AND') { + const hasAllTags = this.includeTags.every((tag) => urlTagSlugs.includes(tag)) + if (!hasAllTags) return false + } else { + const hasAnyTag = this.includeTags.some((tag) => urlTagSlugs.includes(tag)) + if (!hasAnyTag) return false + } + } + + // Apply exclude tags filter + if (this.excludeTags.length > 0) { + const hasExcludedTag = this.excludeTags.some((tag) => urlTagSlugs.includes(tag)) + if (hasExcludedTag) return false + } + + return true + }) + } + + private filterByHttpStatus(urls: any[]): any[] { + return urls.filter((url) => { + const status = url.http_status + + return this.statusFilter.some((filter) => { + if (filter === 'success') return status >= 200 && status < 300 + if (filter === 'redirect') return status >= 300 && status < 400 + if (filter === 'client_error') return status >= 400 && status < 500 + if (filter === 'server_error') return status >= 500 && status < 600 + if (filter === 'informational') return status >= 100 && status < 200 + if (filter === 'unknown') return status === null || status === undefined + // Specific status code + return status === parseInt(filter, 10) + }) + }) + } + + private formatAiAnalysisAsMarkdown(aiAnalysis: any): string { + if (!aiAnalysis || Object.keys(aiAnalysis).length === 0) { + return '' + } + + const sections: string[] = ['## AI Analysis', ''] + + // Summary + if (aiAnalysis.summary) { + sections.push('### Summary', aiAnalysis.summary, '') + } + + // Content Summary + if (aiAnalysis.content_summary) { + sections.push('### Content Summary', aiAnalysis.content_summary, '') + } + + // Category/Type + if (aiAnalysis.category || aiAnalysis.content_type) { + sections.push('### Classification', '') + if (aiAnalysis.category) { + sections.push(`**Category:** ${aiAnalysis.category}`) + } + if (aiAnalysis.content_type) { + sections.push(`**Content Type:** ${aiAnalysis.content_type}`) + } + sections.push('') + } + + // Topics + if (aiAnalysis.topics) { + sections.push('### Topics', '') + if (Array.isArray(aiAnalysis.topics)) { + aiAnalysis.topics.forEach((topic: string) => { + sections.push(`- ${topic}`) + }) + } else if (typeof aiAnalysis.topics === 'string') { + sections.push(aiAnalysis.topics) + } + sections.push('') + } + + // Key Points/Insights + if (aiAnalysis.key_points || aiAnalysis.key_insights) { + sections.push('### Key Points', '') + const points = aiAnalysis.key_points || aiAnalysis.key_insights + if (Array.isArray(points)) { + points.forEach((point: string) => { + sections.push(`- ${point}`) + }) + } + sections.push('') + } + + // Confidence Score + if (aiAnalysis.confidence_score) { + sections.push(`**Confidence Score:** ${aiAnalysis.confidence_score}`, '') + } + + // Additional fields (catch-all) + const knownFields = [ + 'summary', + 'content_summary', + 'category', + 'content_type', + 'topics', + 'key_points', + 'key_insights', + 'confidence_score', + 'analysis_version' + ] + const additionalFields = Object.keys(aiAnalysis).filter((key) => !knownFields.includes(key)) + + if (additionalFields.length > 0) { + sections.push('### Additional Information', '') + additionalFields.forEach((key) => { + const value = aiAnalysis[key] + if (value && typeof value !== 'object') { + sections.push(`**${key.replace(/_/g, ' ').replace(/\b\w/g, (l) => l.toUpperCase())}:** ${value}`) + } else if (value && typeof value === 'object') { + sections.push(`**${key.replace(/_/g, ' ').replace(/\b\w/g, (l) => l.toUpperCase())}:**`) + sections.push('```json') + sections.push(JSON.stringify(value, null, 2)) + sections.push('```') + } + }) + } + + return sections.filter(Boolean).join('\n') + } + + private createDocumentFromUrl(url: any): Document { + // Extract custom data and AI analysis + const customData = url.custom_data || {} + const aiAnalysis = url.ai_analysis || {} + + // Format AI analysis as markdown if requested + const aiAnalysisMarkdown = this.includeAiAnalysis ? this.formatAiAnalysisAsMarkdown(aiAnalysis) : '' + + // Create page content with proper markdown formatting + const pageContent = [ + `# ${url.page_title || url.url}`, + '', + `**URL:** ${url.url}`, + url.domain_name ? `**Domain:** ${url.domain_name}` : '', + url.http_status ? `**Status:** ${url.http_status}` : '', + '', + url.meta_description ? url.meta_description : '', + '', + aiAnalysisMarkdown + ] + .filter(Boolean) + .join('\n') + + // Build comprehensive metadata + const metadata: ICommonObject = { + // Core fields + id: url.id, + url: url.url, + domain_name: url.domain_name, + domain_id: url.domain_id, + http_status: url.http_status, + status_text: url.status_text, + content_type: url.content_type, + page_title: url.page_title, + meta_description: url.meta_description, + canonical_url: url.canonical_url, + + // Custom data (spread all custom fields) + ...customData, + + // AI analysis + ai_summary: aiAnalysis.summary, + ai_category: aiAnalysis.category, + ai_content_type: aiAnalysis.content_type, + ai_topics: aiAnalysis.topics, + ai_confidence: aiAnalysis.confidence_score, + + // Tags + tags: url.tags || [], + tag_slugs: (url.tags || []).map((t: any) => t.slug).filter(Boolean), + tag_labels: (url.tags || []).map((t: any) => t.label).filter(Boolean), + + // Timestamps (ISO strings) + created_at: url.created_at, + updated_at: url.updated_at, + processed_at: url.processed_at, + first_seen_at: url.first_seen_at, + last_crawled_at: url.last_crawled_at, + last_analyzed_at: url.last_analyzed_at, + analysis_version: url.analysis_version, + + // Timestamps (Unix timestamps in seconds for Pinecone numeric filters) + created_at_timestamp: url.created_at ? Math.floor(new Date(url.created_at).getTime() / 1000) : null, + updated_at_timestamp: url.updated_at ? Math.floor(new Date(url.updated_at).getTime() / 1000) : null, + processed_at_timestamp: url.processed_at ? Math.floor(new Date(url.processed_at).getTime() / 1000) : null, + first_seen_at_timestamp: url.first_seen_at ? Math.floor(new Date(url.first_seen_at).getTime() / 1000) : null, + last_crawled_at_timestamp: url.last_crawled_at ? Math.floor(new Date(url.last_crawled_at).getTime() / 1000) : null, + last_analyzed_at_timestamp: url.last_analyzed_at ? Math.floor(new Date(url.last_analyzed_at).getTime() / 1000) : null, + + // Source + source: 'aai_datastore', + source_type: 'url' + } + + // Remove undefined/null values + Object.keys(metadata).forEach((key) => { + if (metadata[key] === undefined || metadata[key] === null) { + delete metadata[key] + } + }) + + return new Document({ + pageContent, + metadata + }) + } +} + +module.exports = { + nodeClass: AAIUrls_DocumentLoaders +} diff --git a/packages/components/nodes/documentloaders/AAIUrls/answerai-square-black.png b/packages/components/nodes/documentloaders/AAIUrls/answerai-square-black.png new file mode 100644 index 00000000000..24dfd99f98c Binary files /dev/null and b/packages/components/nodes/documentloaders/AAIUrls/answerai-square-black.png differ