diff --git a/.github/workflows/PUBLISH_PACKAGES_README.md b/.github/workflows/PUBLISH_PACKAGES_README.md new file mode 100644 index 00000000000..1f7f496ab8a --- /dev/null +++ b/.github/workflows/PUBLISH_PACKAGES_README.md @@ -0,0 +1,245 @@ +# Automated NPM Package Publishing + +This document describes the automated publishing workflows for `aai-embed` and `aai-embed-react` packages. + +## Architecture Overview + +Two separate workflows in two different repositories: + +1. **aai-embed** (submodule: `the-answerai/chat-embed`) + - Publishes `@answerai/aai-embed` when changes merge to `a-main` branch + - Manages its own versioning, building, and npm publishing + +2. **aai-embed-react** (parent: `the-answerai/theanswer`) + - Publishes `@answerai/aai-embed-react` when: + - Code changes in `packages/embed-react/` (publishes immediately, no verification) + - Submodule hash updates (only publishes if aai-embed version exists on npm) + +--- + +## Publishing Flow + +``` +1. Developer merges PR to chat-embed a-main branch + └─> Submodule workflow: + ├─> Bumps version (e.g., 3.0.4 → 3.0.5) + ├─> Publishes @answerai/aai-embed@3.0.5 to npm + ├─> Commits version bump back to a-main [skip ci] + └─> Creates tag v3.0.5 + +2. Developer updates submodule hash in parent repo + $ cd packages/embed && git pull origin a-main && cd ../.. + $ git add packages/embed + $ git commit -m "chore: update aai-embed to v3.0.5" + $ git push origin staging + +3. Parent repo workflow: + ├─> Detects submodule hash changed + ├─> Verifies @answerai/aai-embed@3.0.5 exists on npm ✅ + ├─> Bumps aai-embed-react (3.0.10 → 3.0.11) + ├─> Updates peerDependencies to "aai-embed": "^3.0.5" + └─> Publishes @answerai/aai-embed-react@3.0.11 +``` + +--- + +## Workflow 1: aai-embed (Submodule Repo) + +**Location:** `the-answerai/chat-embed/.github/workflows/publish-npm.yml` + +### Triggers +- **Automatic**: Push to `a-main` branch +- **Manual**: Workflow dispatch with optional version override + +### Version Bumping +Based on commit messages: +- `BREAKING`/`MAJOR` → major version +- `feat`/`feature`/`minor` → minor version +- Default → patch version + +### Required Setup +- **Secret**: `NPM_TOKEN` with publish access to `@answerai` scope +- **Status**: ✅ Tested and working + +--- + +## Workflow 2: aai-embed-react (Parent Repo) + +**Location:** `.github/workflows/publish-packages.yml` + +### Triggers +- **Automatic**: Push to `staging` branch +- **Manual**: Workflow dispatch with optional version override + +### Three Jobs + +**1. detect-changes** +- Checks if `packages/embed-react/` files changed +- Checks if submodule hash changed + +**2. verify-embed-published** (conditional) +- Only runs if submodule hash changed +- Verifies aai-embed version exists on npm +- Fails with clear error if not found + +**3. publish-aai-embed-react** +- Runs if: + - Direct code changes in embed-react (no verification needed), OR + - Submodule hash changed AND verification passed + +### Version Bumping +- **Submodule-only updates**: Patch bump +- **Code changes**: Based on commit messages (BREAKING/feat/fix) + +--- + +## Safety Features + +### Version Verification +Only applies when submodule hash changes (not for direct code changes). + +**Scenario: Premature Hash Update** +```bash +cd packages/embed && git checkout && cd ../.. +git push + +# Workflow: Detects hash change → npm check fails → Nothing published ✅ +``` + +**Scenario: Correct Hash Update** +```bash +# After aai-embed@3.0.5 published +git push + +# Workflow: Detects hash change → npm check passes → Publishes aai-embed-react ✅ +``` + +**Scenario: Direct Code Changes** +```bash +# Changes in packages/embed-react/ +git push + +# Workflow: Publishes immediately, no npm verification ✅ +``` + +--- + +## Manual Publishing + +### aai-embed-react +1. Go to Actions → Publish aai-embed-react to NPM +2. Click "Run workflow" +3. Optionally provide version override +4. Click "Run workflow" + +### aai-embed +1. In chat-embed repository +2. Go to Actions → Publish aai-embed to NPM +3. Click "Run workflow" → Run on `a-main` branch +4. After publishing, update hash in parent repo + +--- + +## Common Scenarios + +### Update aai-embed Code +```bash +# 1. In chat-embed repo - merge changes to a-main +# 2. Workflow auto-publishes and tags + +# 3. In parent repo (theanswer) +cd packages/embed && git pull origin a-main && cd ../.. +git add packages/embed +git commit -m "chore: update aai-embed to v3.0.5" +git push origin staging + +# 4. Parent workflow auto-publishes aai-embed-react +``` + +### Update aai-embed-react Code Only +```bash +cd packages/embed-react +# Make changes +git commit -m "feat: add TypeScript types" +git push origin staging + +# Workflow auto-publishes immediately (no verification) +``` + +--- + +## Troubleshooting + +### aai-embed-react Won't Publish + +**Check npm:** +```bash +cd packages/embed +npm view @answerai/aai-embed@$(node -p "require('./package.json').version") +``` + +If not found, aai-embed workflow needs to run first. + +### Build Failures + +Test locally: +```bash +cd packages/embed && pnpm build +cd packages/embed-react && pnpm build +``` + +### Version Conflicts + +Check published versions: +```bash +npm view @answerai/aai-embed versions +npm view @answerai/aai-embed-react versions +``` + +Manually trigger with higher version override if needed. + +--- + +## Best Practices + +### Commit Messages +```bash +git commit -m "fix: correct styling" # patch bump +git commit -m "feat: add new option" # minor bump +git commit -m "BREAKING: remove old API" # major bump +``` + +### Updating Submodule Hash +Always include version in commit: +```bash +git commit -m "chore: update aai-embed to v3.0.5" +``` + +--- + +## Setup Status + +### chat-embed Repository +- [x] Workflow created and tested +- [x] NPM_TOKEN configured +- [x] Successfully publishes to npm + +### theanswer Repository +- [x] Workflow file updated +- [x] Documentation updated +- [ ] Test with submodule hash update +- [ ] Verify verification logic + +--- + +## Monitoring + +**Workflow URLs:** +- Parent: https://github.com/the-answerai/theanswer/actions/workflows/publish-packages.yml +- Submodule: https://github.com/the-answerai/chat-embed/actions/workflows/publish-npm.yml + +**Check versions:** +```bash +npm view @answerai/aai-embed versions +npm view @answerai/aai-embed-react versions +``` diff --git a/.github/workflows/publish-packages.yml b/.github/workflows/publish-packages.yml new file mode 100644 index 00000000000..b819ad14347 --- /dev/null +++ b/.github/workflows/publish-packages.yml @@ -0,0 +1,314 @@ +name: Publish aai-embed-react to NPM + +# This workflow only publishes aai-embed-react. +# aai-embed is published by its own workflow in the chat-embed submodule repository. + +on: + push: + branches: + - staging + workflow_dispatch: + inputs: + version_override: + description: 'Version override for aai-embed-react (optional, e.g., 1.2.3)' + required: false + +jobs: + detect-changes: + name: Detect aai-embed-react Changes + runs-on: ubuntu-latest + outputs: + embed_react_changed: ${{ steps.changes.outputs.embed_react_changed }} + submodule_hash_changed: ${{ steps.changes.outputs.submodule_hash_changed }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: recursive + + - name: Detect changes + id: changes + run: | + # Skip if last commit was made by GitHub Action (prevents infinite loop) + COMMIT_AUTHOR="${{ github.event.head_commit.author.name || '' }}" + if [ "$COMMIT_AUTHOR" == "GitHub Action" ]; then + echo "⏭️ Skipping - last commit was automated version bump" + echo "embed_react_changed=false" >> $GITHUB_OUTPUT + echo "submodule_hash_changed=false" >> $GITHUB_OUTPUT + exit 0 + fi + + # For manual triggers, always mark as changed + if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then + echo "embed_react_changed=true" >> $GITHUB_OUTPUT + echo "submodule_hash_changed=false" >> $GITHUB_OUTPUT + exit 0 + fi + + # For push events, detect actual changes + PREV_COMMIT="${{ github.event.before }}" + CURR_COMMIT="${{ github.sha }}" + + # If this is the first push to the branch, compare with HEAD~1 + if [ "$PREV_COMMIT" == "0000000000000000000000000000000000000000" ]; then + PREV_COMMIT="HEAD~1" + fi + + echo "Comparing changes between $PREV_COMMIT and $CURR_COMMIT" + + # Check for changes in aai-embed-react + if git diff --name-only $PREV_COMMIT $CURR_COMMIT | grep -q "^packages/embed-react/"; then + echo "✓ Changes detected in packages/embed-react/" + echo "embed_react_changed=true" >> $GITHUB_OUTPUT + else + echo "✗ No changes in packages/embed-react/" + echo "embed_react_changed=false" >> $GITHUB_OUTPUT + fi + + # Check if submodule commit hash changed (indicates aai-embed dependency update) + if git diff $PREV_COMMIT $CURR_COMMIT -- packages/embed | grep -q "^+Subproject commit"; then + echo "✓ Submodule commit hash changed" + echo "submodule_hash_changed=true" >> $GITHUB_OUTPUT + else + echo "✗ Submodule commit hash unchanged" + echo "submodule_hash_changed=false" >> $GITHUB_OUTPUT + fi + + verify-embed-published: + name: Verify aai-embed is Published + needs: detect-changes + # Only run if submodule hash changed (not needed for direct aai-embed-react changes) + if: needs.detect-changes.outputs.submodule_hash_changed == 'true' + runs-on: ubuntu-latest + outputs: + is_published: ${{ steps.verify.outputs.is_published }} + embed_version: ${{ steps.verify.outputs.embed_version }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: recursive + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20.17.0' + registry-url: 'https://registry.npmjs.org' + + - name: Verify aai-embed version on npm + id: verify + run: | + cd packages/embed + + # Get version from submodule's package.json + EMBED_VERSION=$(node -p "require('./package.json').version") + echo "📦 Submodule aai-embed version: $EMBED_VERSION" + + # Get current commit hash + CURRENT_COMMIT=$(git rev-parse HEAD) + echo "📍 Submodule commit: $CURRENT_COMMIT" + + # Check if this version exists on npm + if npm view @answerai/aai-embed@$EMBED_VERSION version 2>/dev/null; then + echo "✅ @answerai/aai-embed@$EMBED_VERSION is published on npm" + + # Optional: Verify git tag matches commit for extra safety + EXPECTED_TAG="v${EMBED_VERSION}" + TAG_COMMIT=$(git rev-list -n 1 $EXPECTED_TAG 2>/dev/null || echo "") + + if [ -n "$TAG_COMMIT" ]; then + if [ "$CURRENT_COMMIT" == "$TAG_COMMIT" ]; then + echo "✅ Submodule commit matches version tag $EXPECTED_TAG" + else + echo "⚠️ WARNING: Submodule commit ($CURRENT_COMMIT) doesn't match tag $EXPECTED_TAG ($TAG_COMMIT)" + echo "This might indicate an unpublished commit. Proceeding anyway since version exists on npm." + fi + else + echo "ℹ️ No git tag found for $EXPECTED_TAG (this is okay if using a different tagging scheme)" + fi + + echo "is_published=true" >> $GITHUB_OUTPUT + echo "embed_version=$EMBED_VERSION" >> $GITHUB_OUTPUT + else + echo "❌ @answerai/aai-embed@$EMBED_VERSION NOT found on npm" + echo "" + echo "🛑 Cannot publish aai-embed-react because aai-embed@$EMBED_VERSION hasn't been published yet." + echo "" + echo "Next steps:" + echo "1. Ensure aai-embed workflow ran successfully in the chat-embed repository" + echo "2. Verify the package was published: npm view @answerai/aai-embed@$EMBED_VERSION" + echo "3. If not published, trigger the workflow manually in the chat-embed repo" + + echo "is_published=false" >> $GITHUB_OUTPUT + exit 1 + fi + + publish-aai-embed-react: + name: Publish aai-embed-react + needs: [detect-changes, verify-embed-published] + # Run if: + # 1. embed-react files changed (always publish), OR + # 2. submodule hash changed AND aai-embed is published (dependency update) + if: | + always() && + (needs.detect-changes.outputs.embed_react_changed == 'true' || + (needs.detect-changes.outputs.submodule_hash_changed == 'true' && + needs.verify-embed-published.outputs.is_published == 'true')) + runs-on: ubuntu-latest + permissions: + contents: write + packages: write + id-token: write + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: recursive + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20.17.0' + registry-url: 'https://registry.npmjs.org' + scope: '@answerai' + + - uses: pnpm/action-setup@v4 + with: + version: 9.15.9 + + - name: Setup Git + run: | + git config --local user.email "action@github.com" + git config --local user.name "GitHub Action" + + - name: Get aai-embed version + id: embed_version + run: | + cd packages/embed + EMBED_VERSION=$(node -p "require('./package.json').version") + echo "embed_version=$EMBED_VERSION" >> $GITHUB_OUTPUT + echo "aai-embed dependency version: $EMBED_VERSION" + + - name: Determine version bump + id: bump + working-directory: ./packages/embed-react + run: | + # Check if there are any tags + if git tag -l "aai-embed-react-v*" | grep -q .; then + LAST_TAG=$(git tag -l "aai-embed-react-v*" | sort -V | tail -n1) + echo "Last tag: $LAST_TAG" + COMMITS=$(git log --pretty=format:"%s" $LAST_TAG..HEAD -- packages/embed-react/) + else + echo "No tags found for aai-embed-react" + COMMITS=$(git log --pretty=format:"%s" -- packages/embed-react/) + fi + + echo "Commits:" + echo "$COMMITS" + + # If only submodule changed, do patch bump + if [ "${{ needs.detect-changes.outputs.embed_react_changed }}" == "false" ]; then + BUMP_TYPE="patch" + echo "Only submodule dependency updated, using patch bump" + else + # Determine version bump type from commits + BUMP_TYPE="patch" + echo "$COMMITS" | grep -i -E '\b(BREAKING|MAJOR)\b' > /dev/null && BUMP_TYPE="major" + if [ "$BUMP_TYPE" == "patch" ]; then + echo "$COMMITS" | grep -i -E '\b(feat|feature|minor)\b' > /dev/null && BUMP_TYPE="minor" + fi + fi + + echo "bump_type=$BUMP_TYPE" >> $GITHUB_OUTPUT + echo "Determined version bump: $BUMP_TYPE" + + - name: Bump version + id: version + working-directory: ./packages/embed-react + run: | + # Use manual version if provided + if [ -n "${{ github.event.inputs.version_override }}" ]; then + NEW_VERSION="${{ github.event.inputs.version_override }}" + node -e "const pkg = require('./package.json'); pkg.version = '$NEW_VERSION'; require('fs').writeFileSync('package.json', JSON.stringify(pkg, null, 2));" + else + BUMP_TYPE="${{ steps.bump.outputs.bump_type }}" + # Manually bump version to avoid npm/pnpm issues with workspace: protocol + if [ "$BUMP_TYPE" == "major" ]; then + NEW_VERSION=$(node -e "const pkg = require('./package.json'); const [major] = pkg.version.split('.').map(Number); console.log((major + 1) + '.0.0');") + elif [ "$BUMP_TYPE" == "minor" ]; then + NEW_VERSION=$(node -e "const pkg = require('./package.json'); const [major, minor] = pkg.version.split('.').map(Number); console.log(major + '.' + (minor + 1) + '.0');") + else + NEW_VERSION=$(node -e "const pkg = require('./package.json'); const [major, minor, patch] = pkg.version.split('.').map(Number); console.log(major + '.' + minor + '.' + (patch + 1));") + fi + node -e "const pkg = require('./package.json'); pkg.version = '$NEW_VERSION'; require('fs').writeFileSync('package.json', JSON.stringify(pkg, null, 2));" + fi + + echo "new_version=$NEW_VERSION" >> $GITHUB_OUTPUT + echo "New version: $NEW_VERSION" + + - name: Update package metadata + working-directory: ./packages/embed-react + run: | + # Update package name to scoped name + node -e "const pkg = require('./package.json'); pkg.name = '@answerai/aai-embed-react'; require('fs').writeFileSync('package.json', JSON.stringify(pkg, null, 2));" + + # Add publishConfig + node -e "const pkg = require('./package.json'); pkg.publishConfig = { access: 'public' }; require('fs').writeFileSync('package.json', JSON.stringify(pkg, null, 2));" + + # Add repository + node -e "const pkg = require('./package.json'); pkg.repository = { type: 'git', url: 'git+https://github.com/the-answerai/theanswer.git' }; require('fs').writeFileSync('package.json', JSON.stringify(pkg, null, 2));" + + # Update peerDependencies to match the aai-embed version from submodule + EMBED_VERSION="${{ steps.embed_version.outputs.embed_version }}" + echo "Updating aai-embed peer dependency to ^$EMBED_VERSION" + node -e "const pkg = require('./package.json'); pkg.peerDependencies['aai-embed'] = '^$EMBED_VERSION'; require('fs').writeFileSync('package.json', JSON.stringify(pkg, null, 2));" + + - name: Install dependencies + run: pnpm install + + - name: Build package + working-directory: ./packages/embed-react + run: pnpm build + + - name: Publish to npm + working-directory: ./packages/embed-react + run: npm publish --access public + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + + - name: Commit and push version bump + run: | + NEW_VERSION="aai-embed-react-v${{ steps.version.outputs.new_version }}" + echo "Creating tag $NEW_VERSION" + + cd packages/embed-react + git add package.json + cd ../.. + + # Commit the version bump + if git commit -m "chore: bump aai-embed-react to ${{ steps.version.outputs.new_version }} [skip ci]"; then + echo "✅ Version bump committed" + + # Push the commit back to staging + git push origin staging + + # Create and push tag + if ! git tag -l | grep -q "^$NEW_VERSION$"; then + git tag -a $NEW_VERSION -m "Release aai-embed-react ${{ steps.version.outputs.new_version }}" + git push origin $NEW_VERSION + fi + + echo "✅ Pushed commit and tag to origin" + else + echo "⚠️ No changes to commit (package.json unchanged)" + fi + + - name: Summary + run: | + echo "✅ Published @answerai/aai-embed-react@${{ steps.version.outputs.new_version }} to npm" + echo "✅ Peer dependency: aai-embed@^${{ steps.embed_version.outputs.embed_version }}" + echo "✅ Created tag aai-embed-react-v${{ steps.version.outputs.new_version }}" diff --git a/packages-answers/ui/src/components/ExportImportComponent.tsx b/packages-answers/ui/src/components/ExportImportComponent.tsx index f75f855bde5..2cd4fc7e3d6 100644 --- a/packages-answers/ui/src/components/ExportImportComponent.tsx +++ b/packages-answers/ui/src/components/ExportImportComponent.tsx @@ -1,19 +1,42 @@ import { useEffect, useRef, useState } from 'react' +import Image from 'next/image' +// Remove Redux import and replace with local state +// import { useDispatch } from 'react-redux' +// @ts-ignore import { stringify, exportData } from '@/utils/exportImport' +// Remove Redux action import +// @ts-ignore +// import { enqueueSnackbar as enqueueSnackbarAction } from '@/store/actions' // Material UI -import { Button, Dialog, DialogTitle, DialogContent, Stack, FormControlLabel, Checkbox, DialogActions, Box, MenuItem } from '@mui/material' - -// Assets -import ExportingGIF from '@/assets/images/Exporting.gif' +import { + Button, + Dialog, + DialogTitle, + DialogContent, + Stack, + FormControlLabel, + Checkbox, + DialogActions, + Box, + MenuItem, + Snackbar, + Alert +} from '@mui/material' // API +// @ts-ignore import exportImportApi from '@/api/exportimport' // Hooks +// @ts-ignore import useApi from '@/hooks/useApi' +// @ts-ignore import { getErrorMessage } from '@/utils/errorHandler' +//Assets +import ExportingGIF from '../../../../packages/ui/src/assets/images/Exporting.gif' + interface ExportDialogProps { show: boolean onCancel: () => void @@ -25,10 +48,25 @@ interface ExportImportComponentProps { onSuccess?: () => void } -const dataToExport = ['Chatflows', 'Agentflows', 'Tools', 'Variables', 'Assistants'] +const dataToExport = [ + 'Agentflows', + 'Agentflows V2', + 'Assistants Custom', + 'Assistants OpenAI', + 'Assistants Azure', + 'Chatflows', + 'Chats', + 'Chat Messages', + 'Chat Feedbacks', + 'Custom Templates', + 'Document Stores', + 'Executions', + 'Tools', + 'Variables' +] const ExportDialog = ({ show, onCancel, onExport }: ExportDialogProps) => { - const [selectedData, setSelectedData] = useState(['Chatflows', 'Agentflows', 'Tools', 'Variables', 'Assistants']) + const [selectedData, setSelectedData] = useState(dataToExport) const [isExporting, setIsExporting] = useState(false) useEffect(() => { @@ -78,7 +116,7 @@ const ExportDialog = ({ show, onCancel, onExport }: ExportDialogProps) => { {isExporting && (
- { }} src={ExportingGIF} alt='ExportingGIF' + width={100} + height={100} /> Exporting data might take a while
@@ -115,22 +155,86 @@ export const ExportImportMenuItems = ({ onClose, onSuccess }: ExportImportCompon const [exportDialogOpen, setExportDialogOpen] = useState(false) const inputRef = useRef(null) + // Replace Redux with local state for notifications + const [notification, setNotification] = useState<{ + open: boolean + message: string + severity: 'error' | 'success' | 'info' | 'warning' + }>({ + open: false, + message: '', + severity: 'info' + }) + + // Remove Redux dispatch + // const dispatch = useDispatch() + // const enqueueSnackbar = (...args: any[]) => dispatch(enqueueSnackbarAction(...args)) + const importAllApi = useApi(exportImportApi.importData) const exportAllApi = useApi(exportImportApi.exportData) + // Secure random function - no longer needed with local notifications + // const getSecureRandom = () => { + // const array = new Uint32Array(1) + // crypto.getRandomValues(array) + // return array[0] + // } + + // Replace Redux notification with local state + const showErrorNotification = (message: string) => { + setNotification({ + open: true, + message, + severity: 'error' + }) + } + + // Add function to close notification + const handleCloseNotification = () => { + setNotification((prev) => ({ ...prev, open: false })) + } + + // Helper function to extract error message from API response + const getApiErrorMessage = (error: any, fallbackMessage: string): string => { + if (!error?.response?.data) { + return fallbackMessage + } + + const responseData = error.response.data + return typeof responseData === 'object' ? responseData.message : responseData + } + const fileChange = (e: React.ChangeEvent) => { - if (!e.target.files) return + if (!e.target.files) { + return + } const file = e.target.files[0] - const reader = new FileReader() reader.onload = (evt) => { if (!evt?.target?.result) { return } - const body = JSON.parse(evt.target.result as string) - importAllApi.request(body) + + try { + const fileContent = evt.target.result as string + const body = JSON.parse(fileContent) + importAllApi.request(body) + } catch (error) { + // Handle specific JSON parsing errors + if (error instanceof SyntaxError) { + showErrorNotification(`Invalid JSON format: ${error.message}`) + } else { + // Handle other potential errors (e.g., API request setup) + showErrorNotification(`Import failed: ${getErrorMessage(error)}`) + } + } + } + + reader.onerror = () => { + showErrorNotification('Error reading file') } + reader.readAsText(file) } @@ -143,24 +247,63 @@ export const ExportImportMenuItems = ({ onClose, onSuccess }: ExportImportCompon const onExport = (data: string[]) => { const body: Record = {} - if (data.includes('Chatflows')) body.chatflow = true + // Use the same parameters as ProfileSection if (data.includes('Agentflows')) body.agentflow = true + if (data.includes('Agentflows V2')) body.agentflowv2 = true + if (data.includes('Assistants Custom')) body.assistantCustom = true + if (data.includes('Assistants OpenAI')) body.assistantOpenAI = true + if (data.includes('Assistants Azure')) body.assistantAzure = true + if (data.includes('Chatflows')) body.chatflow = true + if (data.includes('Chats')) body.chat = true + if (data.includes('Chat Messages')) body.chat_message = true + if (data.includes('Chat Feedbacks')) body.chat_feedback = true + if (data.includes('Custom Templates')) body.custom_template = true + if (data.includes('Document Stores')) body.document_store = true + if (data.includes('Executions')) body.execution = true if (data.includes('Tools')) body.tool = true if (data.includes('Variables')) body.variable = true - if (data.includes('Assistants')) body.assistant = true exportAllApi.request(body) } + // Import success effect useEffect(() => { - if (importAllApi.data && onSuccess) { - onSuccess() - if (onClose) onClose() + if (importAllApi.data) { + // Show success notification + setNotification({ + open: true, + message: 'Data imported successfully!', + severity: 'success' + }) + + if (onSuccess) { + onSuccess() + } + if (onClose) { + onClose() + } window.location.href = '/' } // eslint-disable-next-line react-hooks/exhaustive-deps }, [importAllApi.data]) + // Import error effect + useEffect(() => { + if (importAllApi.error) { + const error = importAllApi.error + const errMsg = getApiErrorMessage(error, 'Invalid Imported File') + showErrorNotification(`Failed to import: ${errMsg}`) + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [importAllApi.error]) + + // Import loading effect + useEffect(() => { + if (importAllApi.loading) { + // Loading state can be handled by UI components + } + }, [importAllApi.loading]) + useEffect(() => { if (exportAllApi.data) { setExportDialogOpen(false) @@ -174,20 +317,43 @@ export const ExportImportMenuItems = ({ onClose, onSuccess }: ExportImportCompon linkElement.setAttribute('download', exportAllApi.data.FileDefaultName) linkElement.click() + // Show success notification + setNotification({ + open: true, + message: 'Data exported successfully!', + severity: 'success' + }) + if (onClose) onClose() } catch (error) { - console.error(`Failed to export all: ${getErrorMessage(error)}`) + showErrorNotification(`Export failed: ${getErrorMessage(error)}`) } } // eslint-disable-next-line react-hooks/exhaustive-deps }, [exportAllApi.data]) + // Add error handling for export + useEffect(() => { + if (exportAllApi.error) { + setExportDialogOpen(false) + const error = exportAllApi.error + const errMsg = getApiErrorMessage(error, 'Internal Server Error') + showErrorNotification(`Failed to export: ${errMsg}`) + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [exportAllApi.error]) + return ( <> setExportDialogOpen(true)}>Export Data Import Data setExportDialogOpen(false)} onExport={(data) => onExport(data)} /> + + + {notification.message} + + ) } diff --git a/packages/embed b/packages/embed index fa836ede40e..e0f204cdb55 160000 --- a/packages/embed +++ b/packages/embed @@ -1 +1 @@ -Subproject commit fa836ede40e0e74fd7df445e367aeaaa2b3d8cfc +Subproject commit e0f204cdb5512eb1ed485921c8440ccb87fb6e60 diff --git a/packages/server/src/aai-utils/billing/stripe/StripeProvider.ts b/packages/server/src/aai-utils/billing/stripe/StripeProvider.ts index b0ee15dca0c..8a90b173292 100644 --- a/packages/server/src/aai-utils/billing/stripe/StripeProvider.ts +++ b/packages/server/src/aai-utils/billing/stripe/StripeProvider.ts @@ -300,6 +300,44 @@ export class StripeProvider { } } + /** + * Adjust timestamp for Stripe's 35-day limitation on meter events + * Stripe doesn't accept meter events older than 35 days + * + * @param originalTimestamp - Original timestamp in seconds + * @returns Adjusted timestamp info with original date preserved + */ + private adjustTimestampForStripe(originalTimestamp: number): { + adjustedTimestamp: number + wasAdjusted: boolean + originalDate?: string + } { + const now = Date.now() / 1000 // Current time in seconds + const thirtyFiveDaysAgo = now - 35 * 24 * 60 * 60 + const thirtyFourDaysAgo = now - 34 * 24 * 60 * 60 // Use 34 days as the "historical catch-up" date + + // Stripe allows timestamps up to and including 35 days ago + // We only need to adjust if it's MORE than 35 days old + if (originalTimestamp < thirtyFiveDaysAgo) { + log.info('Adjusting old timestamp for Stripe', { + originalDate: new Date(originalTimestamp * 1000).toISOString(), + adjustedDate: new Date(thirtyFourDaysAgo * 1000).toISOString(), + ageInDays: Math.floor((now - originalTimestamp) / (24 * 60 * 60)) + }) + + return { + adjustedTimestamp: Math.floor(thirtyFourDaysAgo), + wasAdjusted: true, + originalDate: new Date(originalTimestamp * 1000).toISOString() + } + } + + return { + adjustedTimestamp: originalTimestamp, + wasAdjusted: false + } + } + async syncUsageToStripe(creditsData: Array): Promise<{ meterEvents: Stripe.Billing.MeterEvent[] failedEvents: Array<{ traceId: string; error: string }> @@ -319,6 +357,7 @@ export class StripeProvider { const processedTraces: string[] = [] let selfHealedCount = 0 let duplicateEventCount = 0 + let adjustedTimestampCount = 0 // console.log('Syncing usage to Stripe', { // count: creditsData.length, @@ -333,7 +372,10 @@ export class StripeProvider { const batchResults = await Promise.allSettled( batch.map(async (data) => { - const timestamp = data.timestampEpoch || Math.floor(new Date(data.metadata.timestamp).getTime() / 1000) + const originalTimestamp = data.timestampEpoch || Math.floor(new Date(data.metadata.timestamp).getTime() / 1000) + + // Adjust timestamp for Stripe's 35-day limitation + const { adjustedTimestamp, wasAdjusted, originalDate } = this.adjustTimestampForStripe(originalTimestamp) // Ensure all unknown credit types are counted as AI tokens const aiTokens = data.credits.ai_tokens + (data.credits.unknown || 0) @@ -346,12 +388,17 @@ export class StripeProvider { let retryCount = 0 + // Track if this event's timestamp was adjusted + if (wasAdjusted) { + adjustedTimestampCount++ + } + while (retryCount < BILLING_CONFIG.VALIDATION.MAX_RETRIES) { try { const stripeMeterEvent = { event_name: 'credits', identifier: `${data.traceId}_credits`, - timestamp, + timestamp: adjustedTimestamp, // Use adjusted timestamp payload: { value: totalCredits.toString(), stripe_customer_id: data.stripeCustomerId, @@ -453,6 +500,16 @@ export class StripeProvider { }) } + // Log adjusted timestamps summary + if (adjustedTimestampCount > 0) { + log.info('Historical data: Adjusted timestamps for Stripe 35-day limitation', { + count: adjustedTimestampCount, + totalProcessed: processedTraces.length, + percentage: ((adjustedTimestampCount / processedTraces.length) * 100).toFixed(2) + '%', + note: 'Traces older than 35 days were batched to 34 days ago with original dates preserved in metadata' + }) + } + return { meterEvents, failedEvents, diff --git a/packages/server/src/routes/export-import/index.ts b/packages/server/src/routes/export-import/index.ts index 40c3930d2fd..bd7986fd16c 100644 --- a/packages/server/src/routes/export-import/index.ts +++ b/packages/server/src/routes/export-import/index.ts @@ -1,9 +1,11 @@ import express from 'express' import exportImportController from '../../controllers/export-import' +import enforceAbility from '../../middlewares/authentication/enforceAbility' const router = express.Router() -router.post('/export', exportImportController.exportData) +// SECURITY: Export/import operations require proper authorization +router.post('/export', enforceAbility('ChatFlow'), exportImportController.exportData) -router.post('/import', exportImportController.importData) +router.post('/import', enforceAbility('ChatFlow'), exportImportController.importData) export default router diff --git a/packages/server/src/services/chat-messages/index.ts b/packages/server/src/services/chat-messages/index.ts index 8fa20430d44..a0c4a9da639 100644 --- a/packages/server/src/services/chat-messages/index.ts +++ b/packages/server/src/services/chat-messages/index.ts @@ -196,12 +196,24 @@ const abortChatMessage = async (chatId: string, chatflowid: string) => { async function getAllMessages(user: IUser): Promise { const appServer = getRunningExpressApp() - return await appServer.AppDataSource.getRepository(ChatMessage).find({ where: { userId: user.id } }) + // SECURITY: Filter by both userId and organizationId for defense-in-depth + return await appServer.AppDataSource.getRepository(ChatMessage).find({ + where: { + userId: user.id, + organizationId: user.organizationId + } + }) } async function getAllMessagesFeedback(user: IUser): Promise { const appServer = getRunningExpressApp() - return await appServer.AppDataSource.getRepository(ChatMessageFeedback).find({ where: { userId: user.id } }) + // SECURITY: Filter by both userId and organizationId for defense-in-depth + return await appServer.AppDataSource.getRepository(ChatMessageFeedback).find({ + where: { + userId: user.id, + organizationId: user.organizationId + } + }) } export default { diff --git a/packages/server/src/services/export-import/index.ts b/packages/server/src/services/export-import/index.ts index 7989ff7a0b4..3d4d47d5ae4 100644 --- a/packages/server/src/services/export-import/index.ts +++ b/packages/server/src/services/export-import/index.ts @@ -2,6 +2,7 @@ import { StatusCodes } from 'http-status-codes' import { In, QueryRunner } from 'typeorm' import { v4 as uuidv4 } from 'uuid' import { Assistant } from '../../database/entities/Assistant' +import { Chat } from '../../database/entities/Chat' import { ChatFlow } from '../../database/entities/ChatFlow' import { ChatMessage } from '../../database/entities/ChatMessage' import { ChatMessageFeedback } from '../../database/entities/ChatMessageFeedback' @@ -13,16 +14,17 @@ import { Tool } from '../../database/entities/Tool' import { Variable } from '../../database/entities/Variable' import { InternalFlowiseError } from '../../errors/internalFlowiseError' import { getErrorMessage } from '../../errors/utils' +import { ChatType, IUser } from '../../Interface' import { getRunningExpressApp } from '../../utils/getRunningExpressApp' import assistantService from '../assistants' import chatMessagesService from '../chat-messages' +import chatsService from '../chats' import chatflowService from '../chatflows' import documenStoreService from '../documentstore' import executionService from '../executions' import marketplacesService from '../marketplaces' import toolsService from '../tools' import variableService from '../variables' -import { IUser } from '../../Interface' type ExportInput = { agentflow: boolean @@ -31,6 +33,7 @@ type ExportInput = { assistantOpenAI: boolean assistantAzure: boolean chatflow: boolean + chat: boolean chat_message: boolean chat_feedback: boolean custom_template: boolean @@ -47,6 +50,7 @@ type ExportData = { AssistantFlow: ChatFlow[] AssistantOpenAI: Assistant[] AssistantAzure: Assistant[] + Chat: Chat[] ChatFlow: ChatFlow[] ChatMessage: ChatMessage[] ChatMessageFeedback: ChatMessageFeedback[] @@ -63,8 +67,14 @@ const convertExportInput = (body: any): ExportInput => { if (!body || typeof body !== 'object') throw new Error('Invalid ExportInput object in request body') if (body.agentflow && typeof body.agentflow !== 'boolean') throw new Error('Invalid agentflow property in ExportInput object') if (body.agentflowv2 && typeof body.agentflowv2 !== 'boolean') throw new Error('Invalid agentflowv2 property in ExportInput object') - if (body.assistant && typeof body.assistant !== 'boolean') throw new Error('Invalid assistant property in ExportInput object') + if (body.assistantCustom && typeof body.assistantCustom !== 'boolean') + throw new Error('Invalid assistantCustom property in ExportInput object') + if (body.assistantOpenAI && typeof body.assistantOpenAI !== 'boolean') + throw new Error('Invalid assistantOpenAI property in ExportInput object') + if (body.assistantAzure && typeof body.assistantAzure !== 'boolean') + throw new Error('Invalid assistantAzure property in ExportInput object') if (body.chatflow && typeof body.chatflow !== 'boolean') throw new Error('Invalid chatflow property in ExportInput object') + if (body.chat && typeof body.chat !== 'boolean') throw new Error('Invalid chat property in ExportInput object') if (body.chat_message && typeof body.chat_message !== 'boolean') throw new Error('Invalid chat_message property in ExportInput object') if (body.chat_feedback && typeof body.chat_feedback !== 'boolean') @@ -88,6 +98,13 @@ const convertExportInput = (body: any): ExportInput => { const FileDefaultName = 'ExportData.json' const exportData = async (exportInput: ExportInput, user: IUser): Promise<{ FileDefaultName: string } & ExportData> => { try { + // SECURITY: Validate user has proper permissions and is authenticated + if (!user?.id || !user?.organizationId) { + throw new InternalFlowiseError(StatusCodes.UNAUTHORIZED, 'Error: exportImportService.exportData - User authentication required') + } + + // SECURITY: Validate user has proper permissions and is authenticated + let AgentFlow: ChatFlow[] = exportInput.agentflow === true ? await chatflowService.getAllChatflows(user, 'MULTIAGENT') : [] let AgentFlowV2: ChatFlow[] = exportInput.agentflowv2 === true ? await chatflowService.getAllChatflows(user, 'AGENTFLOW') : [] @@ -102,7 +119,15 @@ const exportData = async (exportInput: ExportInput, user: IUser): Promise<{ File let ChatFlow: ChatFlow[] = exportInput.chatflow === true ? await chatflowService.getAllChatflows(user, 'CHATFLOW') : [] - let ChatMessage: ChatMessage[] = exportInput.chat_message === true ? await chatMessagesService.getAllMessages(user) : [] + let Chat: Chat[] = [] + if (exportInput.chat === true) { + Chat = await chatsService.getAllChats(user) + } + + let ChatMessage: ChatMessage[] = [] + if (exportInput.chat_message === true) { + ChatMessage = await chatMessagesService.getAllMessages(user) + } let ChatMessageFeedback: ChatMessageFeedback[] = exportInput.chat_feedback === true ? await chatMessagesService.getAllMessagesFeedback(user) : [] @@ -116,11 +141,23 @@ const exportData = async (exportInput: ExportInput, user: IUser): Promise<{ File let DocumentStoreFileChunk: DocumentStoreFileChunk[] = exportInput.document_store === true ? await documenStoreService.getAllDocumentFileChunks(user) : [] - const { data: totalExecutions } = exportInput.execution === true ? await executionService.getAllExecutions(user) : { data: [] } + // SECURITY: Only export executions owned by the user + const { data: totalExecutions } = + exportInput.execution === true + ? await executionService.getAllExecutions( + {}, + { + userId: user.id, + organizationId: user.organizationId + } + ) + : { data: [] } let Execution: Execution[] = exportInput.execution === true ? totalExecutions : [] - let Tool: Tool[] = exportInput.tool === true ? await toolsService.getAllTools(user) : [] + let Tool: Tool[] = exportInput.tool === true ? await toolsService.getAllTools(user, true) : [] - let Variable: Variable[] = exportInput.variable === true ? await variableService.getAllVariables(user) : [] + let Variable: Variable[] = exportInput.variable === true ? await variableService.getAllVariables(user, true) : [] + + // Export completed successfully return { FileDefaultName, @@ -130,6 +167,7 @@ const exportData = async (exportInput: ExportInput, user: IUser): Promise<{ File AssistantFlow, AssistantOpenAI, AssistantAzure, + Chat, ChatFlow, ChatMessage, ChatMessageFeedback, @@ -148,19 +186,27 @@ const exportData = async (exportInput: ExportInput, user: IUser): Promise<{ File } } -async function replaceDuplicateIdsForChatFlow(queryRunner: QueryRunner, originalData: ExportData, chatflows: ChatFlow[]) { +async function replaceDuplicateIdsForChatFlow( + queryRunner: QueryRunner, + originalData: ExportData, + chatflows: ChatFlow[] +): Promise<{ data: ExportData; idMappings: Map }> { try { const ids = chatflows.map((chatflow) => chatflow.id) const records = await queryRunner.manager.find(ChatFlow, { where: { id: In(ids) } }) - if (records.length < 0) return originalData + const idMappings = new Map() + + if (records.length === 0) return { data: originalData, idMappings } + for (let record of records) { const oldId = record.id const newId = uuidv4() + idMappings.set(oldId, newId) originalData = JSON.parse(JSON.stringify(originalData).replaceAll(oldId, newId)) } - return originalData + return { data: originalData, idMappings } } catch (error) { throw new InternalFlowiseError( StatusCodes.INTERNAL_SERVER_ERROR, @@ -175,7 +221,7 @@ async function replaceDuplicateIdsForAssistant(queryRunner: QueryRunner, origina const records = await queryRunner.manager.find(Assistant, { where: { id: In(ids) } }) - if (records.length < 0) return originalData + if (records.length === 0) return originalData for (let record of records) { const oldId = record.id const newId = uuidv4() @@ -190,8 +236,54 @@ async function replaceDuplicateIdsForAssistant(queryRunner: QueryRunner, origina } } -async function replaceDuplicateIdsForChatMessage(queryRunner: QueryRunner, originalData: ExportData, chatMessages: ChatMessage[]) { +async function replaceDuplicateIdsForChat( + queryRunner: QueryRunner, + originalData: ExportData, + chats: Chat[] +): Promise<{ data: ExportData; idMappings: Map }> { try { + const ids = chats.map((chat) => chat.id) + const records = await queryRunner.manager.find(Chat, { + where: { id: In(ids) } + }) + const idMappings = new Map() + + if (records.length === 0) return { data: originalData, idMappings } + + for (let record of records) { + const oldId = record.id + const newId = uuidv4() + idMappings.set(oldId, newId) + originalData = JSON.parse(JSON.stringify(originalData).replaceAll(oldId, newId)) + } + return { data: originalData, idMappings } + } catch (error) { + throw new InternalFlowiseError( + StatusCodes.INTERNAL_SERVER_ERROR, + `Error: exportImportService.replaceDuplicateIdsForChat - ${getErrorMessage(error)}` + ) + } +} + +async function replaceDuplicateIdsForChatMessage( + queryRunner: QueryRunner, + originalData: ExportData, + chatMessages: ChatMessage[], + chatflowIdMappings: Map, + chatIdMappings?: Map +) { + try { + // First, update chatflowid references based on chatflow ID mappings + chatMessages.forEach((chatMessage) => { + if (chatflowIdMappings.has(chatMessage.chatflowid)) { + chatMessage.chatflowid = chatflowIdMappings.get(chatMessage.chatflowid)! + } + // Update chatId references based on chat ID mappings + if (chatIdMappings?.has(chatMessage.chatId)) { + chatMessage.chatId = chatIdMappings.get(chatMessage.chatId)! + } + }) + const chatmessageChatflowIds = chatMessages.map((chatMessage) => { return { id: chatMessage.chatflowid, qty: 0 } }) @@ -227,7 +319,7 @@ async function replaceDuplicateIdsForChatMessage(queryRunner: QueryRunner, origi const records = await queryRunner.manager.find(ChatMessage, { where: { id: In(ids) } }) - if (records.length < 0) return originalData + if (records.length === 0) return originalData for (let record of records) { const oldId = record.id const newId = uuidv4() @@ -355,7 +447,7 @@ async function replaceDuplicateIdsForChatMessageFeedback( const records = await queryRunner.manager.find(ChatMessageFeedback, { where: { id: In(ids) } }) - if (records.length < 0) return originalData + if (records.length === 0) return originalData for (let record of records) { const oldId = record.id const newId = uuidv4() @@ -376,7 +468,7 @@ async function replaceDuplicateIdsForCustomTemplate(queryRunner: QueryRunner, or const records = await queryRunner.manager.find(CustomTemplate, { where: { id: In(ids) } }) - if (records.length < 0) return originalData + if (records.length === 0) return originalData for (let record of records) { const oldId = record.id const newId = uuidv4() @@ -397,7 +489,7 @@ async function replaceDuplicateIdsForDocumentStore(queryRunner: QueryRunner, ori const records = await queryRunner.manager.find(DocumentStore, { where: { id: In(ids) } }) - if (records.length < 0) return originalData + if (records.length === 0) return originalData for (let record of records) { const oldId = record.id const newId = uuidv4() @@ -422,7 +514,7 @@ async function replaceDuplicateIdsForDocumentStoreFileChunk( const records = await queryRunner.manager.find(DocumentStoreFileChunk, { where: { id: In(ids) } }) - if (records.length < 0) return originalData + if (records.length === 0) return originalData for (let record of records) { const oldId = record.id const newId = uuidv4() @@ -443,7 +535,7 @@ async function replaceDuplicateIdsForTool(queryRunner: QueryRunner, originalData const records = await queryRunner.manager.find(Tool, { where: { id: In(ids) } }) - if (records.length < 0) return originalData + if (records.length === 0) return originalData for (let record of records) { const oldId = record.id const newId = uuidv4() @@ -464,7 +556,7 @@ async function replaceDuplicateIdsForVariable(queryRunner: QueryRunner, original const records = await queryRunner.manager.find(Variable, { where: { id: In(ids) } }) - if (records.length < 0) return originalData + if (records.length === 0) return originalData for (let record of records) { const oldId = record.id const newId = uuidv4() @@ -485,7 +577,7 @@ async function replaceDuplicateIdsForExecution(queryRunner: QueryRunner, origina const records = await queryRunner.manager.find(Execution, { where: { id: In(ids) } }) - if (records.length < 0) return originalData + if (records.length === 0) return originalData for (let record of records) { const oldId = record.id const newId = uuidv4() @@ -506,7 +598,37 @@ function reduceSpaceForChatflowFlowData(chatflows: ChatFlow[]) { }) } +function replaceUserIdOrganizationId(user: IUser, importData: ExportData) { + return Object.keys(importData).reduce((acc: any, key: string) => { + if (Array.isArray(importData[key as keyof ExportData])) { + acc[key as keyof ExportData] = importData[key as keyof ExportData].map((item) => { + const updatedItem = { ...item, userId: user.id, organizationId: user.organizationId } + + // Set chatType to INTERNAL for all imported chat messages so they appear in UI + if (key === 'ChatMessage') { + ;(updatedItem as any).chatType = ChatType.INTERNAL + } + + // Set ownerId for Chat entities so they appear in UI + if (key === 'Chat') { + ;(updatedItem as any).ownerId = user.id + } + + return updatedItem + }) + } + return acc + }, {}) +} + const importData = async (user: IUser, importData: ExportData) => { + // SECURITY: Validate user has proper permissions and is authenticated + if (!user?.id || !user?.organizationId) { + throw new InternalFlowiseError(StatusCodes.UNAUTHORIZED, 'Error: exportImportService.importData - User authentication required') + } + + // SECURITY: Validate user has proper permissions and is authenticated + // Initialize missing properties with empty arrays to avoid "undefined" errors importData.AgentFlow = importData.AgentFlow || [] importData.AgentFlowV2 = importData.AgentFlowV2 || [] @@ -514,6 +636,7 @@ const importData = async (user: IUser, importData: ExportData) => { importData.AssistantFlow = importData.AssistantFlow || [] importData.AssistantOpenAI = importData.AssistantOpenAI || [] importData.AssistantAzure = importData.AssistantAzure || [] + importData.Chat = importData.Chat || [] importData.ChatFlow = importData.ChatFlow || [] importData.ChatMessage = importData.ChatMessage || [] importData.ChatMessageFeedback = importData.ChatMessageFeedback || [] @@ -524,25 +647,36 @@ const importData = async (user: IUser, importData: ExportData) => { importData.Tool = importData.Tool || [] importData.Variable = importData.Variable || [] + // Validate import data structure + let queryRunner try { queryRunner = getRunningExpressApp().AppDataSource.createQueryRunner() await queryRunner.connect() - + importData = replaceUserIdOrganizationId(user, importData) try { + // Collect all chatflow ID mappings + const allChatflowIdMappings = new Map() + if (importData.AgentFlow.length > 0) { importData.AgentFlow = reduceSpaceForChatflowFlowData(importData.AgentFlow) - importData = await replaceDuplicateIdsForChatFlow(queryRunner, importData, importData.AgentFlow) + const result = await replaceDuplicateIdsForChatFlow(queryRunner, importData, importData.AgentFlow) + importData = result.data + result.idMappings.forEach((newId, oldId) => allChatflowIdMappings.set(oldId, newId)) } if (importData.AgentFlowV2.length > 0) { importData.AgentFlowV2 = reduceSpaceForChatflowFlowData(importData.AgentFlowV2) - importData = await replaceDuplicateIdsForChatFlow(queryRunner, importData, importData.AgentFlowV2) + const result = await replaceDuplicateIdsForChatFlow(queryRunner, importData, importData.AgentFlowV2) + importData = result.data + result.idMappings.forEach((newId, oldId) => allChatflowIdMappings.set(oldId, newId)) } if (importData.AssistantCustom.length > 0) importData = await replaceDuplicateIdsForAssistant(queryRunner, importData, importData.AssistantCustom) if (importData.AssistantFlow.length > 0) { importData.AssistantFlow = reduceSpaceForChatflowFlowData(importData.AssistantFlow) - importData = await replaceDuplicateIdsForChatFlow(queryRunner, importData, importData.AssistantFlow) + const result = await replaceDuplicateIdsForChatFlow(queryRunner, importData, importData.AssistantFlow) + importData = result.data + result.idMappings.forEach((newId, oldId) => allChatflowIdMappings.set(oldId, newId)) } if (importData.AssistantOpenAI.length > 0) importData = await replaceDuplicateIdsForAssistant(queryRunner, importData, importData.AssistantOpenAI) @@ -550,10 +684,28 @@ const importData = async (user: IUser, importData: ExportData) => { importData = await replaceDuplicateIdsForAssistant(queryRunner, importData, importData.AssistantAzure) if (importData.ChatFlow.length > 0) { importData.ChatFlow = reduceSpaceForChatflowFlowData(importData.ChatFlow) - importData = await replaceDuplicateIdsForChatFlow(queryRunner, importData, importData.ChatFlow) + const result = await replaceDuplicateIdsForChatFlow(queryRunner, importData, importData.ChatFlow) + importData = result.data + result.idMappings.forEach((newId, oldId) => allChatflowIdMappings.set(oldId, newId)) } + + // Collect chatflow ID mappings for chat message processing + + let allChatIdMappings = new Map() + if (importData.Chat.length > 0) { + const result = await replaceDuplicateIdsForChat(queryRunner, importData, importData.Chat) + importData = result.data + result.idMappings.forEach((newId, oldId) => allChatIdMappings.set(oldId, newId)) + } + if (importData.ChatMessage.length > 0) { - importData = await replaceDuplicateIdsForChatMessage(queryRunner, importData, importData.ChatMessage) + importData = await replaceDuplicateIdsForChatMessage( + queryRunner, + importData, + importData.ChatMessage, + allChatflowIdMappings, + allChatIdMappings + ) importData = await replaceExecutionIdForChatMessage(queryRunner, importData, importData.ChatMessage) } if (importData.ChatMessageFeedback.length > 0) @@ -578,8 +730,13 @@ const importData = async (user: IUser, importData: ExportData) => { if (importData.AssistantCustom.length > 0) await queryRunner.manager.save(Assistant, importData.AssistantCustom) if (importData.AssistantOpenAI.length > 0) await queryRunner.manager.save(Assistant, importData.AssistantOpenAI) if (importData.AssistantAzure.length > 0) await queryRunner.manager.save(Assistant, importData.AssistantAzure) + if (importData.Chat.length > 0) { + await queryRunner.manager.save(Chat, importData.Chat) + } if (importData.ChatFlow.length > 0) await queryRunner.manager.save(ChatFlow, importData.ChatFlow) - if (importData.ChatMessage.length > 0) await queryRunner.manager.save(ChatMessage, importData.ChatMessage) + if (importData.ChatMessage.length > 0) { + await queryRunner.manager.save(ChatMessage, importData.ChatMessage) + } if (importData.ChatMessageFeedback.length > 0) await queryRunner.manager.save(ChatMessageFeedback, importData.ChatMessageFeedback) if (importData.CustomTemplate.length > 0) await queryRunner.manager.save(CustomTemplate, importData.CustomTemplate) @@ -592,7 +749,7 @@ const importData = async (user: IUser, importData: ExportData) => { await queryRunner.commitTransaction() } catch (error) { - if (queryRunner && !queryRunner.isTransactionActive) await queryRunner.rollbackTransaction() + if (queryRunner && queryRunner.isTransactionActive) await queryRunner.rollbackTransaction() throw error } finally { if (queryRunner && !queryRunner.isReleased) await queryRunner.release() diff --git a/packages/server/src/services/tools/index.ts b/packages/server/src/services/tools/index.ts index 969ada62c65..5f323bf4e0f 100644 --- a/packages/server/src/services/tools/index.ts +++ b/packages/server/src/services/tools/index.ts @@ -43,12 +43,27 @@ const deleteTool = async (toolId: string, user: IUser): Promise => { } } -const getAllTools = async (user: IUser): Promise => { +const getAllTools = async (user: IUser, exportMode: boolean = false): Promise => { try { const appServer = getRunningExpressApp() const toolRepo = appServer.AppDataSource.getRepository(Tool) const isAdmin = user?.roles?.includes('Admin') + // SECURITY: For export operations, only include tools owned by the user + if (exportMode) { + const tools = await toolRepo.find({ + where: { + organizationId: user.organizationId, + userId: user.id + } + }) + return tools.map((tool) => ({ + ...tool, + isOwner: true // All tools in export mode are user-owned + })) + } + + // Regular mode - include shared tools for viewing const tools = await toolRepo.find({ // @ts-ignore where: isAdmin diff --git a/packages/server/src/services/variables/index.ts b/packages/server/src/services/variables/index.ts index 7f276cc902a..de9cfc79e6c 100644 --- a/packages/server/src/services/variables/index.ts +++ b/packages/server/src/services/variables/index.ts @@ -36,11 +36,26 @@ const deleteVariable = async (variableId: string): Promise => { } } -const getAllVariables = async (user: IUser) => { +const getAllVariables = async (user: IUser, exportMode: boolean = false) => { try { const appServer = getRunningExpressApp() const variableRepo = appServer.AppDataSource.getRepository(Variable) + // SECURITY: For export operations, only include variables owned by the user + if (exportMode) { + const variables = await variableRepo.find({ + where: { + organizationId: user.organizationId, + userId: user.id + } + }) + return variables.map((variable) => ({ + ...variable, + isOwner: true // All variables in export mode are user-owned + })) + } + + // Regular mode - include shared variables for viewing const isAdmin = user?.roles?.includes('Admin') let conditions: FindOptionsWhere | FindOptionsWhere[] diff --git a/packages/ui/src/layout/MainLayout/Header/ProfileSection/index.jsx b/packages/ui/src/layout/MainLayout/Header/ProfileSection/index.jsx index 9ddaaf92601..6488e7bb849 100644 --- a/packages/ui/src/layout/MainLayout/Header/ProfileSection/index.jsx +++ b/packages/ui/src/layout/MainLayout/Header/ProfileSection/index.jsx @@ -25,7 +25,10 @@ import exportImportApi from '@/api/exportimport' // Hooks import useApi from '@/hooks/useApi' -import { useNavigate } from '@/utils/navigation' +import { useAuth0 } from '@auth0/auth0-react' +import useNotifier from '@/utils/useNotifier' +import { getErrorMessage } from '@/utils/errorHandler' +import { stringify, exportData } from '@/utils/exportImport' const dataToExport = [ 'Agentflows', @@ -34,6 +37,7 @@ const dataToExport = [ 'Assistants OpenAI', 'Assistants Azure', 'Chatflows', + 'Chats', 'Chat Messages', 'Chat Feedbacks', 'Custom Templates', @@ -108,7 +112,7 @@ export const ExportDialog = ({ show, onCancel, onExport }) => { src={ExportingGIF} alt='ExportingGIF' /> - Exporting data might takes a while + Exporting data might take a while )} @@ -140,10 +144,23 @@ ExportDialog.propTypes = { onExport: PropTypes.func } +// ==============================|| ERROR ACTION COMPONENT ||============================== // + +const ErrorAction = ({ snackbarKey, onClose }) => ( + +) + +ErrorAction.propTypes = { + snackbarKey: PropTypes.oneOfType([PropTypes.string, PropTypes.number]).isRequired, + onClose: PropTypes.func.isRequired +} + // ==============================|| PROFILE MENU ||============================== // const ProfileSection = ({ username, handleLogout }) => { - const { user } = useUser() + const { user } = useAuth0() const customization = useSelector((state) => state.customization) const [open, setOpen] = useState(false) @@ -153,11 +170,19 @@ const ProfileSection = ({ username, handleLogout }) => { const anchorRef = useRef(null) const inputRef = useRef() - const navigate = useNavigate() const importAllApi = useApi(exportImportApi.importData) const exportAllApi = useApi(exportImportApi.exportData) const prevOpen = useRef(open) + // ==============================|| Secure Random ||============================== // + const getSecureRandom = () => { + const array = new Uint32Array(1) + crypto.getRandomValues(array) + return array[0] + } + + + // ==============================|| Snackbar ||============================== // useNotifier() @@ -177,11 +202,45 @@ const ProfileSection = ({ username, handleLogout }) => { } const errorFailed = (message) => { - showErrorNotification(message, enqueueSnackbar, closeSnackbar) + enqueueSnackbar({ + message, + options: { + key: new Date().getTime() + getSecureRandom(), + variant: 'error', + action: (key) => + } + }) } const fileChange = (e) => { - handleFileChange(e, importAllApi) + if (!e.target.files) return + + const file = e.target.files[0] + const reader = new FileReader() + reader.onload = (evt) => { + if (!evt?.target?.result) return + + try { + const fileContent = evt.target.result + const body = JSON.parse(fileContent) + importAllApi.request(body) + } catch (error) { + // Handle specific JSON parsing errors + if (error instanceof SyntaxError) { + errorFailed(`Invalid JSON format: ${error.message}`) + } else { + // Handle other potential errors (e.g., API request setup) + errorFailed(`Import failed: ${getErrorMessage(error)}`) + } + } + } + + reader.onerror = (error) => { + console.error('FileReader error:', error) + errorFailed('Error reading file') + } + + reader.readAsText(file) } const importAll = () => { @@ -196,6 +255,7 @@ const ProfileSection = ({ username, handleLogout }) => { if (data.includes('Assistants OpenAI')) body.assistantOpenAI = true if (data.includes('Assistants Azure')) body.assistantAzure = true if (data.includes('Chatflows')) body.chatflow = true + if (data.includes('Chats')) body.chat = true if (data.includes('Chat Messages')) body.chat_message = true if (data.includes('Chat Feedbacks')) body.chat_feedback = true if (data.includes('Custom Templates')) body.custom_template = true @@ -210,7 +270,16 @@ const ProfileSection = ({ username, handleLogout }) => { // Import success effect useEffect(() => { if (importAllApi.data) { - handleImportSuccess(dispatch, enqueueSnackbar, closeSnackbar, navigate) + enqueueSnackbar({ + message: 'Import completed successfully!', + options: { + key: new Date().getTime() + getSecureRandom(), + variant: 'success' + } + }) + setTimeout(() => { + window.location.href = '/' + }, 1000) } // eslint-disable-next-line react-hooks/exhaustive-deps }, [importAllApi.data]) @@ -232,7 +301,26 @@ const ProfileSection = ({ username, handleLogout }) => { useEffect(() => { if (exportAllApi.data) { setExportDialogOpen(false) - processExportedData(exportAllApi.data, errorFailed) + try { + const dataStr = stringify(exportData(exportAllApi.data)) + const blob = new Blob([dataStr], { type: 'application/json' }) + const dataUri = URL.createObjectURL(blob) + + const linkElement = document.createElement('a') + linkElement.setAttribute('href', dataUri) + linkElement.setAttribute('download', exportAllApi.data.FileDefaultName || 'export.json') + linkElement.click() + + enqueueSnackbar({ + message: 'Export completed successfully!', + options: { + key: new Date().getTime() + getSecureRandom(), + variant: 'success' + } + }) + } catch (error) { + errorFailed(`Export failed: ${getErrorMessage(error)}`) + } } // eslint-disable-next-line react-hooks/exhaustive-deps }, [exportAllApi.data]) diff --git a/packages/ui/src/utils/exportImport.js b/packages/ui/src/utils/exportImport.js index 1d8b3005fa2..43807e67aa2 100644 --- a/packages/ui/src/utils/exportImport.js +++ b/packages/ui/src/utils/exportImport.js @@ -22,12 +22,14 @@ const sanitizeTool = (Tool) => { const sanitizeChatflow = (ChatFlow) => { try { return ChatFlow.map((chatFlow) => { - const sanitizeFlowData = generateExportFlowData(JSON.parse(chatFlow.flowData)) + const sanitizeFlowData = generateExportFlowData(chatFlow) return { id: chatFlow.id, name: chatFlow.name, flowData: stringify(sanitizeFlowData), - type: chatFlow.type + type: chatFlow.type, + userId: chatFlow.userId, + organizationId: chatFlow.organizationId } }) } catch (error) { @@ -83,6 +85,7 @@ export const exportData = (exportAllData) => { AssistantCustom: sanitizeAssistant(exportAllData.AssistantCustom), AssistantOpenAI: sanitizeAssistant(exportAllData.AssistantOpenAI), AssistantAzure: sanitizeAssistant(exportAllData.AssistantAzure), + Chat: exportAllData.Chat, ChatFlow: sanitizeChatflow(exportAllData.ChatFlow), ChatMessage: exportAllData.ChatMessage, ChatMessageFeedback: exportAllData.ChatMessageFeedback,