From 29679a8a47f8f9783f59a0f49a17f4124f1da447 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Fri, 22 May 2026 12:50:00 -0600 Subject: [PATCH 01/72] Branding: Update with latest NLR logos --- ...-for-energy-innovation-fy26-logo-black.svg | 44 +++++++++++++ .../public/images/nlr-logo-horizontal.svg | 61 +++++++++++++++++++ frontend/src/components/layout/Footer.tsx | 8 +-- 3 files changed, 109 insertions(+), 4 deletions(-) create mode 100644 frontend/public/images/alliance-for-energy-innovation-fy26-logo-black.svg create mode 100755 frontend/public/images/nlr-logo-horizontal.svg diff --git a/frontend/public/images/alliance-for-energy-innovation-fy26-logo-black.svg b/frontend/public/images/alliance-for-energy-innovation-fy26-logo-black.svg new file mode 100644 index 0000000..bd213fb --- /dev/null +++ b/frontend/public/images/alliance-for-energy-innovation-fy26-logo-black.svg @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/frontend/public/images/nlr-logo-horizontal.svg b/frontend/public/images/nlr-logo-horizontal.svg new file mode 100755 index 0000000..4017b68 --- /dev/null +++ b/frontend/public/images/nlr-logo-horizontal.svg @@ -0,0 +1,61 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/frontend/src/components/layout/Footer.tsx b/frontend/src/components/layout/Footer.tsx index 2a643a4..c2906dd 100644 --- a/frontend/src/components/layout/Footer.tsx +++ b/frontend/src/components/layout/Footer.tsx @@ -26,14 +26,14 @@ export default function Footer() {
Alliance for Energy Innovation Date: Fri, 22 May 2026 12:51:22 -0600 Subject: [PATCH 02/72] Dev: Lint --- frontend/src/components/layout/Footer.tsx | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/frontend/src/components/layout/Footer.tsx b/frontend/src/components/layout/Footer.tsx index c2906dd..e80a83f 100644 --- a/frontend/src/components/layout/Footer.tsx +++ b/frontend/src/components/layout/Footer.tsx @@ -36,21 +36,13 @@ export default function Footer() { className="h-12" /> - - U.S. Department of Energy + + U.S. Department of Energy

- The National Laboratory of the Rockies is a national laboratory of the - U.S. Department of Energy, Office of Critical Minerals and Energy Innovation. + The National Laboratory of the Rockies is a national laboratory of the U.S. Department + of Energy, Office of Critical Minerals and Energy Innovation.

From b40e1323acb88a4faadbd8f0ee814d8275976a78 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Fri, 22 May 2026 12:52:09 -0600 Subject: [PATCH 03/72] Dev: Add biome front end lint --- frontend/biome.json | 54 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 frontend/biome.json diff --git a/frontend/biome.json b/frontend/biome.json new file mode 100644 index 0000000..4ef3a2c --- /dev/null +++ b/frontend/biome.json @@ -0,0 +1,54 @@ +{ + "$schema": "https://biomejs.dev/schemas/2.4.15/schema.json", + "assist": { + "enabled": true, + "actions": { + "source": { + "organizeImports": "on" + } + } + }, + "linter": { + "enabled": true, + "rules": { + "recommended": true + } + }, + "formatter": { + "enabled": true, + "indentStyle": "space", + "indentWidth": 2, + "lineWidth": 100 + }, + "css": { + "parser": { + "tailwindDirectives": true + }, + "linter": { + "enabled": true + }, + "formatter": { + "enabled": true, + "indentStyle": "space", + "indentWidth": 2 + } + }, + "javascript": { + "formatter": { + "quoteStyle": "single", + "trailingCommas": "all" + } + }, + "overrides": [ + { + "includes": ["tests/**"], + "linter": { + "rules": { + "style": { + "noNonNullAssertion": "off" + } + } + } + } + ] +} From c8552247b12ded41a09e1abedabc6a13a4831a3c Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Fri, 22 May 2026 12:55:35 -0600 Subject: [PATCH 04/72] Frontend: Add folder picker component --- .../src/components/common/FolderPicker.tsx | 183 ++++++++++++++++++ 1 file changed, 183 insertions(+) create mode 100644 frontend/src/components/common/FolderPicker.tsx diff --git a/frontend/src/components/common/FolderPicker.tsx b/frontend/src/components/common/FolderPicker.tsx new file mode 100644 index 0000000..bb989b7 --- /dev/null +++ b/frontend/src/components/common/FolderPicker.tsx @@ -0,0 +1,183 @@ +/** + * Lightweight folder picker for the Large Folder Upload page. + * + * Navigates the local filesystem (via /api/files/browse) and lets the user + * select the current directory with a single button click. Unlike FolderBrowser + * it does not show individual file upload status — it's purely a directory chooser. + */ + +import { useCallback, useEffect, useState } from 'react'; + +import { apiGet } from '../../api/client.ts'; +import type { BrowseResponse } from '../../types/api.ts'; +import { formatBytes } from '../../utils/format/bytes.ts'; +import { ChevronRightIcon, FolderIcon, RefreshIcon } from '../../utils/icons.tsx'; +import Breadcrumb from '../common/Breadcrumb.tsx'; +import Spinner from '../common/Spinner.tsx'; + +interface FolderPickerProps { + onFolderSelected: (folderPath: string) => void; + initialPath?: string; +} + +export default function FolderPicker({ onFolderSelected, initialPath }: FolderPickerProps) { + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + const [data, setData] = useState(null); + + const navigate = useCallback(async (path?: string) => { + setLoading(true); + setError(null); + try { + const params: Record = path ? { path } : {}; + const res = await apiGet('/api/files/browse', params); + setData(res); + } catch (err) { + setError(err instanceof Error ? err.message : 'Failed to browse folder'); + } finally { + setLoading(false); + } + }, []); + + useEffect(() => { + navigate(initialPath); + }, [navigate, initialPath]); + + const breadcrumbItems = (data?.breadcrumbs ?? []).map((b, i, arr) => ({ + label: b.name, + onClick: i < arr.length - 1 ? () => navigate(b.path) : undefined, + })); + + const handleSelect = useCallback(() => { + if (data) onFolderSelected(data.current_path); + }, [data, onFolderSelected]); + + return ( +
+ {/* Header */} +
+

Select Folder to Sync

+

+ Navigate to the folder you want to sync, then click Select This Folder. +

+
+ +
+ {/* Quick links sidebar */} +
+

+ Quick Links +

+
    + {(data?.quick_links ?? []).map((link) => ( +
  • + +
    • + ))} +
+
+ + {/* Main content */} +
+ {/* Breadcrumbs + refresh */} +
+ + +
+ + {/* "Select this folder" bar */} + {!loading && data && ( +
+
+ + {data.current_path} + + {data.total_file_count > 0 && ( + + {data.total_file_count.toLocaleString()} file + {data.total_file_count !== 1 ? 's' : ''} total + {data.total_file_count > data.file_count ? ' (includes subfolders)' : ''} + + )} +
+ +
+ )} + + {/* Loading / Error */} + {loading && ( +
+ +
+ )} + {error &&
{error}
} + + {/* Subfolders */} + {!loading && !error && data && ( +
+ {data.folders.length === 0 && ( +

+ No subfolders — this is a leaf directory. +

+ )} + {data.folders.map((folder) => ( + + ))} + + {/* Show file count summary if there are direct files */} + {data.files.length > 0 && ( +
+ {data.files.length.toLocaleString()} direct file + {data.files.length !== 1 ? 's' : ''} in this folder + {' · '} + {formatBytes(data.files.reduce((sum, f) => sum + f.size, 0))} +
+ )} +
+ )} +
+
+
+ ); +} From 3757c4f2b93a72bbc7d00ef6214e4a2908c33712 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Fri, 22 May 2026 12:56:24 -0600 Subject: [PATCH 05/72] Frontend: Add large folder service helpers --- frontend/src/api/largeFolderUpload.ts | 23 +++++ frontend/src/stores/largeFolderUploadStore.ts | 99 +++++++++++++++++++ 2 files changed, 122 insertions(+) create mode 100644 frontend/src/api/largeFolderUpload.ts create mode 100644 frontend/src/stores/largeFolderUploadStore.ts diff --git a/frontend/src/api/largeFolderUpload.ts b/frontend/src/api/largeFolderUpload.ts new file mode 100644 index 0000000..4d41c95 --- /dev/null +++ b/frontend/src/api/largeFolderUpload.ts @@ -0,0 +1,23 @@ +/** API helpers for the Large Folder Upload feature. */ + +import { apiPost } from './client.ts'; + +export interface StartSyncResponse { + job_id: string; + s3_uri: string; + cmd: string; +} + +export async function startLargeFolderSync( + folderPath: string, + s3Prefix: string, +): Promise { + return apiPost('/api/large-folder-upload/start', { + folder_path: folderPath, + s3_prefix: s3Prefix, + }); +} + +export async function cancelLargeFolderSync(jobId: string): Promise { + await apiPost(`/api/large-folder-upload/cancel/${jobId}`); +} diff --git a/frontend/src/stores/largeFolderUploadStore.ts b/frontend/src/stores/largeFolderUploadStore.ts new file mode 100644 index 0000000..7fe4e23 --- /dev/null +++ b/frontend/src/stores/largeFolderUploadStore.ts @@ -0,0 +1,99 @@ +import { create } from 'zustand'; + +export type SyncPhase = 'setup' | 'running' | 'done'; +export type SyncStatus = 'running' | 'completed' | 'failed' | 'cancelled'; + +export interface SyncProgress { + done: number; + total: number; + elapsedS: number; + etaS: number | null; +} + +interface LargeFolderUploadState { + phase: SyncPhase; + jobId: string | null; + s3Uri: string | null; + cmd: string | null; + folderPath: string | null; + s3Prefix: string; + lines: string[]; + status: SyncStatus | null; + returnCode: number | null; + error: string | null; + progress: SyncProgress | null; + /** Wall-clock ms when the real upload (post-dry-run) started. */ + uploadStartMs: number | null; + /** Total elapsed seconds when the job finished, for the done screen. */ + completedElapsedS: number | null; + + setFolderPath: (path: string) => void; + setS3Prefix: (prefix: string) => void; + startJob: (jobId: string, s3Uri: string, cmd: string) => void; + appendLine: (line: string) => void; + setProgress: (p: SyncProgress) => void; + setUploadStartMs: (ms: number) => void; + finish: (status: SyncStatus, returnCode: number | null) => void; + setError: (error: string) => void; + reset: () => void; +} + +function defaultPrefix(): string { + const now = new Date(); + const pad = (n: number, len = 2) => String(n).padStart(len, '0'); + return `user_upload_${now.getFullYear()}-${pad(now.getMonth() + 1)}-${pad(now.getDate())}T${pad(now.getHours())}-${pad(now.getMinutes())}-${pad(now.getSeconds())}`; +} + +const initialState = { + phase: 'setup' as SyncPhase, + jobId: null, + s3Uri: null, + cmd: null, + folderPath: null, + s3Prefix: defaultPrefix(), + lines: [], + status: null, + returnCode: null, + error: null, + progress: null, + uploadStartMs: null, + completedElapsedS: null, +}; + +export const useLargeFolderUploadStore = create((set, get) => ({ + ...initialState, + + setFolderPath: (path) => set({ folderPath: path }), + setS3Prefix: (prefix) => set({ s3Prefix: prefix }), + + startJob: (jobId, s3Uri, cmd) => + set({ + jobId, + s3Uri, + cmd, + phase: 'running', + lines: [], + status: 'running', + error: null, + progress: null, + uploadStartMs: null, + completedElapsedS: null, + }), + + appendLine: (line) => set((s) => ({ lines: [...s.lines, line] })), + + setProgress: (progress) => set({ progress }), + + setUploadStartMs: (ms) => set({ uploadStartMs: ms }), + + finish: (status, returnCode) => { + const { uploadStartMs } = get(); + const completedElapsedS = + uploadStartMs != null ? Math.round((Date.now() - uploadStartMs) / 1000) : null; + set({ phase: 'done', status, returnCode, completedElapsedS }); + }, + + setError: (error) => set({ error, phase: 'done', status: 'failed' }), + + reset: () => set({ ...initialState, s3Prefix: defaultPrefix() }), +})); From f57b314f7a8fbd2431c20c9488f0f8b8c7f538bb Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Fri, 22 May 2026 12:57:07 -0600 Subject: [PATCH 06/72] Frontend: Add modal for suggesting large folder usage --- .../upload/LargeFolderSuggestionModal.tsx | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 frontend/src/components/upload/LargeFolderSuggestionModal.tsx diff --git a/frontend/src/components/upload/LargeFolderSuggestionModal.tsx b/frontend/src/components/upload/LargeFolderSuggestionModal.tsx new file mode 100644 index 0000000..60095b7 --- /dev/null +++ b/frontend/src/components/upload/LargeFolderSuggestionModal.tsx @@ -0,0 +1,74 @@ +/** + * Modal shown when a scan finds 500+ files, suggesting the Large Folder Upload page. + */ + +import { useNavigate } from 'react-router-dom'; +import Modal from '../common/Modal.tsx'; + +const LARGE_FOLDER_THRESHOLD = 500; + +interface LargeFolderSuggestionModalProps { + isOpen: boolean; + fileCount: number; + onContinueAnyway: () => void; +} + +export { LARGE_FOLDER_THRESHOLD }; + +export default function LargeFolderSuggestionModal({ + isOpen, + fileCount, + onContinueAnyway, +}: LargeFolderSuggestionModalProps) { + const navigate = useNavigate(); + + return ( + + + + + } + > +
+

+ This folder contains{' '} + + {fileCount.toLocaleString()} files + + , which is a very large upload. +

+

+ The standard upload analyzes every file individually (checking timestamps, detecting + duplicates). For this many files that process can be slow. +

+

+ Large Folder Upload uses{' '} + aws s3 sync to mirror + the entire folder directly — much faster for large datasets, with real-time console + output and the ability to cancel at any time. +

+

+ Note: Large Folder Upload preserves your folder structure as-is rather than applying + Hive partitioning. +

+
+ + ); +} From 043cdfa5347f43ca8f942a1076d5d32b6d40b4cb Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Fri, 22 May 2026 12:58:21 -0600 Subject: [PATCH 07/72] Frontend: Add user facing aws sync page with virtualization --- frontend/src/pages/LargeFolderUploadPage.tsx | 650 +++++++++++++++++++ 1 file changed, 650 insertions(+) create mode 100644 frontend/src/pages/LargeFolderUploadPage.tsx diff --git a/frontend/src/pages/LargeFolderUploadPage.tsx b/frontend/src/pages/LargeFolderUploadPage.tsx new file mode 100644 index 0000000..e2ba756 --- /dev/null +++ b/frontend/src/pages/LargeFolderUploadPage.tsx @@ -0,0 +1,650 @@ +/** + * Large Folder Upload page — runs `aws s3 sync` and streams terminal output. + * + * Phase 1a (pick-folder): FolderPicker + * Phase 1b (name-prefix): S3 destination prefix input + * Phase 2 (running): Virtualized terminal + Cancel + * Phase 3 (done): Banner + terminal + action buttons + */ + +import { useVirtualizer } from '@tanstack/react-virtual'; +import { useCallback, useEffect, useRef, useState } from 'react'; +import { useNavigate } from 'react-router-dom'; + +import { cancelLargeFolderSync, startLargeFolderSync } from '../api/largeFolderUpload.ts'; +import FolderPicker from '../components/common/FolderPicker.tsx'; +import { useAppStore } from '../stores/appStore.ts'; +import { + type SyncProgress, + type SyncStatus, + useLargeFolderUploadStore, +} from '../stores/largeFolderUploadStore.ts'; +import { + ChevronRightIcon, + CloudIcon, + ErrorIcon, + FolderIcon, + SpinnerIcon, + SuccessIcon, + UploadIcon, + WarningIcon, + XIcon, +} from '../utils/icons.tsx'; + +// ─── Shared terminal ────────────────────────────────────────────────────────── + +/** Virtualized terminal that handles thousands of lines without DOM overflow. */ +function Terminal({ lines }: { lines: string[] }) { + const parentRef = useRef(null); + // Track whether the user has scrolled away from the bottom + const atBottomRef = useRef(true); + + const virtualizer = useVirtualizer({ + count: lines.length, + getScrollElement: () => parentRef.current, + estimateSize: () => 20, + overscan: 10, + }); + + // Auto-scroll only when already pinned to bottom + useEffect(() => { + if (atBottomRef.current && lines.length > 0) { + virtualizer.scrollToIndex(lines.length - 1, { align: 'end' }); + } + }, [lines.length, virtualizer]); + + const handleScroll = useCallback(() => { + const el = parentRef.current; + if (!el) return; + atBottomRef.current = el.scrollHeight - el.scrollTop - el.clientHeight < 60; + }, []); + + return ( +
+ {lines.length === 0 ? ( + Waiting for output… + ) : ( +
+ {virtualizer.getVirtualItems().map((item) => ( +
+ {lines[item.index] || '\u00A0'} +
+ ))} +
+ )} +
+ ); +} + +// ─── Progress bar ───────────────────────────────────────────────────────────── + +/** Format seconds as M:SS or H:MM:SS */ +function formatTimer(secs: number): string { + const s = Math.floor(secs); + if (s < 3600) { + return `${Math.floor(s / 60)}:${String(s % 60).padStart(2, '0')}`; + } + const h = Math.floor(s / 3600); + const m = Math.floor((s % 3600) / 60); + return `${h}:${String(m).padStart(2, '0')}:${String(s % 60).padStart(2, '0')}`; +} + +function ProgressBar({ + progress, + uploadStartMs, +}: { + progress: SyncProgress; + uploadStartMs: number | null; +}) { + const { done, total } = progress; + const pct = total > 0 ? Math.min(100, Math.round((done / total) * 100)) : 0; + + // Live ticking elapsed & eta — update every second from wall clock + const [liveElapsedS, setLiveElapsedS] = useState(0); + + useEffect(() => { + if (uploadStartMs == null) return; + const tick = () => setLiveElapsedS(Math.round((Date.now() - uploadStartMs) / 1000)); + tick(); + const id = setInterval(tick, 1000); + return () => clearInterval(id); + }, [uploadStartMs]); + + // Only show ETA after 5 files and 10+ seconds (stable estimate) + const stableEstimate = done >= 5 && liveElapsedS >= 10; + const liveEtaS = + stableEstimate && done > 0 && total > done + ? Math.round((liveElapsedS / done) * (total - done)) + : null; + + return ( +
+ {/* File count + percentage */} +
+ + {done.toLocaleString()} / {total.toLocaleString()} files + + {pct}% +
+ + {/* Progress bar */} +
+
+
+ + {/* Timer row */} +
+ + Elapsed time:{' '} + + {formatTimer(liveElapsedS)} + + + + Estimated time remaining:{' '} + {liveEtaS != null ? ( + + {formatTimer(liveEtaS)} + + ) : ( + Calculating… + )} + +
+ +

Already-uploaded files are skipped automatically.

+
+ ); +} + +// ─── CLI command display ────────────────────────────────────────────────────── + +/** Build a preview command from known values before the job starts. */ +function buildPreviewCmd( + folderPath: string, + s3Prefix: string, + bucket: string, + region: string, + profile: string, +): string { + const dest = `s3://${bucket}/${s3Prefix.trim().replace(/\/$/, '')}/`; + const parts = ['aws', 's3', 'sync', folderPath, dest, '--no-progress', '--region', region]; + if (profile && profile !== 'default') parts.push('--profile', profile); + return parts.join(' '); +} + +function CliCommand({ cmd }: { cmd: string }) { + const [copied, setCopied] = useState(false); + + const handleCopy = useCallback(() => { + void navigator.clipboard.writeText(cmd).then(() => { + setCopied(true); + setTimeout(() => setCopied(false), 2000); + }); + }, [cmd]); + + return ( +
+
+ + Run this upload from the command line + +
+ +
+

+ If the upload fails or you need to resume, paste this command in a terminal. It uses your{' '} + aws credential profile from Settings and will + automatically skip already-uploaded files. +

+ + {/* Command box */} +
+ 
+            {cmd}
+
+ +
+ +

+ Files already in the destination are automatically skipped — safe to re-run. +

+
+
+ ); +} + +// ─── Phase components ───────────────────────────────────────────────────────── + +function PickFolderStep() { + const { folderPath, setFolderPath } = useLargeFolderUploadStore(); + const defaultUploadFolder = useAppStore((s) => s.settings?.default_upload_folder); + const [chosen, setChosen] = useState(folderPath); + + const handleSelect = useCallback((path: string) => { + setChosen(path); + }, []); + + const handleConfirm = useCallback(() => { + if (chosen) setFolderPath(chosen); + }, [chosen, setFolderPath]); + + return ( +
+ + {chosen && ( +
+ + {chosen} +
+ )} +
+ ); +} + +function NamePrefixStep() { + const { folderPath, s3Prefix, setS3Prefix, setFolderPath, startJob, setError } = + useLargeFolderUploadStore(); + const settings = useAppStore((s) => s.settings); + const notifications = useAppStore((s) => s.addNotification); + + const handleBack = useCallback(() => { + // Return to folder picker — clear folderPath so SetupPhase shows picker again + setFolderPath(''); + }, [setFolderPath]); + + const handleStart = useCallback(async () => { + if (!folderPath || !s3Prefix.trim()) { + notifications('error', 'Folder and S3 prefix are required.'); + return; + } + try { + const { job_id, s3_uri, cmd } = await startLargeFolderSync(folderPath, s3Prefix.trim()); + startJob(job_id, s3_uri, cmd); + } catch (err) { + const msg = err instanceof Error ? err.message : 'Failed to start upload'; + setError(msg); + } + }, [folderPath, s3Prefix, startJob, setError, notifications]); + + return ( +
+ {/* Source summary */} +
+ +
+

Source folder

+

{folderPath}

+
+
+ + {/* Cloud destination prefix */} +
+
+ +

+ Name your cloud destination folder +

+
+ +
+ The default name is fine. It's a timestamp so each + upload gets its own folder in NLR Cloud Storage and nothing gets overwritten. You can + change it if you're continuing a previous upload or want a friendlier name. +
+ +
+ + setS3Prefix(e.target.value)} + className="w-full border border-gray-300 rounded-md px-3 py-2 text-sm font-mono focus:outline-none focus:ring-2 focus:ring-nlr-blue" + placeholder="user_upload_2025-01-01T12-00-00" + // biome-ignore lint/a11y/noAutofocus: focus is intentional — user is here to name the folder + autoFocus + /> +

+ Files land at{' '} + s3://<bucket>/{s3Prefix.trim() || '…'}/ +

+
+ +
+ + +
+ + {/* CLI fallback — preview before job starts */} + {settings && folderPath && s3Prefix.trim() && ( + + )} +
+
+ ); +} + +function SetupPhase() { + const folderPath = useLargeFolderUploadStore((s) => s.folderPath); + return folderPath ? : ; +} + +function RunningPhase() { + const { jobId, s3Uri, lines, status, progress, uploadStartMs, cmd } = useLargeFolderUploadStore(); + const finish = useLargeFolderUploadStore((s) => s.finish); + const appendLine = useLargeFolderUploadStore((s) => s.appendLine); + const setProgress = useLargeFolderUploadStore((s) => s.setProgress); + const setUploadStartMs = useLargeFolderUploadStore((s) => s.setUploadStartMs); + const esRef = useRef(null); + // "Scanning…" phase before dry-run completes + const [scanning, setScanning] = useState(true); + + useEffect(() => { + if (!jobId) return; + const es = new EventSource(`/api/large-folder-upload/progress/${jobId}`); + esRef.current = es; + + es.onmessage = (evt) => { + try { + const data = JSON.parse(evt.data as string) as { + type?: string; + line?: string; + status?: SyncStatus; + return_code?: number | null; + error?: string; + total_files?: number; + done?: number; + total?: number; + elapsed_s?: number; + eta_s?: number | null; + }; + + if (data.type === 'plan') { + setScanning(false); + setUploadStartMs(Date.now()); + setProgress({ done: 0, total: data.total_files ?? 0, elapsedS: 0, etaS: null }); + } else if (data.type === 'file_done') { + appendLine(data.line ?? ''); + setProgress({ + done: data.done ?? 0, + total: data.total ?? 0, + elapsedS: data.elapsed_s ?? 0, + etaS: data.eta_s ?? null, + }); + } else if (data.type === 'line' && data.line != null) { + appendLine(data.line); + } else if (data.type === 'done') { + finish(data.status ?? 'failed', data.return_code ?? null); + es.close(); + } + } catch { + // ignore parse errors + } + }; + + es.onerror = () => { + finish('failed', null); + es.close(); + }; + + return () => es.close(); + }, [jobId, appendLine, finish, setProgress, setUploadStartMs]); + + const handleCancel = useCallback(async () => { + if (!jobId) return; + esRef.current?.close(); + try { + await cancelLargeFolderSync(jobId); + } catch { + finish('cancelled', null); + } + }, [jobId, finish]); + + return ( +
+ {/* Header row */} +
+
+ +
+

+ {scanning ? 'Scanning for files to upload…' : 'Upload in progress…'} +

+ {s3Uri && ( +

+ NLR Cloud Storage → + {s3Uri} +

+ )} +
+
+ {status === 'running' && ( + + )} +
+ + {/* Scanning indeterminate bar */} + {scanning && ( +
+

Counting files to upload…

+
+
+
+
+ )} + + {/* Progress bar */} + {!scanning && progress && } + + + + {cmd && } +
+ ); +} + +function DonePhase() { + const { status, lines, s3Uri, folderPath, returnCode, reset, completedElapsedS, progress, cmd } = + useLargeFolderUploadStore(); + const navigate = useNavigate(); + + const isSuccess = status === 'completed'; + const isCancelled = status === 'cancelled'; + const uploadedCount = progress?.done ?? lines.length; + + return ( +
+
+ {isSuccess ? ( + + ) : isCancelled ? ( + + ) : ( + + )} +
+

+ {isSuccess + ? `${uploadedCount.toLocaleString()} file${uploadedCount !== 1 ? 's' : ''} uploaded successfully${completedElapsedS != null ? ` in ${formatTimer(completedElapsedS)}` : ''}` + : isCancelled + ? 'Upload cancelled' + : `Upload failed${returnCode != null ? ` (exit code ${returnCode})` : ''}`} +

+ + {isSuccess && s3Uri && ( +
+
+ Source + {folderPath} +
+
+ Destination +
+

NLR Cloud Storage (MODAQ AWS S3)

+ {s3Uri} +
+
+
+ )} + + {!isSuccess && s3Uri && ( +

{s3Uri}

+ )} + +

+ Already-uploaded files were skipped · output saved to the Event Log +

+
+
+ + + + {cmd && } + +
+ + + +
+
+ ); +} + +// ─── Page ───────────────────────────────────────────────────────────────────── + +export default function LargeFolderUploadPage() { + const phase = useLargeFolderUploadStore((s) => s.phase); + + return ( +
+
+ +

Large Folder Upload

+
+ + {/* Info banner — only in setup */} + {phase === 'setup' && ( +
+ +
+

Uploads the entire folder to NLR Cloud Storage

+

+ Copies all files directly to the MODAQ AWS S3 bucket, preserving your folder structure + as-is. Already-uploaded files are automatically skipped — no duplicates. Best for 500+ + files where per-file analysis would be slow. +

+
+
+ )} + + {phase === 'setup' && } + {phase === 'running' && } + {phase === 'done' && } +
+ ); +} From 6a4504866aa9aae94fa794ad51444afc3e2c90b9 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Fri, 22 May 2026 13:03:52 -0600 Subject: [PATCH 08/72] App: Add aws sync upload support --- app/routes/large_folder_upload.py | 467 ++++++++++++++++++++++++++++++ 1 file changed, 467 insertions(+) create mode 100644 app/routes/large_folder_upload.py diff --git a/app/routes/large_folder_upload.py b/app/routes/large_folder_upload.py new file mode 100644 index 0000000..4199453 --- /dev/null +++ b/app/routes/large_folder_upload.py @@ -0,0 +1,467 @@ +"""Large Folder Upload API routes — streams aws s3 sync output via SSE.""" + +import csv +import io +import json +import os +import subprocess +import threading +import time +import uuid +from collections.abc import Generator +from dataclasses import dataclass, field +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from flask import Blueprint, Response, jsonify, request + +from app.config import get_settings +from app.services.sse_manager import get_sse_manager + +large_folder_upload_bp = Blueprint("large_folder_upload", __name__) + +# In-memory registry of active sync jobs +_jobs: dict[str, "SyncJob"] = {} +_jobs_lock = threading.Lock() + + +@dataclass +class SyncJob: + job_id: str + folder_path: str + s3_prefix: str + s3_uri: str + cmd_base: list[str] # base command without --dryrun + status: str = "running" # running | completed | failed | cancelled + process: subprocess.Popen[str] | None = field(default=None, repr=False) + return_code: int | None = None + lines: list[str] = field(default_factory=list) + total_files: int = 0 # populated after dry-run + done_files: int = 0 + started_at: datetime = field(default_factory=lambda: datetime.now(UTC)) + upload_started_at: float = field(default_factory=time.monotonic) + created_at: float = field(default_factory=time.time) + + +def _register(job: SyncJob) -> None: + with _jobs_lock: + _jobs[job.job_id] = job + + +def _get(job_id: str) -> SyncJob | None: + with _jobs_lock: + return _jobs.get(job_id) + + +def _save_sync_log(job: SyncJob, completed_at: datetime) -> None: + """Write a summary JSONL log entry, a raw output text file, and an upload-history CSV.""" + try: + from app.services.log_service import get_log_service + from app.services.utils import format_file_size + + log = get_log_service() + settings = get_settings() + duration_s = (completed_at - job.started_at).total_seconds() + log_dir = settings.log_directory + + # ── Raw text log ────────────────────────────────────────────────────── + hive_txt = ( + log_dir + / "sync" + / f"year={completed_at.year:04d}" + / f"month={completed_at.month:02d}" + / f"day={completed_at.day:02d}" + ) + hive_txt.mkdir(parents=True, exist_ok=True) + time_str = completed_at.strftime("%H%M%S") + short_id = job.job_id[:8] + txt_path = hive_txt / f"sync-{time_str}-{short_id}.txt" + with open(txt_path, "w", encoding="utf-8") as fh: + fh.write(f"# Large Folder Upload — {job.job_id}\n") + fh.write(f"# folder: {job.folder_path}\n") + fh.write(f"# dest: {job.s3_uri}\n") + fh.write(f"# started: {job.started_at.isoformat()}\n") + fh.write(f"# ended: {completed_at.isoformat()}\n") + fh.write(f"# status: {job.status}\n\n") + fh.write("\n".join(job.lines)) + + # ── Upload-history CSV (appears in Upload History tab) ──────────────── + _write_history_csv(job, completed_at, log_dir, time_str, short_id, format_file_size) + + # ── JSONL event log entry ───────────────────────────────────────────── + if job.status == "completed": + level = "INFO" + elif job.status == "cancelled": + level = "WARNING" + else: + level = "ERROR" + log.log( + level, + "large_folder_sync", + f"sync_{job.status}", + f"Large folder sync {job.status}: {Path(job.folder_path).name} → {job.s3_uri}", + { + "job_id": job.job_id, + "folder_path": job.folder_path, + "s3_uri": job.s3_uri, + "s3_prefix": job.s3_prefix, + "return_code": job.return_code, + "duration_seconds": round(duration_s, 1), + "output_lines": len(job.lines), + "log_file": str(txt_path.relative_to(log_dir)), + }, + ) + except Exception: + pass # Never crash the streaming thread over logging + + +def _write_history_csv( + job: SyncJob, + completed_at: datetime, + log_dir: Path, + time_str: str, + short_id: str, + format_file_size: Any, +) -> None: + """Write a CSV into logs/csv/ so this sync shows in the Upload History tab.""" + # Parse "upload: /local/path to s3://bucket/key" lines + bucket = get_settings().s3_bucket + upload_lines = [ln for ln in job.lines if ln.startswith("upload:")] + if not upload_lines: + # Nothing was uploaded (all skipped); still write an empty-session CSV + upload_lines = [] + + num_files = len(upload_lines) + total_duration_s = (completed_at - job.started_at).total_seconds() + per_file_duration = total_duration_s / num_files if num_files > 0 else 0.0 + + columns = [ + "job_id", + "filename", + "file_size_bytes", + "file_size_formatted", + "s3_path", + "status", + "data_start_time", + "upload_started_at", + "upload_completed_at", + "upload_duration_seconds", + "upload_speed_mbps", + "is_duplicate", + "is_valid", + "error_message", + ] + + buf = io.StringIO() + writer = csv.writer(buf) + writer.writerow(columns) + + for line in upload_lines: + # "upload: /local/path to s3://bucket/key" + try: + rest = line[len("upload:") :].strip() + local_path, s3_full = rest.split(" to ", 1) + local_path = local_path.strip() + s3_full = s3_full.strip() + filename = Path(local_path).name + # Strip "s3://bucket/" to get the relative key + s3_path = s3_full.replace(f"s3://{bucket}/", "", 1) if bucket else s3_full + except ValueError: + continue + + try: + size_bytes = os.path.getsize(local_path) + except OSError: + size_bytes = 0 + + speed = ( + round(size_bytes / per_file_duration / 1024 / 1024 * 8, 2) + if per_file_duration > 0 and size_bytes > 0 + else "" + ) + + writer.writerow( + [ + job.job_id, + filename, + size_bytes, + format_file_size(size_bytes), + s3_path, + "completed", + "", # data_start_time — not available for sync + job.started_at.isoformat(), + completed_at.isoformat(), + round(per_file_duration, 3), + speed, + False, # is_duplicate — these were NOT skipped + True, # is_valid + "", + ] + ) + + hive_csv = ( + log_dir + / "csv" + / f"year={completed_at.year:04d}" + / f"month={completed_at.month:02d}" + / f"day={completed_at.day:02d}" + ) + hive_csv.mkdir(parents=True, exist_ok=True) + csv_path = hive_csv / f"upload-summary-{time_str}-{short_id}.csv" + with open(csv_path, "w", encoding="utf-8", newline="") as fh: + fh.write(buf.getvalue()) + + +def _stream_process(job: SyncJob) -> None: + """Dry-run to count files, then stream the real upload with progress events.""" + sse = get_sse_manager() + + # ── Phase 1: dry-run to count files that will actually be uploaded ── + try: + dryrun = subprocess.run( + [*job.cmd_base, "--dryrun"], + capture_output=True, + text=True, + timeout=300, + ) + total = sum(1 for ln in dryrun.stdout.splitlines() if "(dryrun) upload:" in ln) + job.total_files = total + sse.send_event(job.job_id, {"type": "plan", "total_files": total}) + except Exception: + # Dry-run failed — proceed without a known total + sse.send_event(job.job_id, {"type": "plan", "total_files": 0}) + + if job.status != "running": + return # Cancelled during dry-run + + # ── Phase 2: real upload ── + try: + proc = subprocess.Popen( + job.cmd_base, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + ) + job.process = proc + job.upload_started_at = time.monotonic() + + for raw_line in proc.stdout: # type: ignore[union-attr] + stripped = raw_line.rstrip("\n") + job.lines.append(stripped) + + if stripped.startswith("upload:"): + job.done_files += 1 + elapsed = time.monotonic() - job.upload_started_at + eta_s: int | None = None + if job.total_files > 0 and job.done_files > 0: + remaining = job.total_files - job.done_files + eta_s = int(elapsed / job.done_files * remaining) if remaining > 0 else 0 + sse.send_event( + job.job_id, + { + "type": "file_done", + "line": stripped, + "done": job.done_files, + "total": job.total_files, + "elapsed_s": int(elapsed), + "eta_s": eta_s, + }, + ) + else: + sse.send_event(job.job_id, {"type": "line", "line": stripped}) + + proc.wait() + job.return_code = proc.returncode + + if job.status == "running": + job.status = "completed" if job.return_code == 0 else "failed" + + completed_at = datetime.now(UTC) + _save_sync_log(job, completed_at) + + sse.send_event( + job.job_id, + { + "type": "done", + "status": job.status, + "return_code": job.return_code, + "done": job.done_files, + "total": job.total_files, + }, + ) + except Exception as exc: + job.status = "failed" + completed_at = datetime.now(UTC) + _save_sync_log(job, completed_at) + sse.send_event( + job.job_id, + {"type": "done", "status": "failed", "error": str(exc)}, + ) + + +@large_folder_upload_bp.route("/start", methods=["POST"]) +def start_sync() -> tuple[Response, int]: + """Start an aws s3 sync job. + + Request body: + folder_path: Local folder to sync from + s3_prefix: S3 key prefix (e.g. "user_upload_2025-01-01T12-00-00") + + Returns: + JSON with job_id + """ + if not request.is_json: + return jsonify({"error": "JSON body required"}), 400 + + data: dict[str, Any] = request.get_json() or {} + folder_path: str = data.get("folder_path", "").strip() + s3_prefix: str = data.get("s3_prefix", "").strip() + + if not folder_path: + return jsonify({"error": "folder_path is required"}), 400 + if not os.path.isdir(folder_path): + return jsonify({"error": f"folder_path does not exist: {folder_path}"}), 400 + if not s3_prefix: + return jsonify({"error": "s3_prefix is required"}), 400 + + settings = get_settings() + if not settings.s3_bucket: + return jsonify({"error": "S3 bucket not configured. Check Settings."}), 400 + + s3_uri = f"s3://{settings.s3_bucket}/{s3_prefix.strip('/')}/" + + cmd_base = [ + "aws", + "s3", + "sync", + folder_path, + s3_uri, + "--no-progress", + "--region", + settings.aws_region, + ] + if settings.aws_profile and settings.aws_profile != "default": + cmd_base += ["--profile", settings.aws_profile] + + # Verify aws CLI is available before creating the job + try: + subprocess.run(["aws", "--version"], capture_output=True, check=True, timeout=5) + except (FileNotFoundError, subprocess.CalledProcessError): + return jsonify({"error": "aws CLI not found. Install the AWS CLI and try again."}), 500 + + job_id = str(uuid.uuid4()) + job = SyncJob( + job_id=job_id, + folder_path=folder_path, + s3_prefix=s3_prefix, + s3_uri=s3_uri, + cmd_base=cmd_base, + ) + _register(job) + + # Log the start + try: + from app.services.log_service import get_log_service + + get_log_service().info( + "large_folder_sync", + "sync_started", + f"Large folder sync started: {folder_path} → {s3_uri}", + {"job_id": job_id, "folder_path": folder_path, "s3_uri": s3_uri}, + ) + except Exception: + pass + + thread = threading.Thread(target=_stream_process, args=(job,), daemon=True) + thread.start() + + cmd_display = " ".join(cmd_base) + return jsonify({"job_id": job_id, "s3_uri": s3_uri, "cmd": cmd_display}), 202 + + +@large_folder_upload_bp.route("/progress/", methods=["GET"]) +def stream_progress(job_id: str) -> Response: + """SSE stream of aws s3 sync output lines for a job.""" + + def generate() -> Generator[str, None, None]: + sse_mgr = get_sse_manager() + queue, event = sse_mgr.register_client(job_id) + try: + job = _get(job_id) + if not job: + yield f"data: {json.dumps({'error': 'Job not found'})}\n\n" + return + + # Replay lines already captured before client connected + for line in list(job.lines): + yield f"data: {json.dumps({'type': 'line', 'line': line})}\n\n" + + # If job already finished before the SSE connection opened, send done immediately + if job.status in ("completed", "failed", "cancelled"): + done_payload = { + "type": "done", + "status": job.status, + "return_code": job.return_code, + } + yield f"data: {json.dumps(done_payload)}\n\n" + return + + last_heartbeat = time.time() + while True: + while queue: + data = queue.popleft() + yield f"data: {json.dumps(data)}\n\n" + last_heartbeat = time.time() + if data.get("type") == "done": + return + + now = time.time() + if now - last_heartbeat > sse_mgr.heartbeat_interval: + yield ": heartbeat\n\n" + last_heartbeat = now + + event.wait(timeout=sse_mgr.heartbeat_interval) + event.clear() + + # Re-check job existence + if not _get(job_id): + yield f"data: {json.dumps({'error': 'Job not found'})}\n\n" + return + finally: + sse_mgr.deregister_client(job_id, queue) + + return Response( + generate(), + mimetype="text/event-stream", + headers={ + "Cache-Control": "no-cache", + "X-Accel-Buffering": "no", + "Connection": "keep-alive", + }, + ) + + +@large_folder_upload_bp.route("/cancel/", methods=["POST"]) +def cancel_sync(job_id: str) -> tuple[Response, int]: + """Cancel a running sync job.""" + job = _get(job_id) + if not job: + return jsonify({"error": "Job not found"}), 404 + if job.status != "running": + return jsonify({"message": "Job is not running", "status": job.status}), 200 + + job.status = "cancelled" + if job.process and job.process.poll() is None: + job.process.terminate() + try: + job.process.wait(timeout=5) + except subprocess.TimeoutExpired: + job.process.kill() + + get_sse_manager().send_event( + job_id, + {"type": "done", "status": "cancelled", "return_code": None}, + ) + return jsonify({"job_id": job_id, "status": "cancelled"}), 200 From 68c80a505facc261cdf3916df8dfc5c44411dafa Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Fri, 22 May 2026 13:04:32 -0600 Subject: [PATCH 09/72] Frontend: Dev, configure biome linter/formatter --- frontend/package-lock.json | 240 ++++++++++++++++++++++++++++++++++++- frontend/package.json | 11 +- 2 files changed, 247 insertions(+), 4 deletions(-) diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 249a958..98300b4 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "frontend", - "version": "0.0.0", + "version": "1.1.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "frontend", - "version": "0.0.0", + "version": "1.1.0", "dependencies": { "@tanstack/react-table": "^8.21.3", "@tanstack/react-virtual": "^3.13.18", @@ -17,6 +17,7 @@ "zustand": "^5.0.11" }, "devDependencies": { + "@biomejs/biome": "^2.4.15", "@eslint/js": "^9.39.1", "@tailwindcss/vite": "^4.1.18", "@testing-library/dom": "^10.4.0", @@ -418,6 +419,181 @@ "node": ">=18" } }, + "node_modules/@biomejs/biome": { + "version": "2.4.15", + "resolved": "https://registry.npmjs.org/@biomejs/biome/-/biome-2.4.15.tgz", + "integrity": "sha512-j5VH3a/h/HXTKBM50MDMxRCzkeLv9S2XJcW2WgnZT1+xyisi+0bISrXR82gCX+8S9lvK0skEvHJRN+3Ktr2hlw==", + "dev": true, + "license": "MIT OR Apache-2.0", + "bin": { + "biome": "bin/biome" + }, + "engines": { + "node": ">=14.21.3" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/biome" + }, + "optionalDependencies": { + "@biomejs/cli-darwin-arm64": "2.4.15", + "@biomejs/cli-darwin-x64": "2.4.15", + "@biomejs/cli-linux-arm64": "2.4.15", + "@biomejs/cli-linux-arm64-musl": "2.4.15", + "@biomejs/cli-linux-x64": "2.4.15", + "@biomejs/cli-linux-x64-musl": "2.4.15", + "@biomejs/cli-win32-arm64": "2.4.15", + "@biomejs/cli-win32-x64": "2.4.15" + } + }, + "node_modules/@biomejs/cli-darwin-arm64": { + "version": "2.4.15", + "resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-arm64/-/cli-darwin-arm64-2.4.15.tgz", + "integrity": "sha512-rF3PPqLq1yoST79zaQbDjVJwsuIeci/O+9bgNmC5QpgOqz6aqYuzA4abyAGx+mgyiDXn4A049xAN8gijbuR1Qg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT OR Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=14.21.3" + } + }, + "node_modules/@biomejs/cli-darwin-x64": { + "version": "2.4.15", + "resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-x64/-/cli-darwin-x64-2.4.15.tgz", + "integrity": "sha512-/5KHXYMfSJs1fNXiX30xFtI8JcCFV6zaVVLxOa0M2sfqBKHkpQhRTv94yxQWxeTY2lzo2OuTlNvPC+hDQt2wcQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT OR Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=14.21.3" + } + }, + "node_modules/@biomejs/cli-linux-arm64": { + "version": "2.4.15", + "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64/-/cli-linux-arm64-2.4.15.tgz", + "integrity": "sha512-owaAMZD/T4LrD0ELNCk0Km3qrRHuM0X6EAyVE1FSqGY0rbLoiDLrO4Us2tllm6cAeB2Ioa9C2C08NZPdr8+0Ug==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT OR Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=14.21.3" + } + }, + "node_modules/@biomejs/cli-linux-arm64-musl": { + "version": "2.4.15", + "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64-musl/-/cli-linux-arm64-musl-2.4.15.tgz", + "integrity": "sha512-ZPcxznxm0pogHBLZhYntyR3sR+MrZjqJIKEr7ZqVen0Rl+P/4upVmfYXjftizi9RoqZntg33fv/1fbdhbYXpEQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT OR Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=14.21.3" + } + }, + "node_modules/@biomejs/cli-linux-x64": { + "version": "2.4.15", + "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64/-/cli-linux-x64-2.4.15.tgz", + "integrity": "sha512-0jj7THz12GbUOLmMibktK6DZjqz2zV64KFxyBtcFTKPiiOIY0a7vns1elpO1dERvxpsZ5ik0oFfz0oGwFde1+g==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT OR Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=14.21.3" + } + }, + "node_modules/@biomejs/cli-linux-x64-musl": { + "version": "2.4.15", + "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64-musl/-/cli-linux-x64-musl-2.4.15.tgz", + "integrity": "sha512-CNq/9W38SYSH023lfcQ4KKU8K0YX8T//FZUhcgtMMRABDojx5XsMV7jlweAvGSl389wJQB29Qo6Zb/a+jdvt+w==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT OR Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=14.21.3" + } + }, + "node_modules/@biomejs/cli-win32-arm64": { + "version": "2.4.15", + "resolved": "https://registry.npmjs.org/@biomejs/cli-win32-arm64/-/cli-win32-arm64-2.4.15.tgz", + "integrity": "sha512-ouhkYdlhp/1GghEJPdWwD/Vi3gQ1nFxuSpMolWsbq3Lsq3QUR4jl6UdhhscdCugKU5vOEuMiJhvKj66O0OCq+w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT OR Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=14.21.3" + } + }, + "node_modules/@biomejs/cli-win32-x64": { + "version": "2.4.15", + "resolved": "https://registry.npmjs.org/@biomejs/cli-win32-x64/-/cli-win32-x64-2.4.15.tgz", + "integrity": "sha512-zBrGq5mx5wwpnow4+2BxUvleDM+GNd4sLbPaMapsSLQLD0NGRCquqPBTgN+7XkUteHvj7M+BstuI8tmnV7+HgQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT OR Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=14.21.3" + } + }, "node_modules/@csstools/color-helpers": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/@csstools/color-helpers/-/color-helpers-6.0.1.tgz", @@ -1870,6 +2046,66 @@ "node": ">=14.0.0" } }, + "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@emnapi/core": { + "version": "1.7.1", + "dev": true, + "inBundle": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/wasi-threads": "1.1.0", + "tslib": "^2.4.0" + } + }, + "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@emnapi/runtime": { + "version": "1.7.1", + "dev": true, + "inBundle": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@emnapi/wasi-threads": { + "version": "1.1.0", + "dev": true, + "inBundle": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@napi-rs/wasm-runtime": { + "version": "1.1.0", + "dev": true, + "inBundle": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/core": "^1.7.1", + "@emnapi/runtime": "^1.7.1", + "@tybys/wasm-util": "^0.10.1" + } + }, + "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@tybys/wasm-util": { + "version": "0.10.1", + "dev": true, + "inBundle": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/tslib": { + "version": "2.8.1", + "dev": true, + "inBundle": true, + "license": "0BSD", + "optional": true + }, "node_modules/@tailwindcss/oxide-win32-arm64-msvc": { "version": "4.1.18", "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-win32-arm64-msvc/-/oxide-win32-arm64-msvc-4.1.18.tgz", diff --git a/frontend/package.json b/frontend/package.json index 2686499..b3f7ed1 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,17 +1,23 @@ { "name": "frontend", "private": true, - "version": "1.0.0", + "version": "1.1.0", "type": "module", "scripts": { "dev": "vite", "build": "tsc -b && vite build", "preview": "vite preview", "typecheck": "tsc -b --noEmit", + "lint": "eslint src tests", + "lint:fix": "eslint src tests --fix", + "format": "biome format src tests", + "format:fix": "biome format --write src tests", + "biome:check": "biome check src tests", + "biome:fix": "biome check --write src tests", "test": "vitest run", "test:watch": "vitest", "test:coverage": "vitest run --coverage", - "check": "tsc -b --noEmit && vitest run" + "check": "tsc -b --noEmit && biome check src tests && eslint src tests && vitest run" }, "dependencies": { "@tanstack/react-table": "^8.21.3", @@ -23,6 +29,7 @@ "zustand": "^5.0.11" }, "devDependencies": { + "@biomejs/biome": "^2.4.15", "@eslint/js": "^9.39.1", "@tailwindcss/vite": "^4.1.18", "@testing-library/dom": "^10.4.0", From 91b0ed06410069da9546232b3f9df20ab16f5f61 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Fri, 22 May 2026 13:08:01 -0600 Subject: [PATCH 10/72] Frontend: Add services for job tracking --- frontend/src/hooks/useJobProgress.ts | 77 ++++++++++++++++++++++++++ frontend/src/stores/createJobStore.ts | 78 +++++++++++++++++++++++++++ 2 files changed, 155 insertions(+) create mode 100644 frontend/src/hooks/useJobProgress.ts create mode 100644 frontend/src/stores/createJobStore.ts diff --git a/frontend/src/hooks/useJobProgress.ts b/frontend/src/hooks/useJobProgress.ts new file mode 100644 index 0000000..eb8089b --- /dev/null +++ b/frontend/src/hooks/useJobProgress.ts @@ -0,0 +1,77 @@ +/** + * Generic hook that wires up SSE streaming and cancel for any job workflow. + * + * Both ``useUploadJob`` and ``useDeleteJob`` delegate SSE subscription and + * the cancel request to this hook, keeping workflow-specific message handling + * in each caller. + * + * Usage: + * ```ts + * const { isCancelling, cancel } = useJobProgress({ + * sseUrl: jobId && isRunning ? `/api/upload/progress/${jobId}` : null, + * cancelUrl: jobId ? `/api/upload/cancel/${jobId}` : null, + * onMessage: handleMessage, + * onForceClose: () => { setIsRunning(false); setJobId(null); }, + * }); + * ``` + */ + +import { useCallback, useState } from 'react'; + +import { apiPost } from '../api/client.ts'; +import { useSSE } from './useSSE.ts'; + +export interface UseJobProgressOptions { + /** SSE stream URL; pass null when the job is not active. */ + sseUrl: string | null; + /** Cancel endpoint URL; pass null when there is nothing to cancel. */ + cancelUrl: string | null; + /** Invoked for every SSE message — caller handles event type dispatch. */ + onMessage: (data: unknown) => void; + /** + * Called when the SSE connection drops unexpectedly or the cancel POST + * fails. Use this to force-reset any ``isRunning`` / ``jobId`` state. + */ + onForceClose?: () => void; +} + +export interface UseJobProgressResult { + /** True while a cancel request is in-flight. */ + isCancelling: boolean; + /** POST to the cancel URL and set ``isCancelling``. */ + cancel: () => Promise; +} + +export function useJobProgress({ + sseUrl, + cancelUrl, + onMessage, + onForceClose, +}: UseJobProgressOptions): UseJobProgressResult { + const [isCancelling, setIsCancelling] = useState(false); + + useSSE({ + url: sseUrl, + onMessage, + onError: () => { + setIsCancelling(false); + onForceClose?.(); + }, + }); + + const cancel = useCallback(async () => { + if (!cancelUrl) return; + setIsCancelling(true); + try { + await apiPost(cancelUrl); + // Terminal SSE event will arrive shortly and the caller's onMessage + // handler is responsible for clearing isRunning / jobId. + } catch { + // Cancel POST failed — force-close so the UI doesn't hang. + setIsCancelling(false); + onForceClose?.(); + } + }, [cancelUrl, onForceClose]); + + return { isCancelling, cancel }; +} diff --git a/frontend/src/stores/createJobStore.ts b/frontend/src/stores/createJobStore.ts new file mode 100644 index 0000000..dd34606 --- /dev/null +++ b/frontend/src/stores/createJobStore.ts @@ -0,0 +1,78 @@ +/** + * Factory for Zustand stores that follow the stepped-job-workflow pattern. + * + * Both the upload and delete workflows share identical state for: + * - step navigation (``step`` / ``setStep``) + * - folder selection (``folderPath`` / ``setFolderPath``) + * - full reset (``reset``) + * + * ``createJobStore`` generates these fields automatically. Callers provide + * workflow-specific extra state via ``initialExtra`` and ``extraSlice``. + * + * Usage: + * ```ts + * export const useUploadStore = createJobStore( + * 1 as UploadStep, + * { uploadJobId: null, completedJob: null } as UploadExtra, + * (set) => ({ + * setUploadJobId: (id) => set({ uploadJobId: id }), + * setCompletedJob: (job) => set({ completedJob: job }), + * }), + * ); + * ``` + */ + +import { create } from 'zustand'; + +/** The state slice produced automatically by ``createJobStore``. */ +export interface BaseJobState { + step: TStep; + setStep: (step: TStep) => void; + + folderPath: string; + setFolderPath: (path: string) => void; + + reset: () => void; +} + +type FullState = BaseJobState & TExtra; + +// Zustand's set function signature (partial or functional update, no replace needed) +type SetFn = (partial: Partial | ((state: T) => Partial)) => void; + +/** + * Create a Zustand store that includes the common job-workflow base slice + * plus any workflow-specific extra state. + * + * @param initialStep The step value on first render and after reset. + * @param initialExtra Initial values for workflow-specific fields. + * @param extraSlice Function that receives Zustand's ``set`` and returns + * the workflow-specific actions (setters, etc.). + */ +export function createJobStore( + initialStep: TStep, + initialExtra: TExtra, + extraSlice: (set: SetFn>) => TExtra, +) { + const initialBase = { + step: initialStep, + folderPath: '', + }; + const initialState = { ...initialBase, ...initialExtra }; + + // Strip setter functions so reset() only restores data fields — never + // overwrites the real setters with the no-op placeholders in initialExtra. + const initialData = Object.fromEntries( + Object.entries(initialState).filter(([, v]) => typeof v !== 'function'), + ) as Partial>; + + return create>((set) => ({ + ...initialState, + + setStep: (step) => set({ step } as Partial>), + setFolderPath: (folderPath) => set({ folderPath } as Partial>), + reset: () => set(initialData), + + ...extraSlice(set), + })); +} From 67620b9375dee45e44f45ad53fe3d5658b5cc187 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Fri, 22 May 2026 13:10:46 -0600 Subject: [PATCH 11/72] Frontend: lint --- .../src/components/common/AlertBanner.tsx | 50 +++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/frontend/src/components/common/AlertBanner.tsx b/frontend/src/components/common/AlertBanner.tsx index 5b802c4..95573a7 100644 --- a/frontend/src/components/common/AlertBanner.tsx +++ b/frontend/src/components/common/AlertBanner.tsx @@ -2,10 +2,10 @@ * Reusable alert banner for displaying information, warnings, errors, and success messages. */ -import { InfoIcon, WarningIcon, ErrorIcon, SuccessIcon, ShieldIcon } from "../../utils/icons.tsx"; -import type { ReactNode } from "react"; +import type { ReactNode } from 'react'; +import { ErrorIcon, InfoIcon, ShieldIcon, SuccessIcon, WarningIcon } from '../../utils/icons.tsx'; -type AlertType = "info" | "warning" | "error" | "success" | "shield"; +type AlertType = 'info' | 'warning' | 'error' | 'success' | 'shield'; interface AlertBannerProps { type: AlertType; @@ -16,27 +16,27 @@ interface AlertBannerProps { } const alertStyles: Record = { - info: "bg-blue-50 border-blue-200", - warning: "bg-yellow-50 border-yellow-200", - error: "bg-red-50 border-red-200", - success: "bg-green-50 border-green-200", - shield: "bg-green-50 border-green-200", + info: 'bg-blue-50 border-blue-200', + warning: 'bg-yellow-50 border-yellow-200', + error: 'bg-red-50 border-red-200', + success: 'bg-green-50 border-green-200', + shield: 'bg-green-50 border-green-200', }; const titleStyles: Record = { - info: "text-blue-800", - warning: "text-yellow-800", - error: "text-red-800", - success: "text-green-800", - shield: "text-green-800", + info: 'text-blue-800', + warning: 'text-yellow-800', + error: 'text-red-800', + success: 'text-green-800', + shield: 'text-green-800', }; const messageStyles: Record = { - info: "text-blue-700", - warning: "text-yellow-700", - error: "text-red-700", - success: "text-green-700", - shield: "text-green-700", + info: 'text-blue-700', + warning: 'text-yellow-700', + error: 'text-red-700', + success: 'text-green-700', + shield: 'text-green-700', }; const iconMap: Record = { @@ -48,11 +48,11 @@ const iconMap: Record = { }; const iconColorStyles: Record = { - info: "text-blue-500", - warning: "text-yellow-500", - error: "text-red-500", - success: "text-green-700", - shield: "text-green-700", + info: 'text-blue-500', + warning: 'text-yellow-500', + error: 'text-red-500', + success: 'text-green-700', + shield: 'text-green-700', }; export default function AlertBanner({ @@ -60,7 +60,7 @@ export default function AlertBanner({ title, message, icon, - className = "", + className = '', }: AlertBannerProps) { const Icon = iconMap[type]; @@ -70,7 +70,7 @@ export default function AlertBanner({ {icon || }
{title &&

{title}

} -
{message}
+
{message}
From 410e0d63b539d4b893f54a58739b256c24cb3eaa Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Fri, 22 May 2026 13:11:24 -0600 Subject: [PATCH 12/72] Frontend: lint --- frontend/src/components/common/Breadcrumb.tsx | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/frontend/src/components/common/Breadcrumb.tsx b/frontend/src/components/common/Breadcrumb.tsx index 9e408aa..bbe4a42 100644 --- a/frontend/src/components/common/Breadcrumb.tsx +++ b/frontend/src/components/common/Breadcrumb.tsx @@ -1,5 +1,5 @@ -import { useEffect, useRef } from "react"; -import { ChevronRightIcon } from "../../utils/icons.tsx"; +import { useEffect, useRef } from 'react'; +import { ChevronRightIcon } from '../../utils/icons.tsx'; interface BreadcrumbItem { label: string; @@ -19,7 +19,7 @@ export default function Breadcrumb({ items }: BreadcrumbProps) { if (el) { el.scrollLeft = el.scrollWidth; } - }, [items]); + }, []); return (