From e15eb3b27f2896a636bc7c7da07b3b1b9ba67fcb Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Mon, 16 Feb 2026 12:46:50 -0700 Subject: [PATCH 001/173] Templates: Add upload status indicators to index --- app/templates/index.html | 51 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 5 deletions(-) diff --git a/app/templates/index.html b/app/templates/index.html index b9976bf..0444aa1 100644 --- a/app/templates/index.html +++ b/app/templates/index.html @@ -88,9 +88,30 @@

Upload MCAP Files

0 MCAP files in this folder (not subfolders) +
+ + +
@@ -161,6 +182,19 @@

Review Selected Files

{{ stat_card("scan-already-uploaded", "Already Uploaded", "yellow") }}
+ + +
+ +
+ + )} + + )} + +
+ + {scanResults.length > 0 && ( + + )} +
+ + )} + + {/* Step 3: Confirmation */} + {step === 3 && ( + + )} + + {/* Step 4: Deletion Progress */} + {step === 4 && ( +
+
+ + + +
+ + + + +
+ )} + + {/* Step 5: Summary */} + {step === 5 && completedJob && ( +
+
+ + + + +
+ + {/* Result file table */} + {completionFiles.length > 0 && ( +
+ + + + + + + + + + + {completionFiles.map((f) => ( + + + + + + + ))} + +
+ Filename + + Size + + Status + + Details +
+ {f.filename} + + {formatBytes(f.file_size)} + + + {f.status} + + + {f.error_message || (f.verification === "md5+size" ? "Verified: MD5 + size" : f.verification === "size" ? "Verified: size (multipart ETag)" : "—")} +
+
+ )} + + +
+ )} + + ); +} From b98026765caff9beccb5015e0fec9de0c08b632e Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Tue, 17 Feb 2026 16:59:43 -0700 Subject: [PATCH 093/173] Frontend: Add Logs page --- frontend/src/pages/LogsPage.tsx | 78 +++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 frontend/src/pages/LogsPage.tsx diff --git a/frontend/src/pages/LogsPage.tsx b/frontend/src/pages/LogsPage.tsx new file mode 100644 index 0000000..8b0b997 --- /dev/null +++ b/frontend/src/pages/LogsPage.tsx @@ -0,0 +1,78 @@ +import { useCallback, useState } from "react"; +import FilterBar from "../components/logs/FilterBar.tsx"; +import type { LogFilters } from "../components/logs/FilterBar.tsx"; +import LogStatsBar from "../components/logs/LogStatsBar.tsx"; +import LogTable from "../components/logs/LogTable.tsx"; +import UploadSessionList from "../components/logs/UploadSessionList.tsx"; +import UploadStatsBar from "../components/logs/UploadStatsBar.tsx"; +import type { UploadSession, UploadStatsResponse } from "../types/api.ts"; + +type ActiveTab = "uploads" | "events"; + +const DEFAULT_FILTERS: LogFilters = { + date: "", + level: "", + category: "", + search: "", +}; + +const tabs: { key: ActiveTab; label: string }[] = [ + { key: "uploads", label: "Upload History" }, + { key: "events", label: "Event Log" }, +]; + +export default function LogsPage() { + const [activeTab, setActiveTab] = useState("uploads"); + const [filters, setFilters] = useState(DEFAULT_FILTERS); + const [sessions, setSessions] = useState([]); + + const handleUploadDataLoaded = useCallback((data: UploadStatsResponse) => { + setSessions(data.sessions); + }, []); + + const handleFilterChange = useCallback((newFilters: LogFilters) => { + setFilters(newFilters); + }, []); + + const title = activeTab === "uploads" ? "Upload History" : "Event Log"; + + return ( +
+

{title}

+ + {/* Tab bar */} +
+ {tabs.map((tab) => ( + + ))} +
+ + {/* Upload History tab */} + {activeTab === "uploads" && ( +
+ + +
+ )} + + {/* Event Log tab */} + {activeTab === "events" && ( +
+ + + +
+ )} +
+ ); +} From 1ccf1ece0b2255422a93743b0db58381f604ca40 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Tue, 17 Feb 2026 17:02:31 -0700 Subject: [PATCH 094/173] Refactor: Remove deprecated jinja templates --- app/templates/base.html | 399 --------------------------------- app/templates/files.html | 106 --------- app/templates/index.html | 430 ------------------------------------ app/templates/logs.html | 169 -------------- app/templates/macros.html | 16 -- app/templates/settings.html | 241 -------------------- 6 files changed, 1361 deletions(-) delete mode 100644 app/templates/base.html delete mode 100644 app/templates/files.html delete mode 100644 app/templates/index.html delete mode 100644 app/templates/logs.html delete mode 100644 app/templates/macros.html delete mode 100644 app/templates/settings.html diff --git a/app/templates/base.html b/app/templates/base.html deleted file mode 100644 index bbdf0aa..0000000 --- a/app/templates/base.html +++ /dev/null @@ -1,399 +0,0 @@ - - - - - - {% block title %}{{ display_name }}{% endblock %} - - - - - - - - - - - - - - {% block head %}{% endblock %} - - - -
- -
-

{{ display_name }}

- - National Laboratory of the Rockies - -
- - - -
- - - {% with messages = get_flashed_messages(with_categories=true) %} - {% if messages %} -
- {% for category, message in messages %} -
-

{{ message }}

-
- {% endfor %} -
- {% endif %} - {% endwith %} - - -
- {% block content %}{% endblock %} -
- - - - - - - - - - - {% block scripts %}{% endblock %} - - diff --git a/app/templates/files.html b/app/templates/files.html deleted file mode 100644 index 4330026..0000000 --- a/app/templates/files.html +++ /dev/null @@ -1,106 +0,0 @@ -{% extends "base.html" %} -{% from "macros.html" import spinner %} - -{% block title %}Browse Uploaded Files - {{ super() }}{% endblock %} - -{% block body_attrs %}data-page="files"{% endblock %} - -{% block content %} -
- -
-
-

S3 File Browser

-

Browse uploaded MCAP files in S3

-
-
- -
- - - - -
- -
-
- - -
-
-
- - - - Loading... -
-
- -
-
-
- - - - - -
- -
- {{ spinner("Loading files...") }} -
- - - - - - - - - -
- - - -
-{% endblock %} - diff --git a/app/templates/index.html b/app/templates/index.html deleted file mode 100644 index 4549e2f..0000000 --- a/app/templates/index.html +++ /dev/null @@ -1,430 +0,0 @@ -{% extends "base.html" %} -{% from "macros.html" import spinner, stat_card %} - -{% block title %}Upload Files - {{ super() }}{% endblock %} - -{% block body_attrs %}data-page="upload"{% endblock %} - -{% block content %} -
- -
-
-

Upload MCAP Files

-

Select a folder to upload MCAP files to MODAQ Cloud (NLR AWS S3)

-
-
- - -
-
-
- -
-
- 1 -
- Select -
- - -
- - -
-
- 2 -
- Review -
- - -
- - -
-
- 3 -
- Upload -
- - -
- - -
-
- - - -
- Complete -
-
- - -
- Select files or a folder to upload -
-
-
- - -
- - - - -
-
- -
-
- - -
- - 0 MCAP files in this folder (not subfolders) - - -
- - - - - -
- -
- - - - - - - - - - - -
- - Navigate to the folder containing your MCAP files, then click Upload Folder. - - -
-
- - - - - - - - - -
- - - -{% endblock %} diff --git a/app/templates/logs.html b/app/templates/logs.html deleted file mode 100644 index c940502..0000000 --- a/app/templates/logs.html +++ /dev/null @@ -1,169 +0,0 @@ -{% extends "base.html" %} - -{% block title %}Logs - {{ super() }}{% endblock %} - -{% block body_attrs %}data-page="logs"{% endblock %} - -{% block content %} -
- -
-

Application Logs

-

View upload activity, analysis results, and system events

-
- - -
-
-
-
-
Total Entries
-
-
-
-
-
Today
-
-
-
-
-
Errors
-
-
-
-
-
Log Files
-
-
- - -
-
- -
- -
- - -
- -
- - -
- -
- - -
- -
- - - - -
-
- - -
- - -
- - -
- -
- Loading... -
- - -
-
- - -
- - - - - - - - - - - - - - - -
TimestampLevelCategoryEventMessage
Loading...
-
-
-
-{% endblock %} diff --git a/app/templates/macros.html b/app/templates/macros.html deleted file mode 100644 index 117c521..0000000 --- a/app/templates/macros.html +++ /dev/null @@ -1,16 +0,0 @@ -{# Reusable template macros #} - -{% macro spinner(message="Loading...") %} - - - - -

{{ message }}

-{% endmacro %} - -{% macro stat_card(id, label, color, value="0", size="2xl") %} -
-
{{ value }}
-
{{ label }}
-
-{% endmacro %} diff --git a/app/templates/settings.html b/app/templates/settings.html deleted file mode 100644 index 8e61dfd..0000000 --- a/app/templates/settings.html +++ /dev/null @@ -1,241 +0,0 @@ -{% extends "base.html" %} - -{% block title %}Settings - {{ super() }}{% endblock %} - -{% block body_attrs %}data-page="settings"{% endblock %} - -{% block content %} -
- -
-

Settings

-

Configure AWS credentials and S3 bucket

-
- - -
-
- -
- - -

Select AWS profile from ~/.aws/credentials

-
- - -
- - - - -
- - -
- - -

Name of the S3 bucket to upload files to

-
- - -
- - -

Default folder to open when selecting files (optional)

-
- - -
-
-
- Connection Status -
- Not tested -
-
- -
-
- - -
- -
-
-
- - -
-
-

Application Updates

-

Download the latest updates from GitHub/MODAQ2

-
- - -
-
-
- Version: - - -
-
- Branch: - - -
-
- Commit: - - -
-
- Last Updated: - - -
-
-
- - -
-
-
-
- Click "Check for Updates" to see if updates are available -
-
-
- - -
-
-
- - - -
- - -
-
-

Upload Cache

-

Local cache for tracking uploaded files and detecting duplicates

-
- - -
-
-
- Cached Files: - - -
-
- Exists: - - -
-
- Deleted: - - -
-
- Last Sync: - Never -
-
-
- - -
-
-
- Sync with AWS -

Reconcile cache with actual S3 bucket state (marks deleted files)

-
-
- -
-
- -
-
- - -
-
-

Danger Zone

-
-
-
-
- Clear Browser Cache -

Clear remembered folders and local preferences

-
- -
-
-
- Clear Upload Cache -

Delete all cached file records for current bucket

-
- -
-
-
- Reset Settings -

Reset all settings to defaults

-
- -
-
-
-
-{% endblock %} - From f99be6f120356c9d12dd3e17344efcd9e74c9b87 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Tue, 17 Feb 2026 17:03:13 -0700 Subject: [PATCH 095/173] Refactor: Remove migrated assets --- app/static/images/alliance-logo_black.svg | 59 ---- app/static/images/doe-logo.svg | 339 ---------------------- app/static/images/modaq-logo.png | Bin 3833 -> 0 bytes app/static/images/nlr-logo@2x-01.png | Bin 23341 -> 0 bytes 4 files changed, 398 deletions(-) delete mode 100644 app/static/images/alliance-logo_black.svg delete mode 100644 app/static/images/doe-logo.svg delete mode 100644 app/static/images/modaq-logo.png delete mode 100644 app/static/images/nlr-logo@2x-01.png diff --git a/app/static/images/alliance-logo_black.svg b/app/static/images/alliance-logo_black.svg deleted file mode 100644 index be59986..0000000 --- a/app/static/images/alliance-logo_black.svg +++ /dev/null @@ -1,59 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/app/static/images/doe-logo.svg b/app/static/images/doe-logo.svg deleted file mode 100644 index 7075378..0000000 --- a/app/static/images/doe-logo.svg +++ /dev/null @@ -1,339 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/app/static/images/modaq-logo.png b/app/static/images/modaq-logo.png deleted file mode 100644 index e26d9d1da61a979507aa16403ce3bc64b74c6e80..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3833 zcmeHKX+IPW*H#ksr_GvaY=y`gV`nNP{7Frg?7J|w>?SjjT|#zaschMW>@&lVNn=l= zF=O8wgF$0Ax90;qpWwb+!gi%d*R9Qu!Ro}R|<|7UDx zPrZnmY9ABRrA~dF`_Dr1*2pkl$ap^MrYB0e?@^}LH`xe5`Lmo($SmQg60s|sd8N)) z_W2J069ZxcN>FueS8QhgVDV%rDJMw08#?Os;QgSI@a54Iu_XDZo{tjp=h)z{zt8|j zSXpAGi+<(;5EtLFYrSevf1LPxZCByws-2yU%_@AOb@^z}kBf{>yU}T#rh9m9hH|0n z>@_x?lzVlr&qcD_x*{bMVSnbrxqluBS=>AO-<*n?s|-c5X67_o7|l^Rr}`#+l-f{f z*J@EtT^*XuW#^B6(Y3Qmrh~OJ9jt7hriyw;JnNJJD`_9fI;Eg-wL~~;(yZDdMnm{5N6}v`6t?j zBG9cWq&g{6p9Irh$+C%ma8b?OOBK-TH7$2{;6GC)sfjx~uJd;;n%z%;pO){m)7*7W z-r2RtEaY1IYQ1SMx|)u`$hvuYpKY)aPShWk+-S;OZ_0(6X)u}v?O;_Mpx^h+N^vaV zo~bcfxOr|bJsFOuR)lT+x?UwSS+3jIN2e?7RrmoXV97J%rE9yHgk5^EO?q*7%k7fM zr4CATTki>N^Rl|(2!dY ztd-F6XK8g4*RtF%Qq=a3oxLUuNmYJ-d;#-ET*vW(OV#;cR^ptB_x1zGJ9I)sTPlkhY`Tz;J5^^&h)Q9CX$5fkOM*>@gg_ z71B)Jb25<9&r{)&D9e-B*27*Gd9sgT!}udHkhrkYcoheo1mlT*(fB^#hd)@R|LwR= zn77OR(j=alHOxC)H(aXstAlyleJaBbkKLOn=*(YvYFl^EOV(G*=yCABrWlK9w-AF; z^980oLE*LI^nA3M+RWy{trrOmNrwchyHVz%Bre#C*sp6}&#x+Aw(X5#2S6FR85;U! z{UFa%5(50^a&F@tTNqLUmdDV*x_NTLqs*1?hK z{B^By7e%}#&x@(>+P00kyqdIvk1P^#bl8y4|sDh z)9haiURu)?x8Pu&*X~d+6*9sEb!pJNF;jW7U@-D01Xyf}Z0h_xm5cliJ(?}#9||OX zQzzw3je@PKpkL%kf)WN77?3^FCUzeU3s3)gug-ajbeQRDpS>ol#^`co@9adBQ<>Vs zWSi@A8~zBi>Ti-})1U~iz~7&qMPRj2uDTcgpGz5C&HOnA-2`$^#_GY9LXV(NoK-3r zM@41;#w~c8QW6p8Xi~lWD@D=@y2V#6JP-rDxgDrXS1)E|JTCm>*hEgfDl|RY5;94f zDXftL1WJacigupKnf-HU1hM<}MeKfElgx#Qfe9&o75i( z1x)=MICY1gW!h;~1QKaB?0%a&h;)ZPw$Y%x0Tr$VyP%z!uJxj+RU;CqYFur>W&7pc zny(FFTgdA64-l&g|MrO1i-L;;J?Y4ajBg)g1~1CZ;SC956c(V->5{SDb4idN5XSit zdalF!<#eSje8&E^qm*ST9tx9$*W9Ty8Nzu2(}2L^QC91M-X^MeE>Rrl)U${ZskrWn z6cO;)2V$`3B2X&upt`B2dTx?AC~J9T*&K!TIr>y*v!6VWGuM23(D}~Q#U$;UGPOJL z;X((JX0N0Lqi^6iR5S#elQx^7bKE|S2G=p;f3xs!e%kn4=<{16KiXv@edMX!@RzfC zzylBsU~lM7w0I)uxzQGIQkv#<^W?g8=?wa=rm=OLeh%(kBG_H;%f!W6q8;hdj&KjaN{7yF9vm*d6Te zmA-Pr`8{(a5y@q{Q-5?z)F*Bhxo~2Qm|s^5b9rH*(wNQ%c&Hj_}f}M zmn8|IKgg@i4evS*fy5Q8Dwf51!LEH>>wW?)NC@9-?NG2|N#Vz7m91f?E<>KyQh@Wd z@xCx|JGE*2^hY^UA9=QSL`h3+hv6uAP-10ykg^v9xG0g2`(d>QUbj&DWkPM$%&=R2 z)!HVTr)>jpg*_u)_}D(H`zfB+uUz5P9UW#iIJb27XWK#vzLl>WstoGj{t-g$$>n5; z62k76q9k8S5N%IF7yf)^7hSIf+*{v8K*{CSwMRWWcK+7$a_ z(reiqsQQkS4PHO1wFBA!gjs7im4M^Y>GH^8uw!YHj<&=djd)Q)Ur^+UZr=S`F> z{}Z1f!$!WbYqG=!v5fL^vnKZB$NdXWvOh!6+bB_k*@|=s_2v;VazhrWA6x$2amM`V zV9*5e{JEse2RseIKr7p&cF_Ipe9~{#_~vd~WhI4?%LEwq<8ZSN=d*^M43zy3K#T_V zxqP+3$us?jb;sLjfsBqA8}ioj!FBlL&~#RsgyievwGbo&11{{%>HBe`^=l@%fMFtxXkls zc0*k|s;H)ukxlD9W^p}f>_HT+x_S$jSO+<~)r1Uoa#e=7;6yy%e|SWp8J*lyJXBo% zM8@UPUj|N}%_BMyvS=r_zQi(pOo;B}4!9$n$Wo_kNyhE9$l}WZQno>NC|6vYqtDCOUF? zW@b+ZJ5(8o!l&l99u<3V)l)OHm0w|l!Ju4`|HY9);?@DsBp`UWA zR6i~4!~UUY{P`!sJDue<$s*PeFtf65-8&`rgg1*jdzT8R@hd7orEwcsLim=i)=#dS zzZH}Cv|!n+Fe)c*!UM}Ex@MCl@KUfgne;bIo7X2GbQc>^BKHc%Us8%RZsJfdn&I*6 z09)L^QNdGib7PI4T;<&L(q?Hpqu-l^$Kg}VxyCz!(U6<@l|^3K(9qIPe%}}4KeXo3 zpJ#>d&Ym~t0|leA26s=)x!Xq_TyxL&Dx*yNJM>eKKJhVd@!XPwcX|M#?O4p|L|F#o z@P|pu9z8dEQsB>tv)Lmnx1OGM(~(9-U!*IA70HlkLh~j@jNUh2-Ma4(#_fWkuoC$0 zZ0{q1)(8%DO~#3X-fYCS;Kgx?TNG13wC6JobYHNnAq!u z!hpNS((ag+J5KJK(KG%lrEEMnzgof$4Imv$IyBaVjmK=v(^E5stoM5TL8qVJIVr)n zt*8wUyVh0^q$^@OR=XO>)=T}%@$PZnMk5&V>+AkE%lWjzHSmdM0Ge*J@6em)lW;}X!|W_`Q2%?hj6sKwua-P ziubZ-R(IZLc1|s#?xaw`V?j1t{2r)heQRnLXE;}}Wu59z3E~9ud!IKRTBrntdxWxY z^TtU=Lvf?ULzKO%jKVWmKS(;@^#n%1b3-6s&Yg%yDLxNY9IIc-=_9863MhRI4q&>?#`;8fBw>FXNn)I6|{{144&Oq~D# diff --git a/app/static/images/nlr-logo@2x-01.png b/app/static/images/nlr-logo@2x-01.png deleted file mode 100644 index 8e21da089a9f09c41a2246488d2af45bf4eed3e0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 23341 zcmeFZbySsI_cjWMG)PG|0!sI$MM{vA66xABY^1wELP}Z?qy?l)x{q3Zur*r zdGnm_jPrZPINuoOkJn*v*Sy!7_gZt!d0p2sOhxHAHU=360s;cIoUD{80s`VS_<0Wv z1^i8HZuTDh)9sLusIHmSzYDZqv3Z&u}_v&>D}zy#tV# z#d>P^Hy@l{Ltpk>b`|_+_mpRqt zW|E3pPOJ^fl(DXROPR4v;mClj3x~~+1oSK^2~Yde@bHhf%9 z&$w2>nv@FcHvx>;a^Vr#i`w@4Cgr~um%kJR7SW*j_R@+% zWtWnMw@9JV1K%%i#*8t6(D6c}eD9%A5N8lg7I)=4LAe(lu3mHWtrHCbLx)e#)=5+_ ziS8lMy$K-AQ*3-DQfWz^b{0neg|(=2y_0}ZFXlXUE4&AmCeMy3!eea4bJU4vT>i7` z1#x9X0iH5JkGf*M7wvj^uX{wLW^v`I=mloSeQ#*W7H_t!5uSVujl?s`hw4W3N<~WH z!B{&?-q+bB26RahB^V@vL@I$OQzwCUZF_db%Rh6H2$hhUIHHcYP>y$yibGbssD+ib zkkQ2Mz4GWWutuuP$bkrHZ=;dSqOg|;f2cs!-9?~R4zcMiAa9ja(tQ7N zqH%uuO28YT;-|(Xihuxew|sol?$NG_E9@@a^6c#13vq|UIpyeVFz`Gq!2r`zR1h+O z*|Hm%!i=HpZnpMdm_(HKL`E$u|<_8MB~Xe>=d>0a?Eaw^(OLM<$1 zJshBF9!lya9@Zv;rgTrmFhtyhfB{>mlM#)Zt&N?dkeevoAG<=}XZUFjI+{PXI9ZF* zX(_7ENWvVTG`#G*?3`@UZk8_GbYd7ZA`YfzLaI_Se=`A&MCmM?oa}`-I9y#_*lf%}Q<6oa}bdq)fM*h~I|Lqfw>R_I7 zs6riK&JHF}X&0!S6aBxMcrK@?@(&B}E}2`}+W+YkaNfVY%+%x`_t`r;*!;Q2)Pw_S z1GNQ49YOoJ{_TDzOS6AE>)+Z2zw)O$|Jo4nx_{XJx4Zw|_dhp-yA&0Lq+lk_@R!O- ziPFK_D`W~Yu{0I>^CLunmxmwxVl(A6=3(RIF@>@j^9t~=nVA_2m~fjJo0%A!{);I& zJ4YuYI}<3}6fn+i3G8rkaq$=%37E0*85x_g@tSdRvkCC>3$h6am!`@*45*37T<2dCiRf zn1X)?p=T;`qIBHsoc}zdVq@fF26M0#rBgIAp;1=*=M{BJTd10o5!`EB5Pm*xE*?Hk zK7IjiK3?8`-lPe2a0DX`Zj_6YornL=6;l%-8DP=~j2uf_BXcN+y`A}=3-IqE1bPEJ zYXl#t!1AB-pff^}4p1W}n1ebDW+O@mZ!8VmDoEZusn>5&7$z%UU{u_q+c&_1BbAgTDFe=&wT? z%Rh68hUU+#5Hd3Ps|$`sF3`U|II#8CB@+uHJ98)qAAcLKe?M>ezZ(mD+=4u&y!>pO zMgn|nyih(c44}N+Y@7o8{7@5!fRP|S&mS-R!@48P%*oZr0s71w_!0ODjL$#5qIvw+ zCw%hnx4T+E;ok>%jExI!_W$ZJj=%Mo10FU0-enPv|C>`p{@m~{VFo<+*EtZrKs4m| zM>zbO(?Em&PyYJb$M`?_3L2XKzT`jh?|0wJCFvsf^v!5e_{fZS}4Zy&!rHs5r`2Uvpq0w1830fWnViYAh5B+{~$^T zpVESps7`W<(x{tQXn6PU@6c_BAt2Bo$Voj@cbndsGj~&mPNVMM9m}O0&wB>O$e`2E zQRaqz(!z~ler%}Y?3|=AG3WehZUPA@>(eBz)Q1s5X>pok{1Nn=B)L84^c5}UTEFA_ zep9pexVsC^2mWyi!h(*w`LjbeXi`NFw#3nBgJOIXW%K=TCD1bcl))h`#(B{H{NTQq z1b9ZJl?ndL|LoxLA_4pvBr)(ViT~jNVg@izOSq2?f95|q`2TUA{_6VwbDu=zy_meU z3S`m9pUiYyYFZEHjC@$-+cq^0oo))!E|S!X=(fpd(_{{idi3?=PBRh*yEsI&B~TZu zZt~E!k+GVHZ@_B)%#P*Zo2Uf-Dv5la8@`=YW2}>M$uDn_#7-T)&b*+-!}v73`Dy&O zit0po+Hb}*j>3-MEbYqcrY;FYLT)_A+2oe6a$i_6PUCt_!xsj+Z=FaHx~1IZxKR$f zYE?C{XQ$=GfTyGI+9D z?r{N(??Tt->>G^&+MD{5yX{`apcdZ|8{eW8d3@ZJOyIG4BJAsh8_H_7JP335eL&mi}~&_6R!esI*bA>Jc)LzO|g)aU8d#knB7w)cn_v)s%|z(+5}?-^AMV^osHhLj&_e~|I2)i3aa9Ghl#z#gWfAIk#jzgp9 z6T-f;ytBHqQybJKu1~h?o)?_dB9$E3qw8)LKlmD*d3!!hTulVw4$i*Oi$F1fWI#5` zH58dEeSMf%(0yK$L?hbSlKex~wHz2}2xs z=w+URH1meO$1)E!clxBlE1LV(3+yya%nR{_)e=Z3H$wT;I#MpKCJVwOyF>Hz_N3qOosRJx4RgU3|YZ)<{QM8tAo`PE38eTkz~yv|;1A6!cZ# z=parCPia}oL(RIBqDzzghH~g^j#0MD>nX~nupbbqjttg^Yaf&EyS(arZ{D`AcY7@R zQ)yMsCv*ii88Tg&R2dn_+_zKbTS{eN`1~`K<-wbZ_Rg5w1e=r79F@@ESy^|He6rGO zrvcWX((&JAPOfBXfo)UT0k3v(HpU&rYM&Cg^9_Irj4ZTVAojx~opd!s{CSbiU5l>j zTQzsSbHN5A($&bc1?QLu%P=aimRu#ZPZKpdJ&q7;w$m1vwaICkCf6A}?mk3BP-JGS z(PcHGs-!EwV;zn-Qn$xPg?ZeHGgg`P2lN%#bwBlOXyZpl^>i&tG8p~Ry{u=ckTtJE zAc?zXGgq{JoU$~#wUV7vlis9+YV(VfPdj!)Uj7&?unR3-E}3XoJarFSb!~IN=*gOx zAH5e_k~TkC-OG6WUe9^!j(m`i(;uc9%G0j*<*IAJ^jRaZ<902%29X?%yl=KTLV>p+nSbd8tVep71z|i8%bmotO=4->ti`0V|sZZI7)x9d;rf z2LEhbTTY?YSUX(!+%QO=jT^*@TcF+ye_QW93Tc9L>xI|eI*Bp0n!n;-Vrt6a@kelT zkiCfWFgkke{It)@DbDTke8M@q7d@|){i5ShAmXB8 zjW+Jk$Fmm7`$F1nYP2g9)?*ea$nuWUse|Kg)XTODJ5=tHMJSlAd7RmtzSPA14V^#U zH}06Gc@sJbEvhx)p&@9+#SbY+D`2`Vz2n7$zZ4TJCW}{u4C~32)yU+#dsEXJLtD^1 zU+01#IaXx9d#_i&6)rOBJTavcq4M-((!sVjVL?srE#lf547@CfALk;Ue{4#JoLrW( zYpRbu&iIBL3Kl*cGig$dy9IR+xmO$K+rD6!{?>ya#1z`JhBaCZa>c6nU_chS#}FBnWIB+7f>asiAmC zp={IFFUYTf-%yvVNpHpNjfv^Pz4<$_-7q8cn zMO_B3z8{fAF%lVBXG_29Hf62H-9eN5mnMX#j(TTDK_jcICfF0_ zOQ)MtjPM)rGTxFp`&;HG&%BR2nkU;-*wn?#KKi;aOQUiz3C= zR~T+%_S#v`;UU9nHJbGCkFCvjm@dXh6@mk!TagbZstsR#sOan293r^a>TSf{)V$l! zz^mQd9ka^us=3#JIdYtJ=%zb%>uVuIm0;B>@YQov?d_vk^55-Ka^-TlUF#wzZ89bU(FjnXJPvo3Wx zIC2+!{YPwdigCrq`HvA=b5X-->JA%avU^1kTcnG-uJ_bZTGr*WuepBq4ANz+PvOnJ zOfUYjtv`vA{D?PYcUDuAIVKTE zy)cX~Ix|y7FQ{&+dXAXmAH-vCdxA#@Gz9im+g z3Bk94l}SS>%0Oq47Ge64u^bJ|Kgq!~CTc~D&OmXbSo@IwW@PG%mLG%c>AwmFw`E^) z7h&2>cx-shRLm1tOohX69Jq`{>g1e+C}VVN=h=+!oR3ikAC5ajNq?POj4W?6+ncL} z#WAiNP!BWX%PiWQ4p^k}EUyo!C{&bcg(!L>`+0|u_&%8o|R`(4&1+#Q*lNs#R zxEa5NO?_r6s)((<^?&kHo04c-p=)YYQV`v9C~ChensiklQ4M-%{dHRQV1vd^l4D%W z==h{F2~zG>2Dv?$!R9Thc5P3Z~OgzdAWcMtu}esuHBop0kv{ zygLuCld7BIlx*zhjbDF=f$ZiG3c0-=t)5xag;ZJX@9x!|pLOEq`5(_|B~J%F|1JFwrrw57G%cN&+-Lkl4|g*{as2J0|Gw>Ez$b5)q8OwD=?^eFGQ(^3d|A>1+~N2(pe1cipkj4c4VJL=*xU9ZKCJ_rGfy# zDW(?5Uhs|MN^)_K{FY9+jk0#Mw@k6pTZV>Fm4!m1`Hb@F(~~3@q}CpDOa-54!Lb2> zn69}GTC9Y#HNhX@bvu?z%_pM0>-0WBMm+R)({Fv8KGEYh);%0&Px`skI8U~-yUbGI zAUrPFqNV#5>N%PM4F+=@XE34zQi)188QS|6ect69`b>m>Fw|6&kGG&Ixrr@hig9A3 z($|w_FMM z>QndTxLA6zL2wm_(eOW#u6|q$la|iLpz}xA#dtDnn?v$&F*J-;DRh^nt>R^FTgS}% zl>UqKiUVERJn65-4i;zlr=Scl!GD0wHeeIlF_j?ua@@pkhI)NyOeP!^_3Khm9|MYL zPmwDsr_Kr4?|kb784Zj_U_wF26J*|YHK@fN|H74(?mlrqqAccQc-MgSCGT}4EN-9{ z>kIhA=EULq75a8<;Zhkh?2GqWFL~~(@K-Izk;eARhRZ%~mGwn^x4pK;e!rT{%}s|} zY5hze|AP{$NF5V)Qo2%5cETG2Md1_@N0YiyKC)sQkdvjHh}sFL!K;Zn>;Ti*~#M1ah$uKSTF>uAZJCsAoair!>?CVCE2Uuh`HW!&>u(Ol@&RVFo?e1(Xh zd7IouNQvZEfpVIhpcM+A6)S6G_lM7hH<8TT&5LAE%T-!Q6?=;;&2H!O&YO}PSkN{( zkh8|#Iy;fc`Yv112S-%@(mZLuB*KozF8*w%Ir<$yVGlB5ZnM5*K7?Y&FG{1)?hI4> z0lNMZ(CTkM*M9`W{RMQ*W_G%cygjK->UehB^L+fbA0gB`!wsg1PfMp~E6}5wfNh~^ zJ#cvK1K^)YS*%OFVw}e?i{Of9!J3gZJ%j$m>OmF{bdZI#+U7Jd!ekK;R4kp~(@@Eo zpd|ihcC&^_;)G*mY#Ws!%tXjpeQ z(`J0VWE!6>HC-imJae&4c*wLwZnG4oISP?qI5x znQlk2pr?Hy=kPi>&7|lQYB;xvR4H-?60vfd&(SZVhd2F>4^h(ki`q)m8F0*BNfeKz zoH#aU$#X)+2>`r9Kdy!tTXrN|DpS;vyDVhWq8-k`ksvESOgr0ZxC!3dPm27j5E>9Ax8QZ=O6Hj%c++)#^WKo27gr139Lm z{N?{VYmRGnh)VV4#J%VODE2v+xNOAXF=}%*Bc5V)PTz{VbGg*XC`Wq9_Dy62K|TI$ z=EqBG=gA8#EU)da*nPi7Jh0~MQJ5nwCkCzslx;j{!X4qUDdxmn1r3ash>+te1b+lA z?#@)fX71sM&*^U&=3fEWhKRv!si&^|n};3pTBlp`swlu$+{_&wS7cC!xiY?`Ynv(Q zGkkymCN6v-7|K!6_4GlW|47kGTD|#1NfTcI-L(zARJcRwsG?kIH@1pj9GgT9kp4n9 zxBhPHIoH=I(dgic05^9sO3x|uWB3y=_p*dv%UhXdB!kj_$V`R{v6@L%3JL45tg9&d zUd+L^y%#SV61#_Q^anG(^yxnS-mE#>a^vLE?~$u_3K-C+(pv^{;3X#$29TI%JuzZO zbF49odMl)3q8p=@;hxca>R^+?hhWf8hEQI1_0resD`rUvS=74Bd7BLL@6#hKc_}xS zAZk@j?4Dz^34LJ>qrvg>%S?YQVZPa%qmft@b8GOL(t9jnxXPXG^NADJD%8Td9v&p~ z6x24V+0v|phIU_=fJwV!v_wHUX$cX*PI9@v5W4BP#G^`(FPtB$avdc@y6@Be7%rSC ztxL@Jex|pUS*+*ZkwjhBphoxA4sr$nzr_cZ;4HZ0(yzY2qUbzc$j1A>cW? z+Ms!08V^~fZkgls=v*9l)8^)`E}oQ)yc)?Ie*^nEKPJ;VJ2~xGo3MQK9ChjAg5yC_ z$HXTdqxj>dIQLrpT2Y+>lIBzH^U3dcd%24r9~kieL?xr#?{CJcy#1)jcquX2hVE&V z%rXBo0e0}@zN5BKCpq;j@%$>MhhimJE_ShswUd2KV(Xa9$dY?mLU@^rl>n%QTC?;D z7+1Bt-V~5*iin!lX(1bUl7fOe^gyL`PP6w~$uE%NsVLrC+O45apXMkoE3bICK%UNn zZ^VXYH1N}2;pXuD@ZLc)C$WftpRn|3*0}Na-Kv@2+BAgwdyn-_mSLy2JS=1bx6?4{ zp`AllieJXp=BkguFntr%BEpOSfJvDW|D${)#4UO$tiW+J^$X zWTK^Z^hPf0I4lpF?7}1Qt7W}gH)~_Gy&(|v&re@G@u}DJHxV4OP=6E06O4qo+(?*A zxE%EQ-YkHb4bxX*EnL59$=lyeBsEYTyXG<&{-!li3@ixQfETJeZPeInOmG_*p=T1iQA_tQz01PN- z=H*SwL19R)3P3LU*;xtMoiJo0)ba+bG1u1P%@qFFj4GSKRN?tH5S0z8u1%l^PHGdH58h zebGw9hLa&YjeX1p^(+W##~yl#pT;)#^8-^9Tu$RTi7Q$C3DECrl|VRdZoi8>TAi?u zio|wk7jJT3X*T=Rz&5TFX&7J+>hs+xCo0&r$K2=DL?OyDwmL$|@r*W5Z}tySQ!Iug zgf)vEPKgO5L=)7F^)Ezm#2%%1lcr_*AndDu+n-LR3Dn=(MR?8{zRsU!`8Y$RwFR3s z^eo`xc64iu!RTzwPuWt)4h^x1(!1xS%@bMmL}uD;^0a{oQj^f^wUsOS6!IJ3j@g^< zf^muJtD2CNUF9L3j&(47j69InXSKs$WOq9I&*RaOyGb)&+bU- zF5+~^X~r$>N7r~WE#irH3ap*=n@jz!VIha;_pim2Xb{pV+MDEHB06Z`J=o?L$>W!! zO^s3y5PpB)>o>Qr`7P3Mx%v`uU)BCCbpm#Aa@`3=A@*b4kp!?F5N(^rRf#<0ZuF}4 zjZPzOHd<9l*C6iJ=1ZHAoJ%Q`0!e^N&f}CUHb)cDMbj}z(#AmakyJn}%$;|QdjCdNRXuxxX*9Do)qWH)cD5{E-Bhe@7oT8nnqF0)cM5%+s zZ+y(y@SH!)PUuKUAnKEK@i)RphI;OWM=pr_*PG%*j@SN=UrB9hlvVZfed75|wn#o5 z?NYmA-!(|{Ku1J)HrkSRSVb(;&zJlMNBzN^hFxhSY;ydk1ZCdWz6Jl+WUd;ohf;N& zN3#P(l8v09k}S<@a=P4OS4>AKL0rKKLC;a^bgpNS53tBnwa1T1o94X>D-f9uZ~v!{tW^%O(s}v6#ggt=06#E(zit|V#mT3n;be;Ybp^6z9Fw(=rgHrLMYdsK zaksY;fcxoH_$#nxj^2CO&n-jN%LFQmd$dZL$++zRTHeKiwBDuZ`M-16O}hXD7FBB8 z&z7ae!8rBfhss>Br&MI$5*_C3^IC1L?p3{-$?Dclr#bu~?i!Jix*2P8xV#zg!R~2( zYGl0og7y1DL3!+m-`e{oD6aEDt}t9ny0+%Wt?&@g^K3{5kT+i^6W&r7^uA=}Q~$YaHM7i!{efp6`tl zZIpuLwA?!#i0rax+6+qjZDu2&Gr6>qLq{8OQ6?w5kRShgyUE8jK{1VN+1Azi9+4v` zWNVp%+lSI;=;c|gIW50>0P^MAOw&3T)6I45QQ;d9d{0lp(4oiaoz));UPLs_ryp}O zW^~BYN|xOx?*-`s7xO32jHA~(Y(0p7*8-$x-hc@9=I)Lx7M{cOZHNyJ>HO4Yo&Wek z@}lz`2jgQvIy?q)BnnT|*90yNfTiHz>`hzRVI?4g4nQRSVHiTWlIX<)j*5ar^V$ zMkK`jM2a3gegQI+v0=elg%h8zu8q0&>ckwHFrM^!VA)@p$jt+`QtIm@O;h99TMye@ zI)P;b!g5tSzgbM zIR`{wkwamBnXMQT0mlJd;$t!HR>6k76~mL3klL@|nN(peZ=W~3FZQAr-SPIS!KGG0 zUwu1$sLzCtl$Q1ppZ-Qp>sm9_r>*iHQvEvQUmOB#aez~YLutc}qytn<~vy=F%*Kt1K zggK(6N3f4{Cp&@!tj&2MzkP=_P>1y$9g0_6EKCqBn+Mt?g9jeY+6Ih%D?l zK5XSIWFs=;uB`V|m>eN%)`qjU#XM=_zM!_z75(t!v7eDntoD&Q3afRQS1Sk1@ZO78 zUH<(`Zc2RJFtO+@jX7$5D$0#-3Yq)G;})eQ(t)My2^%L8(u}BVj*G&3xuW*ikKOep z*=w%(>>b^17kV7(PbTx-Do$szL5LQ;o(c6CIJ%ee5aoWTb3SG=-xQcR_N+Zsbw88` zoo6S@53QJ6zNY~yT2Z|oibz-(52NoRCpdF@o3r3Y)Wg zpF;|k$m>s3zOml=h}tOjQNj_h5Icd63y9;^!@Syhnt6gv$(t8s$AQvbOPF8*q62Rx$%}u9542FV=p#SBctkszfN5sA%f^) zIG@E>Cqn^vu`cAQU!Uv$+Go^`t*i*N&dCtm47+4$cdrb5^AYE0l ztVS`QZdupNmg4xVi1mhBUh!_Xx75-oj}U+o}&|8SA+( zZz;`=U6T^@oBgANPCFGWWLr@CBTs??_4Rfpx>1&@%bzidmPS+~M@xAwR$@wv9ttv( zX`KId;4+^X2J}tnS%rzqm%hg^@$47(tiwV_kH~(Nw64AvifC*Bm-}Tz2dvme`c-_6 z0VtbHw%cA1#z{Z&aFkAWU(%yaZs&o ztLz6QqVSkdMy{$N5b2BT-naRkJ+hJPJ%gxKRdd6dj^NH*8d4y*KnX`aQ}aI!qN=sS z82DUV)s0i`*Vc?J1k~kI_8FBXJJq_^5VT?d)_U`bfw#(@~s*0A+N%dPsifloFCdz~2RbELIQ zse5DqPwHxknl~(s{GQprrV3!-mX2d7TgT?qv|L%YaSi7x$(foWhJG9uepNhul1=l| zvue)y2)Xe>AO%S!g`KilvFTrIk~v-j5~-0P6g$(vq6Z&dasQA{-Jk@hzl#$A+VrUl zr1qdqNp`@O(hr5_7>|HDcFgSr& zf0KrUU?9pHGnxwOKon|W(n|EV3>!(s_l4#$YCyE)%Rfxf4K{y%zSKa-z_YiYC+5Q4 z$Q6Z6L!I8s<5^v&SMUN%P$UA$+w+F-U?iJlD*xDwG#;zDD+GwnJM>@U%Pp#^(Zmm+ z&Q~|zyCiIU1wK?pKl@zC*NrMn+Q{6QRwYC^FZ3ahgPP*gM4xNd7eqU=)V|t;@cm~E z_#F`MRNe3|;WCvzc_5=5+U{!wiYJBaBd`j?m+L>yWTpPyaqI)h;ArQV%A0em)n#2f ze$POMn22-Gw&3%Whm~-#GUX*n%SC`(@C(11 z&aOE}<3-ahN~M8fyY7Y-ojjHA891g(*J?^>!%?(iE?V)6W0qX>&GSt@L!JQgiz|(W zr>U#PEFl-CFIr)qPZ_lMJ5y6Uw0Za_zB8PB$g7Gj-? z8{ee5pC`*^-P$8UK7H+Jj3$Epg>TW6!3dvZOPhDnrMmOi*_kiXVe-3!t9D)7aUdUY z5;7bvQ8yc8A-0U_!E|ku@Kq z$X$jSAhq@mo4D6ttDt#ZDs5x|;LsPh4~-h1ZC4wVq|VCk_L4s@;X!`~)~#F7;WGDG zsAQ9%JQ3nzgnZX$Yp$l)_gPb3xsTnv>XgNu0stfL;GzAk(Tfnr4-HKAi&(edfztZ zb6x07`cSASI>x~1JS3g1)2*p#@)h=37jY6+>EP+e<2xK*GbREPP^KG; zKBTlubmS}wsLk(N-&v^xm=`y?Cq@by>>Xx%MdrLfvW7n>%>~r7BKi+yKF>sA2;OZL0+NM-B&Py?_m6L

zhJWu*mrgWPtG#vwvI#WIm8RtW{ph@4`rYhN1;yJ`DvHHH-YlhGnk!b8O;htTKpB&h zz-OPQ0Fb@@umd=C>A53ZyQF|vn5uqmDnqEoFrv@iZ4xN5Kl!N!8e~oyBKZa~8XavNF5gct0CQ}<=9{~l_ z2j?TqYRw)UqoI+LQ9f--xQ-4z!(d~4|7fPik?T!g7dwOk)GYfoaR4~k;+|YrhnO4^ z6MOPb-m|Sk8lwDWN+y~)IQ}sz{mZWl@)TjTEvMEGIbLh$qfK9>_VPeTi9J7?InJH0vCCVZ>S6060VHuCx(d0RJiu)zLqzPKZQC4CxtC0O^vL&YnM^9;dyeGq)KJ>z^P~3f0ty}eCScd=7QE}FS-4;%tOEp ztr%+FkGrsa=O|sRP&XScvqDb8L0un`#(Y{IZeK0qAy~WduChAFeL%j;nG)wN)guZZ zOqA&QvnU}FTyAD zSu3Fz-_=DL{@qb!7=Ii&p0DD^No0zDT0vO@h4`!x`5PO#``kQDK8u8G$ zhe#BENG&@2+fu~BX8@WS=_6CX*#`j_u8WtwvsD>=Li0aYyZ@Ug{y!oPMLL}@&}dyN z(1^*g&MJEVR&+kqw?soMNlPGEJeFA~7koW!mJ`pvsHMaQqyfX|TMPgI2iiwm4CK!? zs+ona?WeBU zkybo#N&4cqc5K!c$fh6|l+S1@ub~ckpHQ6H$OgHHtR`0f9Bq-PW$pG z^k`WO<{&2u5cD(?ARn+iP&b_RVo`F1v=P1l}<%)$C%zg^t5 zV@OBFot{5~qZ5RnmkU;tf|w3f5})G6iti?Pi{6x!)p$Z|G7(%+1te$;eG?09!~vMQ zchSQ1Bi50%wh(=SIn~WTlAXvnrzLP|$lXX0d|Yy@TW^q(QIY=g_mup|)jouXk9Q`y ze{qH-chaq-)h06bB_a-7ezJ*IQkLX8XYq4$noLqBPX6E`m`)v% zdbh@gi-5V@xF3z)W?vcWa^()n6}Jl&q8eT1kdg@DjfHOHC721)44<7fm+{K$Yy&l* zBHwz8Vr&*q{)Zz=2JHf(^EGpjCNS$$4hWu}9_Th+9`Vgjpxei+Odrt?`0PFvH2?(s zNW-=0a~3ZBI$>O8>?40&oAWP@fWhFP>Tgd<3V)LP@Lc1d)g|%wT)@II=9kZC* z3c_cDP3TO@&KK3iM*1cN)l#(3l!FPyTpZw=Odczzwm)>OBvGnvKCx_>bM*M6%a}d0#S{~L6$&ejw|Bs?TVuEeYP$SqpGOfn zDEd1A&A5Rgj(8VscsEBM$eFM%k6blS4uoX^86_Gt*^RL+J&RU>VSuM2Sn(H7K4q+R z3WZR7=}c~*&z*Vp^e|BETfwf6zMtSUS>Qy5KkYs0c??pXy)z(rJIGiOL}?n_8IeTy z+0u_(_ zYnMKaB6^PWl?iKvWmK}f)L!VF4+=s8e(wNuD4aQ})xUiw^{6#uGZv7#`)vhY*@ONn zG_AyyC{85X(W~lzsN|W*LZy5hiL@{KZeN%*A6>R*>WKwt=68(DSrVb2j^pFGi1LSm zGX5q7>C1G|ut~=_JQ2G~|Hii|DAcgMkV1j+ps8gTNHCKdR+!`ZF%f{Usm1iwGLWmo zLF#K;1sWh{fGa@(;`z8jt@W-0w5YM|)K_Mti%?qQyoo?)=6b#N&lXXDa47UE*)76quFHnEK zm?}C$Hl7(9!D)E_8h$zuv5I}#<5+l*FZo(vL0YCB=#0KMlEKvN0y7D5=u zA1+?NL-TWGdYk#RD3o#k(zH?sOzFa zH2GBf$n0t2JWfCI`|VXdxRy#|0;uHSd>TF^t!q#{_hqr!#}ZpAwl^K5X>5it8x{g@ z_ev}DO>6#-9PN96Hk8$jx9%z8ozK7zA$i~uWDSblcX!tkNp1QDK(AUGwQ9J%hWU7y z92*zSFM2Qfm#ywTJj9L-BlaJFjatz*^85mmwXbw10t&zm4)E379F*&&1DYb$gPNkx zJ{rO-LHfikzl&p*K;F2Ghj8mj7absa-;4c$&XZeD7>T17;ARs^=vo0o8AxsO0nu#zD^Mhjh+2Y7egRf0*i42( zJE(aK&GB3CJ~9@i9W9!U*BB0M+FUjBEe16zk7O)>tN?S7&k}WPX(V#A;r&{KKhv%D4xZ70`Ejht|pS z8*b^j#&dE3Je^(3LKJJS_0J7a7nne@c)h)vW&-vldJUt>mw+_yjj0h5(CsA;zkgn9 z&_v(csn2Q#d4%Cy(>*>ljFMo=NVqQu_914u|8CpkqJ5LkvamGKk_Eq?8#90_KJAL-i3ioo6q}qTEonX zgQEa}wY1t|y05)gDOcHm37U`1w8Uc7YOS5FSnkF zG#tLJub3JV8KPeK?n{Y$5N{?LKUSM)<39P~U^Ku@s?LGuC%Fb$|MeQiUZ{+3E~v3b z+E~4KJZY}ESp@?--NDO>RhClILrQo0WmY7-(A^T)tMug&A=WXUpsfFWdF)I=R=B z)ePx&h@ArBN`$1%13L}~{qww?NtrGe&rZ7)(n8VMG(vi%YOL_oAY@eV+_%wp;h!W2D=G zvn=ZUV)x&NXPs9(@ks%`TRTJSglFFkA5<|Aku{{KdSav}P$^_Ds}Ol@uQsM4-yGar zEDT2N)@Xq=2*_x}#B}_CQ2b~f7B+v;Kp^(WpvK(&!Q7`~=PEfgP%U#zC44m>bvM=I zCA;=!-tymztI+Q{(w`Kk8@JZjbh5pgIAid-z$wD_2I`(OO3%3hULlc2Hd2toatkc9 z1Fb+`Hd4%Kzx{M>Jksr5^S(&WaazFhGq&S%zj!e6uwBF5nW$#F}_t={;R6p)~%iWW*4Dw#rLy94Ivg=JwbMJ<8f#%fI3zc!=seMHN(w@VOdO6{)!?Y#W;SZ0u=J zBC4X^8+>-v*-YHLDUHR#o5)pRvo6SrX-mQ`uuuy@*yHsrxvw)KIlWUQTgqMh#9bw zlFm@~JSLaCWf=_uz1$u6&L_+^lI$;gkQMMmiP6^+4j5({5u?Gvtrh!3s zDdvhGIc``G5?o*iuqk2H%kGgzc&M1BOVk8g{>H&(RgYZTz8SDb3%)Hmz-93B`-Xsk(E$H{0rbttPJvF- zKturHEV%B6^vod8Er8TkC~M-;(p#LuoM~BUHz!;Tz9y47Rmf=iBhsj!fOu;?$B4is-Euo(`$4O$42x$fP?LjL}mH1^7- zO|0;v#UfszN2`^^w{=AQzr6%L%>A!Q&MT^^bc^E%3K}}G;9$8R8Us=S3K19;k*1g{ zg7g{zXCwv?Fbu|qGzCP3BxGn-nuCg=h#)=E0s)jBS|Uaefk+EYXJ^g4%+sCwn6=JY zDeL4r`(J)L_Y35|A;}EOaOb4saff#{;qScIfov$#&!~uFhUQ!t(T)nf8;3c-x%jp{ zCXA}Qqj?NV#j^9E{(0DB@z-67+GWqOB?} zT{JpKI$tH#cWIzw;iE#pAXE&X>YBo4M2l|QH(aaCm%Gdx)xCk2l-y)+-q!!Og`{ht zArH*68YkrfeVRtcL#jxQlyph))r=ue3#r+$%Er!)aGT|EJ%)M<7z-H(F|1EU9}_E! z4!L|M$4IS!Gni8J8lJe{$t7T8z|=<9GMDyu4GXv*IZNfs`{Rz@U4zk zwQ1^U6>INIfWjPtO`45~e}zjz{oQ4`LvnkWWNxX-8VR{7>qmTBv(J)|RCFOzW$Nhz zX-9d_>U4xg<>hlyD@zWtgZ!;a&KOpVVd?WSNZ+SB_RdSrfv~U_zdwu4ZIUEkyyIRm zJCgiTn}jxYbc&Op6Rs@DXGPXFHjz)vLJ8RG=z7w3k7w^kT6P(}Ax}Vk$>wQQ==)tZ zdCH_Dw|#Z>3zljP?JLWAfTiB5C%p`Q4_47UJI)Lk4)^k}Vf_V~#}H_>M(@t89nQXz1z4S z4|+yO@Jn`%wy&2m_O6|^`Xu}$ZC>u0rziXSE6MVpk4F3y+R^BNMCLTL-$0m!R`BA* zTZx>V#q4Nr@A5_IQ9us3^i?`52%IK4omk5!{y(IbfZHE9eyZG%ML&3HLXxtnk|%MM z;SyI*a`Sar1)CRJ8luZU6=6VW1(6%Q_Kf91i#j4bm(_+=y<&PM=_jc2rHd-mG*W8s zyl9}K)%LOk=n-w8WW+fa$})E{bO)smo9t-bzwIwZrd#G}MoA){Yy6f&zxQ1ZTM)8- zirVWgfPNp3qikZ3qlI=p9tRSa8G0^AGqL+Lh7e30T4HVv+JNOt{vgd3(R}N>qfE$O-!=iRm&iTi(?Ujm)zxJi%Nto8b-P zwM2A5tu$BG#dH)3uN9>QX<7|yyawpveMd_ke-YwV+WeEgfmz7|<~@xEtSviXx)yxA zZKWPjywXmqrWdApuSUA`)X({E5aB>0{OcQ3aVZ|#&R{8?HyKeyRyzJifI5bs?s=C|SZ|}QLBj^5S(*rhu{~C!2r$0Q$QoB24EA&2X6SPkHUozTWpH0z4Nhb8%r`vm!rNgsheaGmp4p z9@9NNF?p)^c-pn5h?|rYy3sU)#v5M2#t)EhIM)Ucb>e61AFX|FINuAsJVrdx{xZ!dKX z_ub;0hqMLXB}SyK9!9N&eG1OmzQ;xC<@Fo_VkrK=*A`&qz%oif&RAZ!Z9$QAz~3`% z7D6|8dbmq-`{q7OVfGJ@(mrSQbJMLK+B}F2EHI6X6U=a97@C5*1+)c)WkAJA?Hfv! zlPXPdUZ*%tNPrs!b7|(cIHLR)*f4;e8rg$-%g1KeC4kJGeqIN~Z@#*{qv10;-p}Uo zbgE{Ke|uX(o&;$Y&jgW<|B%fOI`cwf%TBrztW2l`^06+FeflM;B>k{}aN;})$pcEn z{g4CnjiEBkTAa)?z07t!AtOC+d^%aVe6rBG+M_at4*LrjK5Qo_HaNX8mR-0S)c&Zt zVs$TzH}rIx=!KlbCrHe2S#OLUp+He3fP-l}#H7AXU$@+`GnUvGp$lwg)t75W6e&n9H zLU6OZxcn`GhGHKEyWY=XKmJ`e4FjI=AI9!przA$R%76DWDLah3w==B{PS2lR4l>R> zX2L%>=tg#-078J3*<)_mrUg18Kt(hq0NZMDG6BvNa%kyV!517a=T_M-gBwI_6Z*=^ zHLwn0BokTHqLHkuf8b<-hAfWX&2_?Uy7S4N=i^ai9b6Waf#8g?wK7Xb@ff?B!WXXq z9yk(kN8oc+wl%d)UpnyaS(mjXl1uVqzBb##&Mx*fV|LwNE??GfrteM_cp&0u88+7M zU+B1y%-@(ia8uzHwjZ~ zZC2wi5xlhm=)tT0fqu!F?_#jK2JLPDq1(FA22O}EUs-HOz7jNh=lk=k(8CM5Bv}nu_2d@d^)*hyhRm8YJ+QlGz@mN6z`?a7nkuPnUh1 z(l~=3xyAm`f2T+1WCM)WL3s2p*DL+wvVSB=|MJEE6*&F1zUY6EYGaB& Date: Tue, 17 Feb 2026 17:04:01 -0700 Subject: [PATCH 096/173] Refactor: Remove old js tests --- tests/js/file-browser.test.js | 35 ---------- tests/js/logs.test.js | 44 ------------- tests/js/settings.test.js | 18 ------ tests/js/state.test.js | 39 ------------ tests/js/stepper.test.js | 101 ----------------------------- tests/js/upload-control.test.js | 81 ------------------------ tests/js/upload-exec.test.js | 105 ------------------------------ tests/js/utils.test.js | 109 -------------------------------- 8 files changed, 532 deletions(-) delete mode 100644 tests/js/file-browser.test.js delete mode 100644 tests/js/logs.test.js delete mode 100644 tests/js/settings.test.js delete mode 100644 tests/js/state.test.js delete mode 100644 tests/js/stepper.test.js delete mode 100644 tests/js/upload-control.test.js delete mode 100644 tests/js/upload-exec.test.js delete mode 100644 tests/js/utils.test.js diff --git a/tests/js/file-browser.test.js b/tests/js/file-browser.test.js deleted file mode 100644 index c9b9ebb..0000000 --- a/tests/js/file-browser.test.js +++ /dev/null @@ -1,35 +0,0 @@ -import { describe, it, expect, beforeEach, vi } from 'vitest'; -import state from '../../app/static/js/modules/state.js'; - -describe('file-browser', () => { - beforeEach(() => { - state.currentPrefix = ''; - - document.body.innerHTML = ` -

- -
-
-
Loading...
-
- - -
- - - - -
- `; - }); - - it('state.currentPrefix defaults to empty string', () => { - expect(state.currentPrefix).toBe(''); - }); - - it('currentPrefix can be updated', () => { - state.currentPrefix = 'year=2024/'; - expect(state.currentPrefix).toBe('year=2024/'); - state.currentPrefix = ''; - }); -}); diff --git a/tests/js/logs.test.js b/tests/js/logs.test.js deleted file mode 100644 index c69cb18..0000000 --- a/tests/js/logs.test.js +++ /dev/null @@ -1,44 +0,0 @@ -import { describe, it, expect, beforeEach } from 'vitest'; -import state from '../../app/static/js/modules/state.js'; - -describe('logs state', () => { - beforeEach(() => { - state.logFilters = { date: null, level: null, category: null, search: '' }; - state.logPagination = { offset: 0, limit: 100 }; - }); - - it('logFilters defaults are correct', () => { - expect(state.logFilters.date).toBeNull(); - expect(state.logFilters.level).toBeNull(); - expect(state.logFilters.category).toBeNull(); - expect(state.logFilters.search).toBe(''); - }); - - it('logPagination defaults are correct', () => { - expect(state.logPagination.offset).toBe(0); - expect(state.logPagination.limit).toBe(100); - }); - - it('logFilters can be updated', () => { - state.logFilters.date = '2026-02-07'; - state.logFilters.level = 'ERROR'; - state.logFilters.category = 'upload'; - state.logFilters.search = 'test'; - - expect(state.logFilters.date).toBe('2026-02-07'); - expect(state.logFilters.level).toBe('ERROR'); - expect(state.logFilters.category).toBe('upload'); - expect(state.logFilters.search).toBe('test'); - }); - - it('logPagination offset can be advanced', () => { - state.logPagination.offset = 100; - expect(state.logPagination.offset).toBe(100); - }); - - it('logFilters can be reset', () => { - state.logFilters.level = 'ERROR'; - state.logFilters = { date: null, level: null, category: null, search: '' }; - expect(state.logFilters.level).toBeNull(); - }); -}); diff --git a/tests/js/settings.test.js b/tests/js/settings.test.js deleted file mode 100644 index 534e0ff..0000000 --- a/tests/js/settings.test.js +++ /dev/null @@ -1,18 +0,0 @@ -import { describe, it, expect, beforeEach } from 'vitest'; -import state from '../../app/static/js/modules/state.js'; - -describe('settings', () => { - beforeEach(() => { - state.currentAwsProfile = undefined; - }); - - it('state.currentAwsProfile defaults to undefined', () => { - expect(state.currentAwsProfile).toBeUndefined(); - }); - - it('state.currentAwsProfile can be set', () => { - state.currentAwsProfile = 'my-profile'; - expect(state.currentAwsProfile).toBe('my-profile'); - state.currentAwsProfile = undefined; - }); -}); diff --git a/tests/js/state.test.js b/tests/js/state.test.js deleted file mode 100644 index 3a04e2a..0000000 --- a/tests/js/state.test.js +++ /dev/null @@ -1,39 +0,0 @@ -import { describe, it, expect } from 'vitest'; -import state from '../../app/static/js/modules/state.js'; - -describe('state', () => { - it('has correct default values', () => { - expect(state.currentJobId).toBeNull(); - expect(state.eventSource).toBeNull(); - expect(state.selectedFolderPath).toBeNull(); - expect(state.currentStep).toBe(1); - expect(state.currentPrefix).toBe(''); - expect(state.appVersionData).toBeNull(); - expect(state.currentAwsProfile).toBeUndefined(); - expect(state.scanFilePaths).toEqual([]); - expect(state.scanFileStatuses).toEqual([]); - expect(state.scanTotalSize).toBe(0); - expect(state.scanFolderPath).toBeNull(); - expect(state.reviewSortConfig).toEqual({ column: 'filename', ascending: true }); - }); - - it('is a shared mutable reference', () => { - const originalStep = state.currentStep; - state.currentStep = 3; - expect(state.currentStep).toBe(3); - state.currentStep = originalStep; - }); - - it('allows setting and clearing job id', () => { - state.currentJobId = 'test-job-123'; - expect(state.currentJobId).toBe('test-job-123'); - state.currentJobId = null; - expect(state.currentJobId).toBeNull(); - }); - - it('allows managing scan file paths', () => { - state.scanFilePaths = ['/path/to/file1.mcap', '/path/to/file2.mcap']; - expect(state.scanFilePaths).toHaveLength(2); - state.scanFilePaths = []; - }); -}); diff --git a/tests/js/stepper.test.js b/tests/js/stepper.test.js deleted file mode 100644 index 5e1416d..0000000 --- a/tests/js/stepper.test.js +++ /dev/null @@ -1,101 +0,0 @@ -import { describe, it, expect, beforeEach } from 'vitest'; -import { setUploadStep, showUploadSteps, hideUploadSteps, goToStep } from '../../app/static/js/modules/stepper.js'; -import state from '../../app/static/js/modules/state.js'; - -describe('stepper', () => { - beforeEach(() => { - state.currentStep = 1; - - document.body.innerHTML = ` -
-
-
-
-
-
-
-
-
-
- `; - }); - - describe('setUploadStep', () => { - it('sets the active step', () => { - setUploadStep(3); - - const step3 = document.querySelector('[data-step="3"]'); - expect(step3.classList.contains('active')).toBe(true); - expect(step3.classList.contains('completed')).toBe(false); - }); - - it('marks previous steps as completed', () => { - setUploadStep(3); - - const step1 = document.querySelector('[data-step="1"]'); - const step2 = document.querySelector('[data-step="2"]'); - expect(step1.classList.contains('completed')).toBe(true); - expect(step2.classList.contains('completed')).toBe(true); - }); - - it('leaves future steps unmarked', () => { - setUploadStep(3); - - const step4 = document.querySelector('[data-step="4"]'); - expect(step4.classList.contains('active')).toBe(false); - expect(step4.classList.contains('completed')).toBe(false); - }); - - it('updates the step description', () => { - setUploadStep(1); - const desc = document.getElementById('step-description'); - expect(desc.textContent).toBe('Select files or a folder to upload'); - }); - - it('updates state.currentStep', () => { - setUploadStep(4); - expect(state.currentStep).toBe(4); - }); - - it('colors connectors for completed steps', () => { - setUploadStep(3); - const connectors = document.querySelectorAll('.step-connector'); - expect(connectors[0].style.backgroundColor).toBe('rgb(93, 151, 50)'); - expect(connectors[1].style.backgroundColor).toBe('rgb(93, 151, 50)'); - expect(connectors[2].style.backgroundColor).toBe('rgb(209, 213, 219)'); - }); - }); - - describe('showUploadSteps', () => { - it('delegates to setUploadStep', () => { - showUploadSteps(2); - expect(state.currentStep).toBe(2); - const step2 = document.querySelector('[data-step="2"]'); - expect(step2.classList.contains('active')).toBe(true); - }); - }); - - describe('hideUploadSteps', () => { - it('resets to step 1', () => { - setUploadStep(4); - hideUploadSteps(); - expect(state.currentStep).toBe(1); - const step1 = document.querySelector('[data-step="1"]'); - expect(step1.classList.contains('active')).toBe(true); - }); - }); - - describe('goToStep', () => { - it('does nothing when navigating forward', async () => { - state.currentStep = 2; - await goToStep(3); - expect(state.currentStep).toBe(2); - }); - - it('does nothing when navigating to current step', async () => { - state.currentStep = 2; - await goToStep(2); - expect(state.currentStep).toBe(2); - }); - }); -}); diff --git a/tests/js/upload-control.test.js b/tests/js/upload-control.test.js deleted file mode 100644 index 401e366..0000000 --- a/tests/js/upload-control.test.js +++ /dev/null @@ -1,81 +0,0 @@ -import { describe, it, expect, beforeEach } from 'vitest'; -import { resetUpload } from '../../app/static/js/modules/upload-control.js'; -import state from '../../app/static/js/modules/state.js'; - -describe('upload-control', () => { - beforeEach(() => { - state.currentJobId = 'test-job'; - state.eventSource = null; - state.selectedFolderPath = '/some/path'; - state.scanFilePaths = ['/file1.mcap']; - state.scanFileStatuses = [{ path: '/file1.mcap', filename: 'file1.mcap', size: 100, already_uploaded: false }]; - state.scanTotalSize = 100; - state.scanFolderPath = '/some/path'; - state.currentStep = 3; - - document.body.innerHTML = ` -
-
-
-
-
-
-
-
-
-
- -
-
-
-
- `; - }); - - describe('resetUpload', () => { - it('clears the job ID', () => { - resetUpload(); - expect(state.currentJobId).toBeNull(); - }); - - it('clears selected folder path', () => { - resetUpload(); - expect(state.selectedFolderPath).toBeNull(); - }); - - it('clears scan file paths', () => { - resetUpload(); - expect(state.scanFilePaths).toEqual([]); - }); - - it('clears scan file statuses and total size', () => { - resetUpload(); - expect(state.scanFileStatuses).toEqual([]); - expect(state.scanTotalSize).toBe(0); - expect(state.scanFolderPath).toBeNull(); - }); - - it('shows folder browser panel and hides other sections', () => { - resetUpload(); - - expect(document.getElementById('folder-browser-panel').classList.contains('hidden')).toBe(false); - expect(document.getElementById('upload-section').classList.contains('hidden')).toBe(true); - expect(document.getElementById('completion-section').classList.contains('hidden')).toBe(true); - expect(document.getElementById('scan-results-section').classList.contains('hidden')).toBe(true); - expect(document.getElementById('confirm-upload-modal').classList.contains('hidden')).toBe(true); - }); - - it('resets stepper to step 1', () => { - resetUpload(); - expect(state.currentStep).toBe(1); - }); - - it('closes eventSource if open', () => { - let closeCalled = false; - state.eventSource = { close: () => { closeCalled = true; } }; - resetUpload(); - expect(closeCalled).toBe(true); - expect(state.eventSource).toBeNull(); - }); - }); -}); diff --git a/tests/js/upload-exec.test.js b/tests/js/upload-exec.test.js deleted file mode 100644 index a1262d8..0000000 --- a/tests/js/upload-exec.test.js +++ /dev/null @@ -1,105 +0,0 @@ -import { describe, it, expect, beforeEach } from 'vitest'; -import { updateProgressUI, showCompletionSummary } from '../../app/static/js/modules/upload-exec.js'; -import state from '../../app/static/js/modules/state.js'; - -describe('upload-exec', () => { - beforeEach(() => { - state.currentStep = 3; - - document.body.innerHTML = ` -
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- -
-
-
-
-
-
-
-
- `; - }); - - describe('updateProgressUI', () => { - it('updates progress elements', () => { - updateProgressUI({ - progress_percent: 50.5, - files_completed: 3, - total_files: 6, - uploaded_bytes_formatted: '15 MB', - total_bytes_formatted: '30 MB', - eta_seconds: 120, - files: [ - { filename: 'test.mcap', status: 'completed', file_size_formatted: '5 MB', progress_percent: 100 }, - { filename: 'test2.mcap', status: 'uploading', file_size_formatted: '5 MB', progress_percent: 50 }, - ], - }); - - expect(document.getElementById('progress-percent').textContent).toBe('50.5'); - expect(document.getElementById('files-completed').textContent).toBe('3'); - expect(document.getElementById('files-total').textContent).toBe('6'); - expect(document.getElementById('bytes-uploaded').textContent).toBe('15 MB'); - expect(document.getElementById('bytes-total').textContent).toBe('30 MB'); - expect(document.getElementById('eta').textContent).toBe('2m 0s'); - }); - }); - - describe('showCompletionSummary', () => { - it('updates completion counts', () => { - showCompletionSummary({ - files: [ - { status: 'completed', filename: 'a.mcap', file_size_formatted: '5 MB' }, - { status: 'completed', filename: 'b.mcap', file_size_formatted: '3 MB' }, - { status: 'skipped', filename: 'c.mcap', file_size_formatted: '2 MB' }, - { status: 'failed', filename: 'd.mcap', file_size_formatted: '1 MB' }, - ], - files_uploaded: 2, - files_skipped: 1, - files_failed: 1, - successfully_uploaded_bytes_formatted: '8 MB', - total_upload_duration_formatted: '1m 30s', - average_upload_speed_mbps: 42.5, - }); - - expect(document.getElementById('completed-count').textContent).toBe('2'); - expect(document.getElementById('skipped-count').textContent).toBe('1'); - expect(document.getElementById('failed-count').textContent).toBe('1'); - expect(document.getElementById('total-uploaded-size').textContent).toBe('8 MB'); - expect(document.getElementById('avg-upload-speed').textContent).toBe('42.5 Mbps'); - }); - - it('sets step to 4 (complete)', () => { - showCompletionSummary({ - files: [{ status: 'completed', filename: 'a.mcap', file_size_formatted: '5 MB' }], - }); - - expect(state.currentStep).toBe(4); - }); - - it('shows completion section and hides upload section', () => { - showCompletionSummary({ - files: [{ status: 'completed', filename: 'a.mcap', file_size_formatted: '5 MB' }], - }); - - expect(document.getElementById('upload-section').classList.contains('hidden')).toBe(true); - expect(document.getElementById('completion-section').classList.contains('hidden')).toBe(false); - }); - }); -}); diff --git a/tests/js/utils.test.js b/tests/js/utils.test.js deleted file mode 100644 index 45070b7..0000000 --- a/tests/js/utils.test.js +++ /dev/null @@ -1,109 +0,0 @@ -import { describe, it, expect, beforeEach, afterEach } from 'vitest'; -import { showNotification } from '../../app/static/js/modules/notify.js'; -import { formatBytes, formatEta, formatDuration } from '../../app/static/js/modules/formatters.js'; - -describe('formatBytes', () => { - it('returns "0 B" for zero bytes', () => { - expect(formatBytes(0)).toBe('0 B'); - }); - - it('formats bytes correctly', () => { - expect(formatBytes(500)).toBe('500 B'); - }); - - it('formats kilobytes', () => { - expect(formatBytes(1024)).toBe('1 KB'); - expect(formatBytes(1536)).toBe('1.5 KB'); - }); - - it('formats megabytes', () => { - expect(formatBytes(1048576)).toBe('1 MB'); - expect(formatBytes(1572864)).toBe('1.5 MB'); - }); - - it('formats gigabytes', () => { - expect(formatBytes(1073741824)).toBe('1 GB'); - }); - - it('formats terabytes', () => { - expect(formatBytes(1099511627776)).toBe('1 TB'); - }); -}); - -describe('formatEta', () => { - it('returns "Calculating..." for null/undefined', () => { - expect(formatEta(null)).toBe('Calculating...'); - expect(formatEta(undefined)).toBe('Calculating...'); - }); - - it('returns "Calculating..." for negative values', () => { - expect(formatEta(-5)).toBe('Calculating...'); - }); - - it('formats seconds', () => { - expect(formatEta(30)).toBe('30s'); - expect(formatEta(1)).toBe('1s'); - }); - - it('formats minutes and seconds', () => { - expect(formatEta(90)).toBe('1m 30s'); - expect(formatEta(125)).toBe('2m 5s'); - }); - - it('formats hours and minutes', () => { - expect(formatEta(3661)).toBe('1h 1m'); - expect(formatEta(7200)).toBe('2h 0m'); - }); -}); - -describe('formatDuration', () => { - it('formats sub-second durations as milliseconds', () => { - expect(formatDuration(0.5)).toBe('500ms'); - expect(formatDuration(0.001)).toBe('1ms'); - }); - - it('formats seconds with one decimal', () => { - expect(formatDuration(5.3)).toBe('5.3s'); - expect(formatDuration(30.0)).toBe('30.0s'); - }); - - it('formats minutes and seconds', () => { - expect(formatDuration(90)).toBe('1m 30s'); - expect(formatDuration(125)).toBe('2m 5s'); - }); -}); - -describe('showNotification', () => { - beforeEach(() => { - document.body.innerHTML = ''; - }); - - afterEach(() => { - document.body.innerHTML = ''; - }); - - it('creates a notification element in the DOM', () => { - showNotification('Test message', 'info'); - const notification = document.querySelector('.fixed.top-4.right-4'); - expect(notification).not.toBeNull(); - expect(notification.textContent).toBe('Test message'); - }); - - it('applies error styling for error type', () => { - showNotification('Error!', 'error'); - const notification = document.querySelector('.fixed.top-4.right-4'); - expect(notification.classList.contains('bg-red-500')).toBe(true); - }); - - it('applies success styling for success type', () => { - showNotification('Success!', 'success'); - const notification = document.querySelector('.fixed.top-4.right-4'); - expect(notification.classList.contains('bg-green-500')).toBe(true); - }); - - it('applies info styling by default', () => { - showNotification('Info'); - const notification = document.querySelector('.fixed.top-4.right-4'); - expect(notification.classList.contains('bg-nlr-blue')).toBe(true); - }); -}); From 3dc1705a5178815a9d361583d9e293398f9ef4e0 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Tue, 17 Feb 2026 17:04:31 -0700 Subject: [PATCH 097/173] Refactor: Delete non react js modules --- app/static/js/app.js | 51 -- app/static/js/modules/about.js | 55 -- app/static/js/modules/analysis.js | 421 ---------- app/static/js/modules/api.js | 51 -- app/static/js/modules/debounce.js | 16 - app/static/js/modules/dom.js | 73 -- app/static/js/modules/file-browser.js | 205 ----- app/static/js/modules/folder-browser.js | 952 ----------------------- app/static/js/modules/formatters.js | 60 -- app/static/js/modules/icons.js | 26 - app/static/js/modules/logs.js | 425 ---------- app/static/js/modules/notify.js | 25 - app/static/js/modules/settings.js | 402 ---------- app/static/js/modules/sorting-helpers.js | 40 - app/static/js/modules/state.js | 75 -- app/static/js/modules/stepper.js | 83 -- app/static/js/modules/upload-control.js | 54 -- app/static/js/modules/upload-exec.js | 226 ------ app/static/js/modules/upload-init.js | 53 -- 19 files changed, 3293 deletions(-) delete mode 100644 app/static/js/app.js delete mode 100644 app/static/js/modules/about.js delete mode 100644 app/static/js/modules/analysis.js delete mode 100644 app/static/js/modules/api.js delete mode 100644 app/static/js/modules/debounce.js delete mode 100644 app/static/js/modules/dom.js delete mode 100644 app/static/js/modules/file-browser.js delete mode 100644 app/static/js/modules/folder-browser.js delete mode 100644 app/static/js/modules/formatters.js delete mode 100644 app/static/js/modules/icons.js delete mode 100644 app/static/js/modules/logs.js delete mode 100644 app/static/js/modules/notify.js delete mode 100644 app/static/js/modules/settings.js delete mode 100644 app/static/js/modules/sorting-helpers.js delete mode 100644 app/static/js/modules/state.js delete mode 100644 app/static/js/modules/stepper.js delete mode 100644 app/static/js/modules/upload-control.js delete mode 100644 app/static/js/modules/upload-exec.js delete mode 100644 app/static/js/modules/upload-init.js diff --git a/app/static/js/app.js b/app/static/js/app.js deleted file mode 100644 index 08d325b..0000000 --- a/app/static/js/app.js +++ /dev/null @@ -1,51 +0,0 @@ -/** - * modaq-upload - Entry Point - * - * Detects the current page via data-page attribute and dynamically - * imports the appropriate module. Registers delegated click handlers - * for data-action attributes (replacing inline onclick). - */ -import { - closeAboutModal, - initAboutModal, - loadHeaderVersion, - openAboutModal, -} from './modules/about.js'; -import { hideEl } from './modules/dom.js'; -import { goToStep } from './modules/stepper.js'; - -// Global initialization (runs on every page) -initAboutModal(); -loadHeaderVersion(); - -// Delegated click handler for data-action attributes -document.addEventListener('click', (e) => { - const target = /** @type {HTMLElement} */ (e.target).closest('[data-action]'); - if (!target) return; - - const action = /** @type {HTMLElement} */ (target).dataset.action; - - if (action === 'open-about') { - openAboutModal(); - } else if (action === 'close-about') { - closeAboutModal(); - } else if (action === 'go-to-step') { - const step = Number(/** @type {HTMLElement} */ (target).dataset.step); - if (step) goToStep(step); - } else if (action === 'close-confirm-modal') { - hideEl('confirm-upload-modal'); - } -}); - -// Page-specific module loading -const page = document.body.dataset.page; - -if (page === 'upload') { - import('./modules/upload-init.js').then(({ initUpload }) => initUpload()); -} else if (page === 'files') { - import('./modules/file-browser.js').then(({ initFileBrowser }) => initFileBrowser()); -} else if (page === 'settings') { - import('./modules/settings.js').then(({ initSettings }) => initSettings()); -} else if (page === 'logs') { - import('./modules/logs.js').then(({ initLogs }) => initLogs()); -} diff --git a/app/static/js/modules/about.js b/app/static/js/modules/about.js deleted file mode 100644 index a206e82..0000000 --- a/app/static/js/modules/about.js +++ /dev/null @@ -1,55 +0,0 @@ -/** - * About modal functionality. - */ -import { apiGet } from './api.js'; -import { hideEl, setText, showEl } from './dom.js'; -import state from './state.js'; - -/** - * Load version info from the API and populate header badge. - */ -export async function loadHeaderVersion() { - try { - const data = await apiGet('/api/settings/version'); - state.appVersionData = data; - setText('header-version', data.version || '0.0.0'); - } catch { - setText('header-version', '?'); - } -} - -export function openAboutModal() { - showEl('about-modal'); - document.body.style.overflow = 'hidden'; - - if (state.appVersionData) { - setText('about-version', state.appVersionData.version || '0.0.0'); - setText('about-commit', state.appVersionData.commit || '-'); - setText('about-branch', state.appVersionData.branch || '-'); - } -} - -export function closeAboutModal() { - hideEl('about-modal'); - document.body.style.overflow = ''; -} - -/** - * Initialize about modal event listeners. - */ -export function initAboutModal() { - document.addEventListener('keydown', (e) => { - if (e.key === 'Escape') { - closeAboutModal(); - } - }); - - const modal = document.getElementById('about-modal'); - if (modal) { - modal.addEventListener('click', (e) => { - if (e.target === modal) { - closeAboutModal(); - } - }); - } -} diff --git a/app/static/js/modules/analysis.js b/app/static/js/modules/analysis.js deleted file mode 100644 index d13ce1e..0000000 --- a/app/static/js/modules/analysis.js +++ /dev/null @@ -1,421 +0,0 @@ -/** - * File analysis/validation and combined upload progress (Step 3). - * - * Performance: Instead of creating one DOM row per file (20K+ rows), - * we track status in a Map and only render active uploads (max ~4-8 rows). - * Status counters are maintained via simple arithmetic, not DOM scans. - */ -import { apiGet } from './api.js'; -import { hideEl, setText, showEl } from './dom.js'; -import { formatBytes } from './formatters.js'; -import { showNotification } from './notify.js'; -import state from './state.js'; -import { setUploadStep } from './stepper.js'; -import { showCompletionSummary, updateProgressUI } from './upload-exec.js'; - -// Progress weighting: analysis is typically slower than upload -const ANALYSIS_WEIGHT = 70; -const UPLOAD_WEIGHT = 30; - -/** - * Cache of last-known status per filename. - * @type {Map} - */ -const fileRowStatusCache = new Map(); - -/** Pending file updates to flush in the next animation frame. @type {Map} */ -const pendingFileUpdates = new Map(); - -/** Whether a requestAnimationFrame is already scheduled. */ -let rafScheduled = false; - -/** Pending overall progress data to flush. @type {any} */ -let pendingProgressData = null; - -/** - * Status counters — maintained via transitions, never DOM-scanned. - * @type {{ pending: number, analyzing: number, ready: number, uploading: number, completed: number, skipped: number, failed: number }} - */ -const counts = { - pending: 0, - analyzing: 0, - ready: 0, - uploading: 0, - completed: 0, - skipped: 0, - failed: 0, -}; - -/** - * Reset internal state (call when starting a new job or resetting). - */ -export function resetAnalysisState() { - fileRowStatusCache.clear(); - pendingFileUpdates.clear(); - pendingProgressData = null; - rafScheduled = false; - - counts.pending = 0; - counts.analyzing = 0; - counts.ready = 0; - counts.uploading = 0; - counts.completed = 0; - counts.skipped = 0; - counts.failed = 0; -} - -/** - * Initialize status counters for a new job. - * @param {number} totalFiles - */ -export function initStatusCounts(totalFiles) { - resetAnalysisState(); - counts.pending = totalFiles; - updateCounterDisplay(); -} - -/** - * Check for an active upload job and restore UI state. - */ -export async function checkForActiveJob() { - try { - const data = await apiGet('/api/upload/active'); - if (!data.job_id) return; - - state.currentJobId = data.job_id; - const job = data.job; - - if (job.status === 'analyzing' || job.status === 'uploading') { - setUploadStep(3); - hideEl('folder-browser-panel'); - showEl('upload-section'); - - if (job.status === 'uploading') { - setText('upload-phase-label', 'Uploading files...'); - } - - connectCombinedProgressStream(state.currentJobId); - } else if (job.status === 'ready') { - setUploadStep(3); - hideEl('folder-browser-panel'); - showEl('upload-section'); - connectCombinedProgressStream(state.currentJobId); - } else if ( - job.status === 'completed' || - job.status === 'failed' || - job.status === 'cancelled' - ) { - setUploadStep(4); - hideEl('folder-browser-panel'); - showEl('completion-section'); - showCompletionSummary(job); - } - } catch (error) { - console.log('No active job found:', /** @type {Error} */ (error).message); - } -} - -/** - * Connect to SSE stream for combined validation + upload progress. - * @param {string | null} jobId - */ -export function connectCombinedProgressStream(jobId) { - if (state.eventSource) { - state.eventSource.close(); - } - - resetAnalysisState(); - - state.eventSource = new EventSource(`/api/upload/progress/${jobId}`); - - let analysisTotal = 0; - let analysisCompleted = 0; - - state.eventSource.onmessage = (event) => { - const data = JSON.parse(event.data); - - if (data.error) { - showNotification(data.error, 'error'); - state.eventSource?.close(); - return; - } - - // Analysis progress: queue per-file update for next frame - if (data.type === 'analysis_progress') { - queueFileUpdate(data.file); - - // Set total from the first event that carries it - if (data.total_files && analysisTotal === 0) { - analysisTotal = data.total_files; - counts.pending = analysisTotal; - setText('files-total', data.total_files); - } - - // Only count terminal statuses for progress bar (not pending/analyzing) - if (data.file.status !== 'pending' && data.file.status !== 'analyzing') { - analysisCompleted++; - } - - // Update progress bar for analysis phase - if (analysisTotal > 0) { - const percent = (analysisCompleted / analysisTotal) * ANALYSIS_WEIGHT; - setProgressBar(percent); - } - } - - // Analysis complete: transition to upload phase - if (data.type === 'analysis_complete') { - setProgressBar(ANALYSIS_WEIGHT); - setPhaseLabel('Preparing upload...'); - - if (!data.auto_upload) { - state.eventSource?.close(); - state.eventSource = null; - } - } - - // Auto-upload starting - if (data.type === 'auto_upload_starting') { - setPhaseLabel('Uploading files...'); - } - - // Upload progress updates (job-level data without a type field) - if (!data.type && data.job_id && data.status) { - if (data.status === 'uploading') { - // Remap upload progress into the UPLOAD_WEIGHT portion of the bar - const adjusted = { - ...data, - progress_percent: ANALYSIS_WEIGHT + (data.progress_percent / 100) * UPLOAD_WEIGHT, - }; - pendingProgressData = adjusted; - - // Queue per-file row updates - if (data.files) { - for (const file of data.files) { - queueFileUpdate(file); - } - } - scheduleRaf(); - } else if (['completed', 'failed', 'cancelled'].includes(data.status)) { - state.eventSource?.close(); - state.eventSource = null; - showCompletionSummary(data); - } - } - }; - - state.eventSource.onerror = () => { - if (state.eventSource) { - state.eventSource.close(); - state.eventSource = null; - showNotification('Connection to server lost', 'error'); - } - }; -} - -/** - * Set the overall progress bar value (CSS transition handles smoothing). - * @param {number} percent - */ -function setProgressBar(percent) { - const progressBar = /** @type {HTMLElement | null} */ (document.getElementById('progress-bar')); - if (progressBar) progressBar.style.width = `${percent}%`; - setText('progress-percent', percent.toFixed(1)); -} - -/** - * Update the phase label with an opacity fade transition. - * @param {string} text - */ -function setPhaseLabel(text) { - const phaseLabel = document.getElementById('upload-phase-label'); - if (!phaseLabel || phaseLabel.textContent === text) return; - phaseLabel.style.opacity = '0'; - setTimeout(() => { - phaseLabel.textContent = text; - phaseLabel.style.opacity = '1'; - }, 150); -} - -/** - * Queue a file update for the next animation frame. - * @param {any} fileData - */ -function queueFileUpdate(fileData) { - pendingFileUpdates.set(fileData.filename, fileData); - scheduleRaf(); -} - -/** Schedule a requestAnimationFrame if not already pending. */ -function scheduleRaf() { - if (!rafScheduled) { - rafScheduled = true; - requestAnimationFrame(flushUpdates); - } -} - -/** - * Flush all pending updates in a single animation frame. - * Updates status counters and active upload rows — no per-file DOM scan. - */ -function flushUpdates() { - rafScheduled = false; - - // Process all pending file updates: track status transitions and active files - for (const [filename, fileData] of pendingFileUpdates) { - const oldStatus = fileRowStatusCache.get(filename) || 'pending'; - const newStatus = fileData.is_duplicate ? 'skipped' : fileData.status; - - if (oldStatus !== newStatus) { - // Decrement old counter - if (oldStatus in counts) counts[/** @type {keyof counts} */ (oldStatus)]--; - // Increment new counter - if (newStatus in counts) counts[/** @type {keyof counts} */ (newStatus)]++; - - fileRowStatusCache.set(filename, newStatus); - } - } - - // Update active upload rows (only currently uploading/analyzing files) - updateActiveRows(); - - pendingFileUpdates.clear(); - - // Update status counter display - updateCounterDisplay(); - - // Flush pending overall progress - if (pendingProgressData) { - updateProgressUI(pendingProgressData); - pendingProgressData = null; - } -} - -/** - * Update the active upload rows — only render files that are currently - * uploading or analyzing (max ~4-8 rows instead of 20K). - */ -function updateActiveRows() { - const container = document.getElementById('upload-active-list'); - if (!container) return; - - // Collect currently active files from pending updates - /** @type {any[]} */ - const activeFiles = []; - for (const [, fileData] of pendingFileUpdates) { - if (fileData.status === 'uploading' || fileData.status === 'analyzing') { - activeFiles.push(fileData); - } - } - - // Also keep rows that are still active but weren't in this update batch - const existingRows = container.querySelectorAll('[data-upload-file]'); - for (const row of existingRows) { - const filename = /** @type {HTMLElement} */ (row).getAttribute('data-upload-file') || ''; - const currentStatus = fileRowStatusCache.get(filename); - if (currentStatus === 'uploading' || currentStatus === 'analyzing') { - // Keep it if not being replaced by a pending update - if (!pendingFileUpdates.has(filename)) continue; - } else { - // Status changed to non-active — remove row - row.remove(); - } - } - - // Upsert active file rows - for (const fileData of activeFiles) { - const row = container.querySelector(`[data-upload-file="${CSS.escape(fileData.filename)}"]`); - - if (row) { - // Existing row — micro-update for progress - if (fileData.status === 'uploading') { - const bar = /** @type {HTMLElement | null} */ (row.querySelector('[data-progress-bar]')); - if (bar) bar.style.width = `${fileData.progress_percent || 0}%`; - const label = row.querySelector('[data-progress-label]'); - if (label) { - label.textContent = - fileData.progress_percent != null ? `${fileData.progress_percent.toFixed(0)}%` : ''; - } - } else { - // Status changed (e.g., pending → analyzing) — rebuild - row.innerHTML = buildActiveRowHTML(fileData); - } - } else { - // New active row - const div = document.createElement('div'); - div.className = 'px-6 py-3 flex items-center justify-between'; - div.setAttribute('data-upload-file', fileData.filename); - div.innerHTML = buildActiveRowHTML(fileData); - container.appendChild(div); - } - } - - // Remove rows for files that completed/failed/skipped (no longer active) - for (const row of container.querySelectorAll('[data-upload-file]')) { - const filename = /** @type {HTMLElement} */ (row).getAttribute('data-upload-file') || ''; - const status = fileRowStatusCache.get(filename); - if (status !== 'uploading' && status !== 'analyzing') { - row.remove(); - } - } -} - -/** - * Build HTML for an active file row. - * @param {any} fileData - * @returns {string} - */ -function buildActiveRowHTML(fileData) { - let statusIcon = ''; - let statusText = ''; - - if (fileData.status === 'uploading') { - statusIcon = - ''; - const pct = fileData.progress_percent != null ? `${fileData.progress_percent.toFixed(0)}%` : ''; - statusText = ` -
-
-
-
- ${pct} -
`; - } else if (fileData.status === 'analyzing') { - statusIcon = - ''; - statusText = 'Validating'; - } - - return ` -
- ${statusIcon || '
'} - ${fileData.filename} -
-
- ${fileData.file_size_formatted ? `${fileData.file_size_formatted}` : ''} - ${statusText} -
- `; -} - -/** - * Update the status counter display from the counts object. - * O(1) — just setting text on 5 elements. - */ -function updateCounterDisplay() { - const active = counts.analyzing + counts.uploading; - const queued = counts.pending + counts.ready; - setText('upload-count-active', String(active)); - setText('upload-count-completed', String(counts.completed)); - setText('upload-count-skipped', String(counts.skipped)); - setText('upload-count-failed', String(counts.failed)); - setText('upload-count-queued', String(queued)); -} - -/** - * @param {any} fileData - * @returns {string} - */ -export function formatFileSize(fileData) { - return fileData.file_size_formatted || formatBytes(fileData.file_size || 0); -} diff --git a/app/static/js/modules/api.js b/app/static/js/modules/api.js deleted file mode 100644 index ca68506..0000000 --- a/app/static/js/modules/api.js +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Shared fetch wrappers for JSON API calls. - */ - -/** - * GET a JSON endpoint. Throws on non-ok responses. - * @param {string} url - * @returns {Promise} - */ -export async function apiGet(url) { - const response = await fetch(url); - const data = await response.json(); - if (!response.ok) throw new Error(data.error || `Request failed (${response.status})`); - return data; -} - -/** - * POST to a JSON endpoint. Throws on non-ok responses. - * @param {string} url - * @param {any} [body] - If provided, sent as JSON with Content-Type header. - * @returns {Promise} - */ -export async function apiPost(url, body) { - /** @type {RequestInit} */ - const options = { method: 'POST' }; - if (body !== undefined) { - options.headers = { 'Content-Type': 'application/json' }; - options.body = JSON.stringify(body); - } - const response = await fetch(url, options); - const data = await response.json(); - if (!response.ok) throw new Error(data.error || `Request failed (${response.status})`); - return data; -} - -/** - * PUT to a JSON endpoint. Throws on non-ok responses. - * @param {string} url - * @param {any} body - * @returns {Promise} - */ -export async function apiPut(url, body) { - const response = await fetch(url, { - method: 'PUT', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(body), - }); - const data = await response.json(); - if (!response.ok) throw new Error(data.error || `Request failed (${response.status})`); - return data; -} diff --git a/app/static/js/modules/debounce.js b/app/static/js/modules/debounce.js deleted file mode 100644 index d7b0b72..0000000 --- a/app/static/js/modules/debounce.js +++ /dev/null @@ -1,16 +0,0 @@ -/** - * Debounce utility. - */ - -/** - * @param {(...args: any[]) => void} fn - * @param {number} delay - * @returns {(...args: any[]) => void} - */ -export function debounce(fn, delay) { - let timeout = 0; - return (/** @type {any[]} */ ...args) => { - clearTimeout(timeout); - timeout = window.setTimeout(() => fn(...args), delay); - }; -} diff --git a/app/static/js/modules/dom.js b/app/static/js/modules/dom.js deleted file mode 100644 index 0af1309..0000000 --- a/app/static/js/modules/dom.js +++ /dev/null @@ -1,73 +0,0 @@ -/** - * DOM manipulation helpers. - */ - -/** - * Set the text content of an element by ID. - * @param {string} id - * @param {string | number} value - */ -export function setText(id, value) { - const el = document.getElementById(id); - if (el) el.textContent = String(value); -} - -/** - * Remove the `hidden` class from an element by ID. - * @param {string} id - */ -export function showEl(id) { - document.getElementById(id)?.classList.remove('hidden'); -} - -/** - * Add the `hidden` class to an element by ID. - * @param {string} id - */ -export function hideEl(id) { - document.getElementById(id)?.classList.add('hidden'); -} - -/** - * Disable a button, swap its label, run a callback, then restore. - * @param {HTMLButtonElement | null} btn - * @param {string} loadingText - * @param {() => Promise} callback - */ -/** - * Toggle the `hidden` class on an element by ID. - * @param {string} id - * @param {boolean} show - When true, remove `hidden`; when false, add it. - */ -export function toggleEl(id, show) { - document.getElementById(id)?.classList.toggle('hidden', !show); -} - -/** - * Append text to an element's textContent by ID. - * @param {string} id - * @param {string} value - */ -export function appendText(id, value) { - const el = document.getElementById(id); - if (el) el.textContent += String(value); -} - -/** - * Disable a button, swap its label, run a callback, then restore. - * @param {HTMLButtonElement | null} btn - * @param {string} loadingText - * @param {() => Promise} callback - */ -export async function withLoadingButton(btn, loadingText, callback) { - if (!btn) return; - const originalText = btn.textContent; - btn.disabled = true; - btn.textContent = loadingText; - try { - await callback(); - } finally { - btn.disabled = false; - btn.textContent = originalText; - } -} diff --git a/app/static/js/modules/file-browser.js b/app/static/js/modules/file-browser.js deleted file mode 100644 index cfc58ea..0000000 --- a/app/static/js/modules/file-browser.js +++ /dev/null @@ -1,205 +0,0 @@ -/** - * S3 file browser page functionality. - */ -import { apiGet } from './api.js'; -import { debounce } from './debounce.js'; -import { hideEl, setText, showEl } from './dom.js'; -import { formatBytes } from './formatters.js'; -import { fileIcon, folderIcon } from './icons.js'; -import { showNotification } from './notify.js'; -import state from './state.js'; - -export function initFileBrowser() { - const refreshBtn = document.getElementById('refresh-btn'); - const searchInput = /** @type {HTMLInputElement | null} */ ( - document.getElementById('search-input') - ); - const retryBtn = document.getElementById('retry-btn'); - const closeSearchBtn = document.getElementById('close-search-btn'); - - if (!refreshBtn) return; - - refreshBtn.addEventListener('click', () => loadFiles(state.currentPrefix)); - retryBtn?.addEventListener('click', () => loadFiles(state.currentPrefix)); - closeSearchBtn?.addEventListener('click', hideSearchResults); - - const debouncedSearch = debounce(() => { - const query = searchInput?.value.trim() || ''; - if (query.length >= 2) { - searchFiles(query); - } else { - hideSearchResults(); - } - }, 300); - searchInput?.addEventListener('input', debouncedSearch); - - loadSettings().then(() => loadFiles('')); -} - -async function loadSettings() { - try { - const settings = await apiGet('/api/settings'); - setText('bucket-name', settings.s3_bucket || 'No bucket configured'); - } catch (_error) { - setText('bucket-name', 'Error loading settings'); - } -} - -/** - * @param {string} prefix - */ -async function loadFiles(prefix) { - state.currentPrefix = prefix; - - showEl('loading-state'); - hideEl('error-state'); - hideEl('empty-state'); - hideEl('file-list'); - - try { - const data = await apiGet(`/api/files/list?prefix=${encodeURIComponent(prefix)}`); - - if (!data.success) { - throw new Error(data.error || 'Failed to load files'); - } - - hideEl('loading-state'); - - updateBreadcrumb(data.breadcrumbs || []); - - if (data.folders.length === 0 && data.files.length === 0) { - showEl('empty-state'); - return; - } - - displayFiles(data.folders, data.files); - } catch (error) { - hideEl('loading-state'); - showEl('error-state'); - setText('error-message', /** @type {Error} */ (error).message); - } -} - -/** - * @param {Array<{ name: string, prefix: string }>} breadcrumbs - */ -function updateBreadcrumb(breadcrumbs) { - const nav = document.getElementById('breadcrumb'); - if (!nav) return; - - nav.innerHTML = ` - Root - ${breadcrumbs - .map( - (b) => ` - / - ${b.name} - `, - ) - .join('')} - `; - - for (const link of nav.querySelectorAll('a')) { - link.addEventListener('click', (e) => { - e.preventDefault(); - loadFiles(/** @type {HTMLElement} */ (link).dataset.prefix || ''); - }); - } -} - -/** - * @param {Array<{ name: string, prefix: string }>} folders - * @param {Array<{ name: string, key: string, size: number, last_modified?: string }>} files - */ -function displayFiles(folders, files) { - const fileList = document.getElementById('file-list'); - if (!fileList) return; - - fileList.innerHTML = [ - ...folders.map( - (folder) => ` -
- ${folderIcon()} - ${folder.name}/ -
- `, - ), - ...files.map( - (file) => ` -
-
- ${fileIcon()} - ${file.name} -
-
- ${formatBytes(file.size)} - ${file.last_modified ? new Date(file.last_modified).toLocaleString() : '-'} -
-
- `, - ), - ].join(''); - - showEl('file-list'); - - for (const el of fileList.querySelectorAll('[data-prefix]')) { - el.addEventListener('click', () => - loadFiles(/** @type {HTMLElement} */ (el).dataset.prefix || ''), - ); - } -} - -/** - * @param {string} query - */ -async function searchFiles(query) { - try { - const data = await apiGet( - `/api/files/search?query=${encodeURIComponent(query)}&prefix=${encodeURIComponent(state.currentPrefix)}`, - ); - showSearchResults(data.files, query); - } catch (error) { - showNotification(/** @type {Error} */ (error).message, 'error'); - } -} - -/** - * @param {Array<{ name: string, key: string, size: number }>} files - * @param {string} _query - */ -function showSearchResults(files, _query) { - const container = document.getElementById('search-results'); - const list = document.getElementById('search-results-list'); - if (!container || !list) return; - - list.innerHTML = - files.length === 0 - ? '
No files found
' - : files - .map( - (file) => ` -
-
- ${fileIcon()} -
-
${file.name}
-
${file.key}
-
-
- ${formatBytes(file.size)} -
- `, - ) - .join(''); - - showEl('search-results'); -} - -function hideSearchResults() { - hideEl('search-results'); - - const searchInput = /** @type {HTMLInputElement | null} */ ( - document.getElementById('search-input') - ); - if (searchInput) searchInput.value = ''; -} diff --git a/app/static/js/modules/folder-browser.js b/app/static/js/modules/folder-browser.js deleted file mode 100644 index a9e7292..0000000 --- a/app/static/js/modules/folder-browser.js +++ /dev/null @@ -1,952 +0,0 @@ -import { connectCombinedProgressStream, initStatusCounts } from './analysis.js'; -import { apiGet, apiPost } from './api.js'; -import { hideEl, setText, showEl } from './dom.js'; -import { formatBytes, formatMtime } from './formatters.js'; -import { fileIcon, folderIcon } from './icons.js'; -import { showNotification } from './notify.js'; -import { toggleSort, updateSortIndicators } from './sorting-helpers.js'; -/** - * Inline folder browser and scan results for folder-based upload. - */ -import state from './state.js'; -import { setUploadStep, showUploadSteps } from './stepper.js'; - -/** - * Initialize the inline folder browser and auto-load initial folder. - */ -export async function initFolderBrowser() { - const panel = document.getElementById('folder-browser-panel'); - if (!panel) return; - - document.getElementById('select-folder-btn')?.addEventListener('click', selectCurrentFolder); - - // Delegated click handler for folder navigation (shared across 3 containers) - /** @param {Event} e */ - function handleFolderNavClick(e) { - const target = /** @type {HTMLElement} */ (e.target).closest('[data-action="navigate-folder"]'); - if (target) { - loadFolderBrowser(/** @type {HTMLElement} */ (target).dataset.path || ''); - } - } - - document.getElementById('folder-list')?.addEventListener('click', handleFolderNavClick); - document.getElementById('folder-quick-links')?.addEventListener('click', handleFolderNavClick); - document.getElementById('folder-breadcrumb')?.addEventListener('click', handleFolderNavClick); - - // Click handler for Review table (folder expand/collapse) - const scanSection = document.getElementById('scan-results-section'); - if (scanSection) { - scanSection.addEventListener('click', (e) => { - const folderHeader = /** @type {HTMLElement} */ (e.target).closest('[data-folder-header]'); - if (folderHeader) { - toggleFolderExpand(/** @type {HTMLTableRowElement} */ (folderHeader)); - } - }); - } - - // Sort header click handler (File browser table) - const fileTableHead = document.getElementById('file-table-head'); - if (fileTableHead) { - fileTableHead.addEventListener('click', (e) => { - const th = /** @type {HTMLElement} */ (e.target).closest('[data-file-sort]'); - if (th) { - sortBrowserFileTable(/** @type {HTMLElement} */ (th).dataset.fileSort || 'name'); - } - }); - } - - // Auto-load: use last folder or default from settings - const lastFolder = localStorage.getItem('lastUploadFolder'); - if (lastFolder) { - loadFolderBrowser(lastFolder); - } else { - try { - const settings = await apiGet('/api/settings'); - loadFolderBrowser(settings.default_upload_folder || ''); - } catch (_error) { - loadFolderBrowser(''); - } - } -} - -/** - * Load folder contents from the backend. - * Uses raw fetch for complex retry-on-failure logic. - * @param {string} path - * @param {boolean} [isRetry] - */ -export async function loadFolderBrowser(path, isRetry = false) { - hideEl('folder-list'); - hideEl('folder-error'); - showEl('folder-loading'); - - try { - const url = path ? `/api/files/browse?path=${encodeURIComponent(path)}` : '/api/files/browse'; - const response = await fetch(url); - const data = await response.json(); - - if (!response.ok) { - if (path && !isRetry) { - console.warn(`Failed to load saved folder "${path}", falling back to home`); - loadFolderBrowser('', true); - return; - } - throw new Error(data.error || 'Failed to load folder'); - } - - // Update quick links (using data-action instead of onclick) - const quickLinksContainer = document.getElementById('folder-quick-links'); - if (quickLinksContainer) { - const lastUsedFolder = localStorage.getItem('lastUploadFolder'); - let quickLinksHtml = ''; - - if (lastUsedFolder && lastUsedFolder !== data.current_path) { - const lastFolderName = lastUsedFolder.split('/').pop() || lastUsedFolder; - quickLinksHtml += ` - - `; - } - - quickLinksHtml += data.quick_links - .map( - (/** @type {{ name: string, path: string }} */ link) => ` - - `, - ) - .join(''); - - quickLinksContainer.innerHTML = quickLinksHtml; - } - - // Update breadcrumbs - const breadcrumbContainer = document.getElementById('folder-breadcrumb'); - if (breadcrumbContainer) { - breadcrumbContainer.innerHTML = data.breadcrumbs - .map( - (/** @type {{ name: string, path: string }} */ crumb, /** @type {number} */ i) => ` - ${i > 0 ? '/' : ''} - - `, - ) - .join(''); - } - - // Update MCAP count - setText('folder-mcap-count', data.mcap_count); - - // Enable select button and store current path - const selectFolderBtn = /** @type {HTMLButtonElement | null} */ ( - document.getElementById('select-folder-btn') - ); - if (selectFolderBtn) { - selectFolderBtn.disabled = false; - selectFolderBtn.dataset.path = data.current_path; - } - - // Update bottom summary with MCAP count - if (data.mcap_count > 0) { - setText( - 'folder-select-summary', - `${data.mcap_count} MCAP file${data.mcap_count === 1 ? '' : 's'} in this folder.`, - ); - } else { - setText( - 'folder-select-summary', - 'Navigate to the folder containing your MCAP files, then click Upload Folder.', - ); - } - - // Render folder list - hideEl('folder-loading'); - showEl('folder-list'); - - const folderList = document.getElementById('folder-list'); - if (folderList) { - if (data.folders.length === 0 && data.files.length === 0) { - folderList.innerHTML = - '
This folder is empty
'; - } else { - folderList.innerHTML = [ - data.parent_path - ? ` -
- - - - .. -
- ` - : '', - ...data.folders.map( - (/** @type {{ name: string, path: string, mcap_count: number }} */ folder) => ` -
-
- ${folderIcon()} - ${folder.name} -
- ${ - folder.mcap_count > 0 - ? `${folder.mcap_count} mcap` - : '' - } -
- `, - ), - ].join(''); - } - } - - // Render MCAP file table - if (data.files.length > 0) { - state.browserFiles = data.files; - state.browserFileSortConfig = state.browserFileSortConfig || { - column: 'name', - ascending: true, - }; - renderBrowserFileTable(); - showEl('file-table-section'); - } else { - state.browserFiles = []; - hideEl('file-table-section'); - } - - // Start background scan for upload status enrichment - startBrowserScan(data.current_path); - } catch (error) { - hideEl('folder-loading'); - showEl('folder-error'); - setText('folder-error-message', /** @type {Error} */ (error).message); - } -} - -/** - * Select the current folder and populate Step 2 from stored browser scan data. - * The browser scan has already accumulated state.scanFileStatuses and state.scanFilePaths. - */ -function selectCurrentFolder() { - const btn = /** @type {HTMLButtonElement | null} */ ( - document.getElementById('select-folder-btn') - ); - if (!btn) return; - - const folderPath = btn.dataset.path; - if (!folderPath) return; - - state.selectedFolderPath = folderPath; - state.scanFolderPath = folderPath; - localStorage.setItem('lastUploadFolder', folderPath); - - // Transition to Step 2 — populate from stored browser scan data - showUploadSteps(2); - state.reviewSortConfig = { column: 'filename', ascending: true }; - - setText('selected-folder-path', folderPath); - setText('scan-total', String(state.scanFileStatuses.length)); - setText('scan-total-volume', formatBytes(state.scanTotalSize)); - const uploadedCount = state.scanFileStatuses.filter( - (/** @type {any} */ f) => f.already_uploaded, - ).length; - setText('scan-already-uploaded', String(uploadedCount)); - - // Clear and populate review table from stored scan results - const tbody = document.getElementById('scan-file-list'); - if (tbody) tbody.innerHTML = ''; - - for (let i = 0; i < state.browserScanResults.length; i++) { - appendFolderToReviewTable(state.browserScanResults[i], i); - } - - // Hide scan progress (scan already done), enable continue button - hideEl('scan-progress-container'); - const continueBtn = /** @type {HTMLButtonElement | null} */ ( - document.getElementById('continue-upload-btn') - ); - if (continueBtn) { - continueBtn.disabled = state.scanFilePaths.length === 0; - } - - const hideUploadedCheckbox = /** @type {HTMLInputElement | null} */ ( - document.getElementById('scan-hide-uploaded') - ); - if (hideUploadedCheckbox) { - hideUploadedCheckbox.checked = false; - hideUploadedCheckbox.onchange = () => applyScanFileFilter(); - } - - const hideCompletedFoldersCheckbox = /** @type {HTMLInputElement | null} */ ( - document.getElementById('scan-hide-completed-folders') - ); - if (hideCompletedFoldersCheckbox) { - hideCompletedFoldersCheckbox.checked = false; - hideCompletedFoldersCheckbox.onchange = () => applyScanFileFilter(); - } - - hideEl('folder-browser-panel'); - showEl('scan-results-section'); -} - -/** - * Green checkmark SVG for fully-uploaded folders. - * @returns {string} - */ -function checkmarkIcon() { - return ''; -} - -/** - * Start a background cache-only scan for the current folder to enrich - * folder rows with upload status. Called automatically after navigation. - * @param {string} folderPath - */ -async function startBrowserScan(folderPath) { - // Cancel any existing browser scan - if (state.browserScanJobId) { - try { - await apiPost(`/api/upload/cancel/${state.browserScanJobId}`); - } catch (_error) { - // Scan may have already finished - } - } - if (state.browserScanEventSource) { - state.browserScanEventSource.close(); - state.browserScanEventSource = null; - } - - // Reset browser scan state - state.browserScanResults = []; - state.browserScanJobId = null; - state.browserScanComplete = false; - state.scanFileStatuses = []; - state.scanFilePaths = []; - state.scanTotalSize = 0; - - // Running aggregation map: immediate child folder name → { totalFiles, alreadyUploaded } - /** @type {Map} */ - const folderAggregation = new Map(); - - // Root file aggregation (for files in the browsed folder itself) - let rootTotalFiles = 0; - let rootAlreadyUploaded = 0; - - // Disable Upload button until scan completes - const selectBtn = /** @type {HTMLButtonElement | null} */ ( - document.getElementById('select-folder-btn') - ); - if (selectBtn) { - selectBtn.disabled = true; - const btnSpan = selectBtn.querySelector('span'); - if (btnSpan) btnSpan.textContent = 'Scanning...'; - } - - // Show scan status bar - showEl('browser-scan-status'); - const spinner = document.getElementById('browser-scan-spinner'); - if (spinner) spinner.classList.remove('hidden'); - setText('browser-scan-text', 'Scanning folders...'); - - // Wire up hide-uploaded toggle - const hideCheckbox = /** @type {HTMLInputElement | null} */ ( - document.getElementById('browser-hide-uploaded') - ); - if (hideCheckbox) { - hideCheckbox.onchange = () => applyBrowserHideUploaded(hideCheckbox.checked); - } - - try { - const data = await apiPost('/api/upload/scan-folder-async', { - folder_path: folderPath, - cache_only: true, - }); - - state.browserScanJobId = data.job_id; - - // Connect SSE for browser scan progress - state.browserScanEventSource = new EventSource(`/api/upload/progress/${data.job_id}`); - - state.browserScanEventSource.onmessage = (event) => { - const eventData = JSON.parse(event.data); - - if (eventData.error) { - showNotification(eventData.error, 'error'); - closeBrowserScan(); - enableUploadButton(folderPath); - return; - } - - if (eventData.type === 'scan_started') { - setText('browser-scan-text', `Scanning 0 of ${eventData.folders_total} folders...`); - } - - if (eventData.type === 'scan_folder_complete') { - const folder = eventData.folder; - const totals = eventData.running_totals; - - // Store result - state.browserScanResults.push(folder); - state.scanTotalSize = totals.total_size; - - // Accumulate file statuses and paths - for (const file of folder.files) { - state.scanFileStatuses.push(file); - if (!file.already_uploaded) { - state.scanFilePaths.push(file.path); - } - } - - // Determine which immediate child this leaf folder belongs to - if (folder.relative_path === '.') { - // Root folder files - rootTotalFiles += folder.total_files; - rootAlreadyUploaded += folder.already_uploaded; - updateRootUploadStatus(rootTotalFiles, rootAlreadyUploaded); - } else { - // Extract the first path component - const firstSlash = folder.relative_path.indexOf('/'); - const immediateChild = - firstSlash >= 0 ? folder.relative_path.substring(0, firstSlash) : folder.relative_path; - - const existing = folderAggregation.get(immediateChild) || { - totalFiles: 0, - alreadyUploaded: 0, - }; - existing.totalFiles += folder.total_files; - existing.alreadyUploaded += folder.already_uploaded; - folderAggregation.set(immediateChild, existing); - - // Update the matching folder row in the browser - updateFolderRowBadge(immediateChild, existing); - } - - // Update status bar - setText( - 'browser-scan-text', - `Scanning ${eventData.folders_scanned} of ${eventData.folders_total} folders...`, - ); - - // Update Upload button summary - updateUploadButtonSummary(); - } - - if (eventData.type === 'scan_complete') { - closeBrowserScan(); - state.browserScanComplete = true; - - // Hide spinner, show final status - if (spinner) spinner.classList.add('hidden'); - - const totalFiles = state.scanFileStatuses.length; - const alreadyUploaded = state.scanFileStatuses.filter( - (/** @type {any} */ f) => f.already_uploaded, - ).length; - const toUpload = totalFiles - alreadyUploaded; - - if (totalFiles === 0) { - setText('browser-scan-text', 'No MCAP files found in subfolders.'); - enableUploadButton(folderPath); - } else if (toUpload === 0) { - setText('browser-scan-text', `All ${totalFiles} files already uploaded.`); - enableUploadButton(folderPath, true); - } else { - setText( - 'browser-scan-text', - `Scan complete: ${toUpload} to upload, ${alreadyUploaded} already uploaded.`, - ); - enableUploadButton(folderPath); - } - } - }; - - state.browserScanEventSource.onerror = () => { - closeBrowserScan(); - enableUploadButton(folderPath); - }; - } catch (error) { - showNotification(/** @type {Error} */ (error).message, 'error'); - closeBrowserScan(); - enableUploadButton(folderPath); - } -} - -/** - * Close the browser scan SSE connection. - */ -function closeBrowserScan() { - if (state.browserScanEventSource) { - state.browserScanEventSource.close(); - state.browserScanEventSource = null; - } -} - -/** - * Re-enable the Upload button after scan completes. - * @param {string} folderPath - * @param {boolean} [allUploaded] - If true, keep button disabled (nothing to upload) - */ -function enableUploadButton(folderPath, allUploaded = false) { - const selectBtn = /** @type {HTMLButtonElement | null} */ ( - document.getElementById('select-folder-btn') - ); - if (!selectBtn) return; - - if (allUploaded) { - selectBtn.disabled = true; - const btnSpan = selectBtn.querySelector('span'); - if (btnSpan) btnSpan.textContent = 'All Files Uploaded'; - } else { - selectBtn.disabled = false; - selectBtn.dataset.path = folderPath; - updateUploadButtonSummary(); - } -} - -/** - * Update the Upload button label and summary text based on scan data. - */ -function updateUploadButtonSummary() { - const toUploadCount = state.scanFilePaths.length; - const totalCount = state.scanFileStatuses.length; - const uploadedCount = totalCount - toUploadCount; - - const selectBtn = /** @type {HTMLButtonElement | null} */ ( - document.getElementById('select-folder-btn') - ); - if (selectBtn) { - const btnSpan = selectBtn.querySelector('span'); - if (btnSpan) { - if (toUploadCount > 0) { - btnSpan.textContent = `Upload ${toUploadCount} Files`; - } else if (totalCount > 0) { - btnSpan.textContent = 'All Files Uploaded'; - } else { - btnSpan.textContent = 'Upload This Folder'; - } - } - } - - // Update bottom summary text - if (totalCount > 0) { - const sizeToUpload = state.scanFileStatuses - .filter((/** @type {any} */ f) => !f.already_uploaded) - .reduce((/** @type {number} */ sum, /** @type {any} */ f) => sum + f.size, 0); - if (toUploadCount > 0) { - setText( - 'folder-select-summary', - `${toUploadCount} of ${totalCount} files to upload (${formatBytes(sizeToUpload)}). ${uploadedCount} already uploaded.`, - ); - } else { - setText('folder-select-summary', `All ${totalCount} files are already uploaded.`); - } - } -} - -/** - * Update a folder row's badge with scan results. - * @param {string} folderName - * @param {{ totalFiles: number, alreadyUploaded: number }} stats - */ -function updateFolderRowBadge(folderName, stats) { - const folderList = document.getElementById('folder-list'); - if (!folderList) return; - - const row = folderList.querySelector(`[data-folder-name="${CSS.escape(folderName)}"]`); - if (!row) return; - - const badge = row.querySelector('.folder-mcap-badge'); - if (!badge) return; - - const allUploaded = stats.alreadyUploaded === stats.totalFiles && stats.totalFiles > 0; - - if (allUploaded) { - badge.innerHTML = `${stats.totalFiles} mcap (${stats.alreadyUploaded} uploaded) ${checkmarkIcon()}`; - row.classList.add('bg-green-50'); - /** @type {HTMLElement} */ (row).dataset.allUploaded = 'true'; - } else if (stats.alreadyUploaded > 0) { - badge.textContent = `${stats.totalFiles} mcap (${stats.alreadyUploaded} uploaded)`; - } else { - badge.textContent = `${stats.totalFiles} mcap`; - } -} - -/** - * Update the root upload status in the MCAP count info bar. - * @param {number} totalFiles - * @param {number} alreadyUploaded - */ -function updateRootUploadStatus(totalFiles, alreadyUploaded) { - const statusEl = document.getElementById('browser-root-upload-status'); - if (!statusEl) return; - - if (alreadyUploaded === totalFiles && totalFiles > 0) { - statusEl.innerHTML = `(${alreadyUploaded} uploaded) ${checkmarkIcon()}`; - } else if (alreadyUploaded > 0) { - statusEl.innerHTML = `(${alreadyUploaded} of ${totalFiles} uploaded)`; - } -} - -/** - * Toggle visibility of fully-uploaded folder rows in the browser. - * @param {boolean} hide - */ -function applyBrowserHideUploaded(hide) { - const folderList = document.getElementById('folder-list'); - if (!folderList) return; - - const rows = folderList.querySelectorAll('[data-folder-name]'); - for (const row of rows) { - if (hide && /** @type {HTMLElement} */ (row).dataset.allUploaded === 'true') { - /** @type {HTMLElement} */ (row).classList.add('hidden'); - } else { - /** @type {HTMLElement} */ (row).classList.remove('hidden'); - } - } -} - -/** - * Append a scanned folder as a collapsed summary row in the review table. - * File rows are only rendered on-demand when the folder header is expanded. - * @param {{ relative_path: string, files: any[], total_files: number, already_uploaded: number, all_uploaded: boolean, error: string | null }} folderData - * @param {number} folderIndex - Index into state.browserScanResults - */ -function appendFolderToReviewTable(folderData, folderIndex) { - const tbody = document.getElementById('scan-file-list'); - if (!tbody) return; - - const folderLabel = folderData.relative_path === '.' ? '(root)' : folderData.relative_path; - const toUpload = folderData.total_files - folderData.already_uploaded; - - // Collapsed folder header only — no file rows (rendered on expand) - const headerHtml = ` - - -
-
- - - - ${folderIcon('h-4 w-4 text-nlr-yellow mr-2')} - ${folderLabel} - ${folderData.all_uploaded ? checkmarkIcon() : ''} -
-
- ${folderData.total_files} files - ${toUpload > 0 ? `${toUpload} to upload` : ''} - ${folderData.already_uploaded > 0 ? `${folderData.already_uploaded} uploaded` : ''} - ${folderData.error ? `Error` : ''} -
-
- - - `; - - tbody.insertAdjacentHTML('beforeend', headerHtml); - applyScanFileFilter(); -} - -/** - * Toggle expand/collapse of a folder in the review table. - * @param {HTMLTableRowElement} headerRow - */ -function toggleFolderExpand(headerRow) { - const idx = Number.parseInt(headerRow.dataset.folderIdx || '0', 10); - const folderData = state.browserScanResults[idx]; - if (!folderData) return; - - const isExpanded = headerRow.dataset.expanded === 'true'; - const chevron = headerRow.querySelector('.folder-chevron'); - - if (isExpanded) { - // Collapse: remove file rows after this header until next header - let next = headerRow.nextElementSibling; - while (next && !next.hasAttribute('data-folder-header')) { - const toRemove = next; - next = next.nextElementSibling; - toRemove.remove(); - } - headerRow.dataset.expanded = 'false'; - if (chevron) chevron.classList.remove('rotate-90'); - } else { - // Expand: render file rows after this header using a DocumentFragment - const fragment = document.createDocumentFragment(); - for (const file of folderData.files) { - const tr = document.createElement('tr'); - tr.className = file.already_uploaded ? 'bg-yellow-50' : ''; - tr.dataset.alreadyUploaded = String(file.already_uploaded); - const dirPath = getDirectoryPart(file.relative_path); - tr.innerHTML = ` - - ${file.filename} - - - ${dirPath || '.'} - - ${formatMtime(file.mtime)} - ${formatBytes(file.size)} - - - ${file.already_uploaded ? 'Uploaded' : 'To Upload'} - - - `; - fragment.appendChild(tr); - } - headerRow.after(fragment); - headerRow.dataset.expanded = 'true'; - if (chevron) chevron.classList.add('rotate-90'); - } -} - -/** - * Get the directory part of a relative path (everything before the filename). - * @param {string} relativePath - * @returns {string} - */ -function getDirectoryPart(relativePath) { - const lastSlash = relativePath.lastIndexOf('/'); - return lastSlash >= 0 ? relativePath.substring(0, lastSlash) : ''; -} - -function applyScanFileFilter() { - const hideUploadedEl = /** @type {HTMLInputElement | null} */ ( - document.getElementById('scan-hide-uploaded') - ); - const hideUploaded = hideUploadedEl?.checked || false; - - const hideCompletedFoldersEl = /** @type {HTMLInputElement | null} */ ( - document.getElementById('scan-hide-completed-folders') - ); - const hideCompletedFolders = hideCompletedFoldersEl?.checked || false; - - const tbody = document.getElementById('scan-file-list'); - if (!tbody) return; - - // Determine which folder headers are for completed folders - /** @type {Set} */ - const hiddenFolderHeaders = new Set(); - - // First pass: process folder headers - const folderHeaders = tbody.querySelectorAll('[data-folder-header]'); - for (const header of folderHeaders) { - if ( - hideCompletedFolders && - /** @type {HTMLElement} */ (header).dataset.allUploaded === 'true' - ) { - header.classList.add('hidden'); - hiddenFolderHeaders.add(header); - } else { - header.classList.remove('hidden'); - } - } - - // Second pass: process file rows - // For each file row, find its parent folder header (the nearest preceding [data-folder-header]) - const allRows = /** @type {NodeListOf} */ (tbody.querySelectorAll('tr')); - /** @type {Element | null} */ - let currentFolderHeader = null; - - for (const row of allRows) { - if (row.hasAttribute('data-folder-header')) { - currentFolderHeader = row; - continue; - } - - // This is a file row - const isUploaded = row.dataset.alreadyUploaded === 'true'; - const inHiddenFolder = currentFolderHeader && hiddenFolderHeaders.has(currentFolderHeader); - - if (inHiddenFolder || (hideUploaded && isUploaded)) { - row.classList.add('hidden'); - } else { - row.classList.remove('hidden'); - } - } -} - -/** - * Show the confirm upload modal. - */ -export function showConfirmModal() { - if (!state.scanFileStatuses || state.scanFileStatuses.length === 0) { - showNotification('No files to upload', 'error'); - return; - } - - // Calculate size of non-uploaded files (default view) - updateConfirmModalCounts(false); - - // Wire up force-reupload checkbox to update counts dynamically - const checkbox = /** @type {HTMLInputElement | null} */ ( - document.getElementById('force-reupload-checkbox') - ); - if (checkbox) { - checkbox.checked = false; - checkbox.onchange = () => updateConfirmModalCounts(checkbox.checked); - } - - showEl('confirm-skip-note'); - showEl('confirm-upload-modal'); -} - -/** - * Update confirm modal file count/size based on force-reupload state. - * @param {boolean} forceReupload - */ -function updateConfirmModalCounts(forceReupload) { - let uploadSize = 0; - let uploadCount = 0; - for (const file of state.scanFileStatuses) { - if (forceReupload || !file.already_uploaded) { - uploadSize += file.size; - uploadCount++; - } - } - - setText('confirm-file-count', uploadCount); - setText('confirm-total-size', formatBytes(uploadSize)); - - if (forceReupload) { - hideEl('confirm-skip-note'); - } else { - showEl('confirm-skip-note'); - } -} - -/** - * Start the combined validate + upload flow. - */ -export async function startCombinedUpload() { - // Close confirm modal - hideEl('confirm-upload-modal'); - - // Check force-reupload state - const forceCheckbox = /** @type {HTMLInputElement | null} */ ( - document.getElementById('force-reupload-checkbox') - ); - const forceReupload = forceCheckbox?.checked || false; - - // Determine which file paths to send - const filePaths = forceReupload - ? state.scanFileStatuses.map((/** @type {any} */ f) => f.path) - : state.scanFilePaths; - - if (!filePaths || filePaths.length === 0) { - showNotification('No files to upload', 'error'); - return; - } - - setUploadStep(3); - - hideEl('scan-results-section'); - showEl('upload-section'); - - // Set phase label - setText('upload-phase-label', 'Validating files...'); - - // Initialize file list with pending status - initUploadFileList(filePaths); - - try { - /** @type {Record} */ - const requestBody = { - file_paths: filePaths, - auto_upload: true, - }; - if (forceReupload) { - requestBody.skip_duplicates = false; - } - - const data = await apiPost('/api/upload/bulk-analyze', requestBody); - - state.currentJobId = data.job_id; - setText('files-total', data.total_files); - - connectCombinedProgressStream(data.job_id); - } catch (error) { - showNotification(/** @type {Error} */ (error).message, 'error'); - } -} - -/** - * Render the sortable file table in the folder browser. - */ -function renderBrowserFileTable() { - const tbody = document.getElementById('file-table-body'); - if (!tbody || !state.browserFiles) return; - - const { column, ascending } = state.browserFileSortConfig; - - const sorted = [...state.browserFiles].sort((a, b) => { - let cmp = 0; - if (column === 'name') { - cmp = a.name.localeCompare(b.name); - } else if (column === 'mtime') { - cmp = (a.mtime || 0) - (b.mtime || 0); - } else if (column === 'size') { - cmp = a.size - b.size; - } - return ascending ? cmp : -cmp; - }); - - tbody.innerHTML = sorted - .map( - (/** @type {{ name: string, size: number, mtime: number }} */ file) => ` - - -
- ${fileIcon('h-4 w-4 text-gray-400 mr-2 flex-shrink-0')} - ${file.name} -
- - ${formatMtime(file.mtime)} - ${formatBytes(file.size)} - - `, - ) - .join(''); - - // Update sort indicators - updateSortIndicators('[data-file-sort]', '.file-sort-indicator', 'fileSort', column, ascending); -} - -/** - * Sort the browser file table by column. - * @param {string} column - */ -function sortBrowserFileTable(column) { - if (!state.browserFileSortConfig) { - state.browserFileSortConfig = { column, ascending: true }; - } else { - toggleSort(state.browserFileSortConfig, column); - } - renderBrowserFileTable(); -} - -/** - * Initialize the upload view for a new job. - * Instead of creating 20K+ DOM rows, initialize status counters and - * clear the active upload area. Active files render on-demand. - * @param {string[]} filePaths - */ -function initUploadFileList(filePaths) { - const activeList = document.getElementById('upload-active-list'); - if (activeList) activeList.innerHTML = ''; - - // Initialize counters (all files start as "pending") - initStatusCounts(filePaths.length); - setText('files-total', String(filePaths.length)); -} diff --git a/app/static/js/modules/formatters.js b/app/static/js/modules/formatters.js deleted file mode 100644 index d3ceaa8..0000000 --- a/app/static/js/modules/formatters.js +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Pure formatting functions for bytes, time, and dates. - */ - -/** - * @param {number} bytes - * @returns {string} - */ -export function formatBytes(bytes) { - if (bytes === 0) return '0 B'; - const k = 1024; - const sizes = ['B', 'KB', 'MB', 'GB', 'TB']; - const i = Math.floor(Math.log(bytes) / Math.log(k)); - return `${Number.parseFloat((bytes / k ** i).toFixed(1))} ${sizes[i]}`; -} - -/** - * @param {number | null | undefined} seconds - * @returns {string} - */ -export function formatEta(seconds) { - if (!seconds || seconds < 0) return 'Calculating...'; - if (seconds < 60) return `${seconds}s`; - if (seconds < 3600) return `${Math.floor(seconds / 60)}m ${seconds % 60}s`; - const hours = Math.floor(seconds / 3600); - const mins = Math.floor((seconds % 3600) / 60); - return `${hours}h ${mins}m`; -} - -/** - * @param {number} seconds - * @returns {string} - */ -export function formatDuration(seconds) { - if (seconds < 1) { - return `${Math.round(seconds * 1000)}ms`; - } - if (seconds < 60) { - return `${seconds.toFixed(1)}s`; - } - const mins = Math.floor(seconds / 60); - const secs = Math.round(seconds % 60); - return `${mins}m ${secs}s`; -} - -/** - * Format a Unix epoch (seconds) into a locale date string. - * @param {number | null | undefined} epochSeconds - * @returns {string} - */ -export function formatMtime(epochSeconds) { - if (!epochSeconds) return '-'; - return new Date(epochSeconds * 1000).toLocaleString('en-US', { - month: 'short', - day: 'numeric', - year: 'numeric', - hour: '2-digit', - minute: '2-digit', - }); -} diff --git a/app/static/js/modules/icons.js b/app/static/js/modules/icons.js deleted file mode 100644 index d9bc569..0000000 --- a/app/static/js/modules/icons.js +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Reusable SVG icon templates. - */ - -const FILE_ICON_PATH = - 'M9 12h6m-6 4h6m2 5H7a2 2 0 01-2-2V5a2 2 0 012-2h5.586a1 1 0 01.707.293l5.414 5.414a1 1 0 01.293.707V19a2 2 0 01-2 2z'; - -const FOLDER_ICON_PATH = 'M2 6a2 2 0 012-2h5l2 2h5a2 2 0 012 2v6a2 2 0 01-2 2H4a2 2 0 01-2-2V6z'; - -/** - * File document SVG icon. - * @param {string} [cls] - CSS classes (default: 'h-5 w-5 text-gray-400 mr-3') - * @returns {string} - */ -export function fileIcon(cls = 'h-5 w-5 text-gray-400 mr-3') { - return ``; -} - -/** - * Folder SVG icon. - * @param {string} [cls] - CSS classes (default: 'h-5 w-5 text-nlr-yellow mr-3') - * @returns {string} - */ -export function folderIcon(cls = 'h-5 w-5 text-nlr-yellow mr-3') { - return ``; -} diff --git a/app/static/js/modules/logs.js b/app/static/js/modules/logs.js deleted file mode 100644 index 1a65cfc..0000000 --- a/app/static/js/modules/logs.js +++ /dev/null @@ -1,425 +0,0 @@ -/** - * Logs viewer page module. - * - * Loads log entries from /api/logs/entries with filtering, pagination, - * expandable detail rows, and S3 sync trigger. - */ -import { apiGet, apiPost } from './api.js'; -import { debounce } from './debounce.js'; -import { hideEl, setText, toggleEl, withLoadingButton } from './dom.js'; -import { showNotification } from './notify.js'; -import state from './state.js'; - -/** - * Format an ISO timestamp for display. - * @param {string} iso - * @returns {string} - */ -function formatTimestamp(iso) { - try { - const d = new Date(iso); - return d.toLocaleString(undefined, { - year: 'numeric', - month: '2-digit', - day: '2-digit', - hour: '2-digit', - minute: '2-digit', - second: '2-digit', - }); - } catch { - return iso; - } -} - -/** - * Return Tailwind classes for a log level badge. - * @param {string} level - * @returns {string} - */ -function levelBadgeClass(level) { - const upper = level.toUpperCase(); - if (upper === 'ERROR') return 'bg-red-100 text-red-800'; - if (upper === 'WARNING') return 'bg-yellow-100 text-yellow-800'; - return 'bg-blue-100 text-blue-800'; -} - -/** - * Get the filter DOM elements (shared by clearFilters and applyFilters). - */ -function getFilterEls() { - return { - date: /** @type {HTMLInputElement | null} */ (document.getElementById('filter-date')), - level: /** @type {HTMLSelectElement | null} */ (document.getElementById('filter-level')), - category: /** @type {HTMLSelectElement | null} */ (document.getElementById('filter-category')), - search: /** @type {HTMLInputElement | null} */ (document.getElementById('filter-search')), - }; -} - -/** - * Render the log table body from an array of entries. - * @param {Array>} entries - */ -function renderLogTable(entries) { - const tbody = document.getElementById('log-table-body'); - if (!tbody) return; - - if (entries.length === 0) { - tbody.innerHTML = - 'No log entries found'; - return; - } - - let html = ''; - for (const entry of entries) { - const ts = formatTimestamp(/** @type {string} */ (entry.timestamp) || ''); - const level = /** @type {string} */ (entry.level) || 'INFO'; - const category = /** @type {string} */ (entry.category) || ''; - const event = /** @type {string} */ (entry.event) || ''; - const message = /** @type {string} */ (entry.message) || ''; - const metadata = /** @type {Record | undefined} */ (entry.metadata); - const hasMetadata = metadata && Object.keys(metadata).length > 0; - - const rowClass = hasMetadata ? 'cursor-pointer hover:bg-gray-50' : ''; - const toggleAttr = hasMetadata ? 'data-log-toggle' : ''; - - html += ` - ${ts} - ${level} - ${category} - ${event} - ${message} - `; - - if (hasMetadata) { - const metaRows = Object.entries(metadata) - .map( - ([k, v]) => - `${k}${typeof v === 'object' ? JSON.stringify(v) : String(v)}`, - ) - .join(''); - - html += ` - - ${metaRows}
- - `; - } - } - - tbody.innerHTML = html; - - // Wire toggle for expandable rows - for (const row of tbody.querySelectorAll('[data-log-toggle]')) { - row.addEventListener('click', () => { - const detail = row.nextElementSibling; - if (detail?.classList.contains('log-detail-row')) { - detail.classList.toggle('hidden'); - } - }); - } -} - -/** Fetch and render log entries using current filters + pagination. */ -export async function loadLogEntries() { - const params = new URLSearchParams(); - const { logFilters, logPagination } = state; - - if (logFilters.date) params.set('date', logFilters.date); - if (logFilters.level) params.set('level', logFilters.level); - if (logFilters.category) params.set('category', logFilters.category); - if (logFilters.search) params.set('search', logFilters.search); - params.set('offset', String(logPagination.offset)); - params.set('limit', String(logPagination.limit)); - - try { - const data = await apiGet(`/api/logs/entries?${params}`); - - renderLogTable(data.entries || []); - - // Update count label - const total = data.total || 0; - const from = total > 0 ? data.offset + 1 : 0; - const to = Math.min(data.offset + data.limit, total); - setText('log-count-label', `Showing ${from}-${to} of ${total}`); - - // Pagination buttons - const prevBtn = /** @type {HTMLButtonElement | null} */ ( - document.getElementById('prev-page-btn') - ); - const nextBtn = /** @type {HTMLButtonElement | null} */ ( - document.getElementById('next-page-btn') - ); - if (prevBtn) prevBtn.disabled = data.offset <= 0; - if (nextBtn) nextBtn.disabled = data.offset + data.limit >= total; - } catch { - setText('log-count-label', 'Failed to load log entries'); - } -} - -/** Fetch and render log stats into the stat cards. */ -export async function loadLogStats() { - try { - const data = await apiGet('/api/logs/stats'); - - setText('stat-total', data.total_entries ?? 0); - setText('stat-today', data.today_entries ?? 0); - setText('stat-errors', data.level_counts?.ERROR ?? 0); - setText('stat-files', data.file_count ?? 0); - } catch { - // Silently fail — stats are non-critical - } -} - -/** POST /api/logs/sync and show notification. */ -export async function syncLogs() { - const btn = /** @type {HTMLButtonElement | null} */ (document.getElementById('sync-logs-btn')); - - await withLoadingButton(btn, 'Syncing...', async () => { - try { - const data = await apiPost('/api/logs/sync'); - - if (data.success) { - showNotification(`Synced ${data.synced} log files to S3`, 'success'); - } else { - showNotification(`Sync failed: ${data.error || 'Unknown error'}`, 'error'); - } - } catch (err) { - showNotification(`Sync error: ${err}`, 'error'); - } - }); -} - -/** Track which CSV is currently previewed (to toggle on re-click). */ -let currentPreviewPath = ''; - -/** - * Extract time (HH:MM:SS) from a CSV filename like upload-summary-143022-abcd1234.csv. - * @param {string} filename - * @returns {string} - */ -function extractTimeFromFilename(filename) { - const match = filename.match(/upload-summary-(\d{2})(\d{2})(\d{2})/); - if (match) return `${match[1]}:${match[2]}:${match[3]}`; - return '-'; -} - -/** Fetch CSV file list and render the Upload Summaries table. */ -export async function loadCsvFiles() { - try { - const data = await apiGet('/api/logs/files'); - /** @type {Array<{date: string|null, filename: string, relative_path: string, size_bytes: number, type: string}>} */ - const csvFiles = (data.files || []).filter( - (/** @type {{type: string}} */ f) => f.type === 'csv', - ); - - // Update badge - setText('csv-count-badge', csvFiles.length); - toggleEl('csv-count-badge', csvFiles.length > 0); - - const tbody = document.getElementById('csv-table-body'); - if (!tbody) return; - - if (csvFiles.length === 0) { - tbody.innerHTML = - 'No upload summaries found'; - return; - } - - let html = ''; - for (const file of csvFiles) { - const date = file.date || '-'; - const time = extractTimeFromFilename(file.filename); - const escapedPath = file.relative_path.replace(/"/g, '"'); - - html += ` - ${date} - ${time} - ${file.filename} - - - - - `; - } - tbody.innerHTML = html; - - // Wire download buttons - for (const btn of tbody.querySelectorAll('.csv-download-btn')) { - btn.addEventListener('click', () => { - const path = /** @type {HTMLElement} */ (btn).dataset.path || ''; - downloadCsv(path); - }); - } - - // Wire view buttons - for (const btn of tbody.querySelectorAll('.csv-view-btn')) { - btn.addEventListener('click', () => { - const path = /** @type {HTMLElement} */ (btn).dataset.path || ''; - previewCsv(path); - }); - } - } catch { - // Non-critical — silently fail - } -} - -/** - * Trigger a browser download for a CSV file. - * @param {string} path - Relative path within the log directory - */ -function downloadCsv(path) { - window.location.href = `/api/logs/csv-download?path=${encodeURIComponent(path)}`; -} - -/** - * Fetch and render a CSV preview inline. Toggle on re-click. - * @param {string} path - Relative path within the log directory - */ -async function previewCsv(path) { - const panel = document.getElementById('csv-preview-panel'); - const content = document.getElementById('csv-preview-content'); - if (!panel || !content) return; - - // Toggle off if clicking the same file - if (currentPreviewPath === path && !panel.classList.contains('hidden')) { - hideEl('csv-preview-panel'); - currentPreviewPath = ''; - return; - } - - currentPreviewPath = path; - content.innerHTML = '

Loading...

'; - panel.classList.remove('hidden'); - - const filename = path.split('/').pop() || path; - setText('csv-preview-title', `Preview: ${filename}`); - - try { - const data = await apiGet(`/api/logs/csv-preview?path=${encodeURIComponent(path)}`); - - if (data.error) { - content.innerHTML = `

${data.error}

`; - return; - } - - const columns = /** @type {string[]} */ (data.columns || []); - const rows = /** @type {Array>} */ (data.rows || []); - - if (columns.length === 0) { - content.innerHTML = '

Empty CSV

'; - return; - } - - let tableHtml = ''; - tableHtml += ''; - for (const col of columns) { - tableHtml += ``; - } - tableHtml += ''; - for (const row of rows) { - tableHtml += ''; - for (const col of columns) { - const val = row[col] ?? ''; - tableHtml += ``; - } - tableHtml += ''; - } - tableHtml += '
${col}
${val}
'; - content.innerHTML = tableHtml; - } catch { - content.innerHTML = '

Failed to load CSV preview

'; - } -} - -/** Reset all filters and reload. */ -export function clearFilters() { - state.logFilters = { date: null, level: null, category: null, search: '' }; - state.logPagination = { offset: 0, limit: 100 }; - - const { date, level, category, search } = getFilterEls(); - if (date) date.value = ''; - if (level) level.value = ''; - if (category) category.value = ''; - if (search) search.value = ''; - - loadLogEntries(); - loadLogStats(); -} - -/** Read current filter values from the DOM into state and reload. */ -function applyFilters() { - const { date, level, category, search } = getFilterEls(); - - state.logFilters.date = date?.value || null; - state.logFilters.level = level?.value || null; - state.logFilters.category = category?.value || null; - state.logFilters.search = search?.value || ''; - state.logPagination.offset = 0; - - loadLogEntries(); -} - -/** Initialize the logs page: load data and wire event handlers. */ -export function initLogs() { - // Reset state - state.logFilters = { date: null, level: null, category: null, search: '' }; - state.logPagination = { offset: 0, limit: 100 }; - - // Initial load - loadLogEntries(); - loadLogStats(); - loadCsvFiles(); - - // CSV section toggle - const csvToggle = document.getElementById('csv-section-toggle'); - const csvBody = document.getElementById('csv-section-body'); - const csvIcon = document.getElementById('csv-toggle-icon'); - if (csvToggle && csvBody) { - csvToggle.addEventListener('click', () => { - csvBody.classList.toggle('hidden'); - csvIcon?.classList.toggle('rotate-90'); - }); - } - - // CSV preview close button - document.getElementById('csv-preview-close')?.addEventListener('click', () => { - hideEl('csv-preview-panel'); - currentPreviewPath = ''; - }); - - // Filter controls - const { date: dateEl, level: levelEl, category: catEl, search: searchEl } = getFilterEls(); - const clearBtn = document.getElementById('clear-filters-btn'); - const syncBtn = document.getElementById('sync-logs-btn'); - const prevBtn = document.getElementById('prev-page-btn'); - const nextBtn = document.getElementById('next-page-btn'); - - if (dateEl) dateEl.addEventListener('change', applyFilters); - if (levelEl) levelEl.addEventListener('change', applyFilters); - if (catEl) catEl.addEventListener('change', applyFilters); - - // Debounced search - if (searchEl) { - searchEl.addEventListener('input', debounce(applyFilters, 300)); - } - - if (clearBtn) clearBtn.addEventListener('click', clearFilters); - if (syncBtn) syncBtn.addEventListener('click', syncLogs); - - if (prevBtn) { - prevBtn.addEventListener('click', () => { - state.logPagination.offset = Math.max( - 0, - state.logPagination.offset - state.logPagination.limit, - ); - loadLogEntries(); - }); - } - - if (nextBtn) { - nextBtn.addEventListener('click', () => { - state.logPagination.offset += state.logPagination.limit; - loadLogEntries(); - }); - } -} diff --git a/app/static/js/modules/notify.js b/app/static/js/modules/notify.js deleted file mode 100644 index e121a5b..0000000 --- a/app/static/js/modules/notify.js +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Toast notification system. - */ - -/** - * @param {string} message - * @param {'info' | 'error' | 'success'} [type] - */ -export function showNotification(message, type = 'info') { - const notification = document.createElement('div'); - notification.className = `fixed top-4 right-4 px-6 py-3 rounded-md shadow-lg z-50 transition-opacity duration-300 ${ - type === 'error' - ? 'bg-red-500 text-white' - : type === 'success' - ? 'bg-green-500 text-white' - : 'bg-nlr-blue text-white' - }`; - notification.textContent = message; - document.body.appendChild(notification); - - setTimeout(() => { - notification.classList.add('opacity-0'); - setTimeout(() => notification.remove(), 300); - }, 5000); -} diff --git a/app/static/js/modules/settings.js b/app/static/js/modules/settings.js deleted file mode 100644 index 06c3ea5..0000000 --- a/app/static/js/modules/settings.js +++ /dev/null @@ -1,402 +0,0 @@ -import { apiGet, apiPost, apiPut } from './api.js'; -import { appendText, setText, showEl, withLoadingButton } from './dom.js'; -import { showNotification } from './notify.js'; -/** - * Settings page functionality. - */ -import state from './state.js'; - -const AWS_REGION_OPTIONS = new Set([ - 'us-west-2', - 'us-west-1', - 'us-east-1', - 'us-east-2', - 'us-gov-west-1', - 'us-gov-east-1', -]); - -function getAwsRegion() { - const select = /** @type {HTMLSelectElement | null} */ ( - document.getElementById('aws-region-select') - ); - if (select?.value === 'other') { - const custom = /** @type {HTMLInputElement | null} */ ( - document.getElementById('aws-region-custom') - ); - return custom?.value.trim() || ''; - } - return select?.value || 'us-west-2'; -} - -/** - * @param {string} region - */ -function setAwsRegion(region) { - const select = /** @type {HTMLSelectElement | null} */ ( - document.getElementById('aws-region-select') - ); - const customInput = document.getElementById('aws-region-custom'); - const helpText = document.getElementById('aws-region-help'); - - if (!select || !customInput || !helpText) return; - - if (AWS_REGION_OPTIONS.has(region)) { - select.value = region; - customInput.classList.add('hidden'); - helpText.classList.add('hidden'); - /** @type {HTMLInputElement} */ (customInput).value = ''; - } else { - select.value = 'other'; - /** @type {HTMLInputElement} */ (customInput).value = region; - customInput.classList.remove('hidden'); - helpText.classList.remove('hidden'); - } -} - -export function initSettings() { - const form = document.getElementById('settings-form'); - if (!form) return; - - document.getElementById('aws-region-select')?.addEventListener('change', (e) => { - const customInput = document.getElementById('aws-region-custom'); - const helpText = document.getElementById('aws-region-help'); - if (/** @type {HTMLSelectElement} */ (e.target).value === 'other') { - customInput?.classList.remove('hidden'); - helpText?.classList.remove('hidden'); - /** @type {HTMLInputElement} */ (customInput)?.focus(); - } else { - customInput?.classList.add('hidden'); - helpText?.classList.add('hidden'); - if (customInput) /** @type {HTMLInputElement} */ (customInput).value = ''; - } - }); - - loadCurrentSettings(); - loadAwsProfiles(); - loadVersionInfo(); - loadCacheStats(); - - form.addEventListener('submit', async (e) => { - e.preventDefault(); - await saveSettings(); - }); - - document.getElementById('test-connection-btn')?.addEventListener('click', testConnection); - document.getElementById('check-updates-btn')?.addEventListener('click', checkForUpdates); - document.getElementById('run-update-btn')?.addEventListener('click', runUpdate); - document.getElementById('reset-settings-btn')?.addEventListener('click', resetSettings); - document.getElementById('clear-cache-btn')?.addEventListener('click', clearBrowserCache); - document.getElementById('sync-cache-btn')?.addEventListener('click', syncCacheWithAws); - document.getElementById('invalidate-cache-btn')?.addEventListener('click', invalidateUploadCache); -} - -async function loadCurrentSettings() { - try { - const settings = await apiGet('/api/settings'); - - setAwsRegion(settings.aws_region || 'us-west-2'); - - const s3Bucket = /** @type {HTMLInputElement | null} */ (document.getElementById('s3-bucket')); - if (s3Bucket) s3Bucket.value = settings.s3_bucket || ''; - - const defaultFolder = /** @type {HTMLInputElement | null} */ ( - document.getElementById('default-folder') - ); - if (defaultFolder) defaultFolder.value = settings.default_upload_folder || ''; - - state.currentAwsProfile = settings.aws_profile; - } catch (_error) { - showNotification('Failed to load settings', 'error'); - } -} - -async function loadAwsProfiles() { - try { - const data = await apiGet('/api/settings/profiles'); - - const select = /** @type {HTMLSelectElement | null} */ (document.getElementById('aws-profile')); - if (!select) return; - - select.innerHTML = data.profiles - .map((/** @type {string} */ profile) => ``) - .join(''); - - if (state.currentAwsProfile) { - select.value = state.currentAwsProfile; - } - } catch (_error) { - showNotification('Failed to load AWS profiles', 'error'); - } -} - -async function loadVersionInfo() { - try { - const data = await apiGet('/api/settings/version'); - - setText('git-branch', data.branch || '-'); - setText('git-commit', data.commit || '-'); - setText('git-date', data.last_updated || '-'); - setText('pkg-version', data.version || '-'); - - let versionText = data.version || '0.0.0'; - if (data.commit) { - versionText += ` (${data.commit})`; - } - setText('version-info', versionText); - } catch (error) { - console.error('Failed to load version info:', error); - } -} - -async function saveSettings() { - const settings = { - aws_profile: /** @type {HTMLSelectElement} */ (document.getElementById('aws-profile'))?.value, - aws_region: getAwsRegion(), - s3_bucket: /** @type {HTMLInputElement} */ (document.getElementById('s3-bucket'))?.value, - default_upload_folder: /** @type {HTMLInputElement} */ ( - document.getElementById('default-folder') - )?.value, - }; - - try { - await apiPut('/api/settings', settings); - showNotification('Settings saved successfully', 'success'); - } catch (error) { - showNotification(/** @type {Error} */ (error).message, 'error'); - } -} - -async function testConnection() { - const btn = /** @type {HTMLButtonElement | null} */ ( - document.getElementById('test-connection-btn') - ); - const status = document.getElementById('connection-status'); - if (!btn || !status) return; - - status.textContent = 'Testing connection...'; - status.className = 'mt-1 text-sm text-gray-500'; - - const settings = { - aws_profile: /** @type {HTMLSelectElement} */ (document.getElementById('aws-profile'))?.value, - aws_region: getAwsRegion(), - s3_bucket: /** @type {HTMLInputElement} */ (document.getElementById('s3-bucket'))?.value, - }; - - await withLoadingButton(btn, 'Testing...', async () => { - try { - const data = await apiPost('/api/settings/validate', settings); - status.textContent = data.success ? data.message : data.error; - status.className = `mt-1 text-sm ${data.success ? 'text-green-600' : 'text-red-600'}`; - } catch (error) { - status.textContent = /** @type {Error} */ (error).message; - status.className = 'mt-1 text-sm text-red-600'; - } - }); -} - -async function checkForUpdates() { - const btn = /** @type {HTMLButtonElement | null} */ ( - document.getElementById('check-updates-btn') - ); - const status = document.getElementById('update-status'); - const updateBtn = /** @type {HTMLButtonElement | null} */ ( - document.getElementById('run-update-btn') - ); - if (!btn || !status) return; - - await withLoadingButton(btn, 'Checking...', async () => { - try { - const data = await apiGet('/api/settings/check-updates'); - - if (data.error) { - status.textContent = `Error: ${data.error}`; - status.className = 'text-sm text-red-600'; - } else if (data.updates_available) { - status.textContent = 'Updates available! Click "Update Application" to install.'; - status.className = 'text-sm text-nlr-yellow font-medium'; - if (updateBtn) updateBtn.disabled = false; - } else if (data.up_to_date) { - status.textContent = 'Application is up to date.'; - status.className = 'text-sm text-green-600'; - } else { - status.textContent = 'Could not determine update status.'; - status.className = 'text-sm text-gray-600'; - } - } catch (error) { - status.textContent = /** @type {Error} */ (error).message; - status.className = 'text-sm text-red-600'; - } - }); -} - -async function runUpdate() { - const btn = /** @type {HTMLButtonElement | null} */ (document.getElementById('run-update-btn')); - const status = document.getElementById('update-status'); - if (!btn || !status) return; - - status.textContent = 'Running update...'; - showEl('update-log'); - setText('update-output', 'Starting update...\n'); - - await withLoadingButton(btn, 'Updating...', async () => { - try { - const data = await apiPost('/api/settings/update'); - - let output = ''; - - if (data.results.git_pull) { - output += '=== Git Pull ===\n'; - output += data.results.git_pull.success ? '[SUCCESS]\n' : '[FAILED]\n'; - output += `${data.results.git_pull.output}\n\n`; - } - - if (data.results.pip_install) { - output += '=== Pip Install ===\n'; - output += data.results.pip_install.success ? '[SUCCESS]\n' : '[FAILED]\n'; - output += `${data.results.pip_install.output}\n\n`; - } - - if (data.results.modaq_toolkit) { - output += '=== MODAQ Toolkit Update ===\n'; - output += data.results.modaq_toolkit.success ? '[SUCCESS]\n' : '[FAILED]\n'; - output += `${data.results.modaq_toolkit.output}\n`; - } - - setText('update-output', output); - - if (data.success) { - status.textContent = 'Update completed! Restart the application to apply changes.'; - status.className = 'text-sm text-green-600 font-medium'; - showNotification('Update completed! Please restart the application.', 'success'); - } else { - status.textContent = 'Update completed with some errors. Check the log below.'; - status.className = 'text-sm text-yellow-600'; - } - - loadVersionInfo(); - } catch (error) { - status.textContent = /** @type {Error} */ (error).message; - status.className = 'text-sm text-red-600'; - appendText('update-output', `\nError: ${/** @type {Error} */ (error).message}`); - } - }); -} - -async function resetSettings() { - if (!confirm('Are you sure you want to reset all settings to defaults?')) { - return; - } - - try { - await apiPut('/api/settings', { - aws_profile: 'default', - aws_region: 'us-west-2', - s3_bucket: '', - default_upload_folder: '', - }); - - loadCurrentSettings(); - showNotification('Settings reset to defaults', 'success'); - } catch (error) { - showNotification(/** @type {Error} */ (error).message, 'error'); - } -} - -function clearBrowserCache() { - if ( - !confirm( - 'Are you sure you want to clear the browser cache? This will forget your last used folder and other local preferences.', - ) - ) { - return; - } - - try { - localStorage.removeItem('lastUploadFolder'); - showNotification('Browser cache cleared', 'success'); - } catch (error) { - showNotification(`Failed to clear cache: ${/** @type {Error} */ (error).message}`, 'error'); - } -} - -async function loadCacheStats() { - try { - const data = await apiGet('/api/settings/cache/stats'); - - if (data.success && data.stats) { - const stats = data.stats; - - setText('cache-total', stats.total_entries || 0); - setText('cache-exists', stats.exists_count || 0); - setText('cache-deleted', stats.not_exists_count || 0); - - if (stats.last_full_sync) { - setText('cache-last-sync', new Date(stats.last_full_sync).toLocaleString()); - } else { - setText('cache-last-sync', 'Never'); - } - } - } catch (error) { - console.error('Failed to load cache stats:', error); - } -} - -async function syncCacheWithAws() { - const btn = /** @type {HTMLButtonElement | null} */ (document.getElementById('sync-cache-btn')); - const status = document.getElementById('sync-status'); - if (!btn || !status) return; - - status.classList.remove('hidden'); - status.textContent = 'Fetching file list from S3...'; - status.className = 'mt-2 text-sm text-gray-600'; - - await withLoadingButton(btn, 'Syncing...', async () => { - try { - const data = await apiPost('/api/settings/cache/sync'); - - if (data.success) { - status.textContent = data.message; - status.className = 'mt-2 text-sm text-green-600'; - showNotification(data.message, 'success'); - loadCacheStats(); - } else { - status.textContent = `Error: ${data.error}`; - status.className = 'mt-2 text-sm text-red-600'; - showNotification(data.error, 'error'); - } - } catch (error) { - status.textContent = `Error: ${/** @type {Error} */ (error).message}`; - status.className = 'mt-2 text-sm text-red-600'; - showNotification(/** @type {Error} */ (error).message, 'error'); - } - }); -} - -async function invalidateUploadCache() { - if ( - !confirm( - 'Are you sure you want to clear the upload cache? This will delete all cached file records for the current bucket. The next upload will need to re-check all files against S3.', - ) - ) { - return; - } - - const btn = /** @type {HTMLButtonElement | null} */ ( - document.getElementById('invalidate-cache-btn') - ); - - await withLoadingButton(btn, 'Clearing...', async () => { - try { - const data = await apiPost('/api/settings/cache/invalidate'); - - if (data.success) { - showNotification(data.message, 'success'); - loadCacheStats(); - } else { - showNotification(data.error || 'Failed to clear cache', 'error'); - } - } catch (error) { - showNotification(/** @type {Error} */ (error).message, 'error'); - } - }); -} diff --git a/app/static/js/modules/sorting-helpers.js b/app/static/js/modules/sorting-helpers.js deleted file mode 100644 index 4ce1d70..0000000 --- a/app/static/js/modules/sorting-helpers.js +++ /dev/null @@ -1,40 +0,0 @@ -/** - * Table sorting helpers: toggle column direction and update header indicators. - */ - -/** - * Toggle sort direction on a column config object. - * @param {{ column: string, ascending: boolean }} config - * @param {string} column - */ -export function toggleSort(config, column) { - if (config.column === column) { - config.ascending = !config.ascending; - } else { - config.column = column; - config.ascending = true; - } -} - -/** - * Update sort indicator arrows in table headers. - * @param {string} headerSelector - e.g. `'[data-sort]'` - * @param {string} indicatorSelector - e.g. `'.sort-indicator'` - * @param {string} dataKey - dataset key, e.g. `'sort'` or `'fileSort'` - * @param {string} currentColumn - * @param {boolean} ascending - */ -export function updateSortIndicators( - headerSelector, - indicatorSelector, - dataKey, - currentColumn, - ascending, -) { - for (const th of document.querySelectorAll(headerSelector)) { - const indicator = th.querySelector(indicatorSelector); - if (!indicator) continue; - const col = /** @type {HTMLElement} */ (th).dataset[dataKey]; - indicator.textContent = col === currentColumn ? (ascending ? ' \u25B2' : ' \u25BC') : ''; - } -} diff --git a/app/static/js/modules/state.js b/app/static/js/modules/state.js deleted file mode 100644 index 1febefd..0000000 --- a/app/static/js/modules/state.js +++ /dev/null @@ -1,75 +0,0 @@ -/** - * Centralized mutable state for the modaq_upload. - * Replaces all global `let` variables and `window.*` properties. - */ -const state = { - /** @type {string | null} */ - currentJobId: null, - - /** @type {EventSource | null} */ - eventSource: null, - - /** @type {string | null} */ - selectedFolderPath: null, - - /** Current upload step (1-4) */ - currentStep: 1, - - /** S3 file browser current prefix */ - currentPrefix: '', - - /** @type {{ version?: string, commit?: string, branch?: string } | null} */ - appVersionData: null, - - /** @type {string | undefined} */ - currentAwsProfile: undefined, - - /** @type {string[]} */ - scanFilePaths: [], - - /** @type {Array<{ path: string, filename: string, size: number, mtime?: number, already_uploaded: boolean }>} */ - scanFileStatuses: [], - - /** Total size of all scanned files in bytes */ - scanTotalSize: 0, - - /** @type {string | null} */ - scanFolderPath: null, - - /** @type {string | null} */ - scanJobId: null, - - /** Sort configuration for the review table */ - reviewSortConfig: { column: 'filename', ascending: true }, - - /** @type {Array<{ name: string, size: number, mtime: number }>} */ - browserFiles: [], - - /** Sort configuration for the folder browser file table */ - browserFileSortConfig: { column: 'name', ascending: true }, - - /** @type {Array<{ relative_path: string, files: any[], total_files: number, already_uploaded: number, all_uploaded: boolean, error: string | null }>} */ - browserScanResults: [], - - /** @type {EventSource | null} Browser scan SSE connection (separate from main eventSource) */ - browserScanEventSource: null, - - /** @type {string | null} Browser scan job ID */ - browserScanJobId: null, - - /** Whether the browser scan has completed */ - browserScanComplete: false, - - /** Log viewer filter state */ - logFilters: { - /** @type {string | null} */ date: null, - /** @type {string | null} */ level: null, - /** @type {string | null} */ category: null, - /** @type {string} */ search: '', - }, - - /** Log viewer pagination state */ - logPagination: { offset: 0, limit: 100 }, -}; - -export default state; diff --git a/app/static/js/modules/stepper.js b/app/static/js/modules/stepper.js deleted file mode 100644 index bce8cdd..0000000 --- a/app/static/js/modules/stepper.js +++ /dev/null @@ -1,83 +0,0 @@ -/** - * Upload step indicator management. - */ -import { setText } from './dom.js'; -import { showNotification } from './notify.js'; -import state from './state.js'; - -export const UPLOAD_STEPS = { - 1: { name: 'Select', description: 'Select files or a folder to upload' }, - 2: { - name: 'Review', - description: 'Review files found - click Continue to upload, or Back to select different files', - }, - 3: { name: 'Upload', description: 'Validating and uploading files...' }, - 4: { name: 'Complete', description: 'Upload complete!' }, -}; - -/** - * Set the current step in the upload flow. - * @param {number} step - */ -export function setUploadStep(step) { - state.currentStep = step; - const stepsContainer = document.getElementById('upload-steps'); - if (!stepsContainer) return; - - for (let i = 1; i <= 4; i++) { - const stepEl = stepsContainer.querySelector(`[data-step="${i}"]`); - if (!stepEl) continue; - - stepEl.classList.remove('completed', 'active'); - - if (i < step) { - stepEl.classList.add('completed'); - } else if (i === step) { - stepEl.classList.add('active'); - } - } - - const connectors = stepsContainer.querySelectorAll('.step-connector'); - connectors.forEach((connector, index) => { - /** @type {HTMLElement} */ (connector).style.backgroundColor = - index < step - 1 ? '#5D9732' : '#D1D5DB'; - }); - - if (UPLOAD_STEPS[step]) { - setText('step-description', UPLOAD_STEPS[step].description); - } -} - -/** - * Show the upload steps indicator and set the current step. - * @param {number} step - */ -export function showUploadSteps(step) { - setUploadStep(step); -} - -/** - * Reset the upload steps indicator to step 1. - */ -export function hideUploadSteps() { - setUploadStep(1); -} - -/** - * Navigate to a specific step (for going back). - * Uses dynamic import to avoid circular dependency with upload-control. - * @param {number} targetStep - */ -export async function goToStep(targetStep) { - if (targetStep >= state.currentStep) return; - - if (targetStep === 1 && state.currentStep <= 2) { - const { resetUpload } = await import('./upload-control.js'); - resetUpload(); - return; - } - - if (state.currentStep >= 3) { - showNotification('Cannot go back during or after upload', 'error'); - } -} diff --git a/app/static/js/modules/upload-control.js b/app/static/js/modules/upload-control.js deleted file mode 100644 index 40d1b80..0000000 --- a/app/static/js/modules/upload-control.js +++ /dev/null @@ -1,54 +0,0 @@ -import { resetAnalysisState } from './analysis.js'; -/** - * Cancel and reset operations for uploads. - */ -import { apiPost } from './api.js'; -import { hideEl, showEl } from './dom.js'; -import { showNotification } from './notify.js'; -import state from './state.js'; -import { hideUploadSteps } from './stepper.js'; - -export async function cancelUpload() { - // Cancel scan job if active - if (state.scanJobId) { - try { - await apiPost(`/api/upload/cancel/${state.scanJobId}`); - } catch (_error) { - // Scan may have already finished - } - } - - if (!state.currentJobId) return; - - try { - await apiPost(`/api/upload/cancel/${state.currentJobId}`); - if (state.eventSource) state.eventSource.close(); - showNotification('Upload cancelled', 'info'); - } catch (_error) { - showNotification('Failed to cancel upload', 'error'); - } -} - -export function resetUpload() { - state.currentJobId = null; - if (state.eventSource) { - state.eventSource.close(); - state.eventSource = null; - } - - resetAnalysisState(); - hideUploadSteps(); - - showEl('folder-browser-panel'); - hideEl('upload-section'); - hideEl('completion-section'); - hideEl('scan-results-section'); - hideEl('confirm-upload-modal'); - - state.selectedFolderPath = null; - state.scanFilePaths = []; - state.scanFileStatuses = []; - state.scanTotalSize = 0; - state.scanFolderPath = null; - state.scanJobId = null; -} diff --git a/app/static/js/modules/upload-exec.js b/app/static/js/modules/upload-exec.js deleted file mode 100644 index 85736f8..0000000 --- a/app/static/js/modules/upload-exec.js +++ /dev/null @@ -1,226 +0,0 @@ -/** - * Upload execution: progress tracking and completion summary. - * Completion table is paginated (100 rows per page) for 20K+ file performance. - */ -import { hideEl, setText, showEl } from './dom.js'; -import { formatDuration, formatEta } from './formatters.js'; -import { fileIcon } from './icons.js'; -import { showNotification } from './notify.js'; -import { setUploadStep } from './stepper.js'; - -const COMPLETION_PAGE_SIZE = 100; - -/** @type {any} */ -let lastCompletedJob = null; - -/** Current page (0-indexed) of the completion table. */ -let completionPage = 0; - -/** - * @param {any} job - */ -export function updateProgressUI(job) { - setText('progress-percent', job.progress_percent.toFixed(1)); - setText('files-completed', job.files_completed); - setText('files-total', job.total_files); - setText('bytes-uploaded', job.uploaded_bytes_formatted); - setText('bytes-total', job.total_bytes_formatted); - setText('eta', formatEta(job.eta_seconds)); - - const progressBar = /** @type {HTMLElement | null} */ (document.getElementById('progress-bar')); - if (progressBar) progressBar.style.width = `${job.progress_percent}%`; -} - -/** - * @param {any} job - */ -export function showCompletionSummary(job) { - lastCompletedJob = job; - completionPage = 0; - setUploadStep(4); - - hideEl('upload-section'); - showEl('completion-section'); - - // Use pre-computed counts from backend (avoids 3 filter passes over 20K files) - const completed = - job.files_uploaded ?? - job.files.filter((/** @type {any} */ f) => f.status === 'completed').length; - const skipped = - job.files_skipped ?? job.files.filter((/** @type {any} */ f) => f.status === 'skipped').length; - const failed = - job.files_failed ?? job.files.filter((/** @type {any} */ f) => f.status === 'failed').length; - - setText('completed-count', completed); - setText('skipped-count', skipped); - setText('failed-count', failed); - setText('total-uploaded-size', job.successfully_uploaded_bytes_formatted || '-'); - setText('total-upload-time', job.total_upload_duration_formatted || '-'); - setText( - 'avg-upload-speed', - job.average_upload_speed_mbps ? `${job.average_upload_speed_mbps} Mbps` : '-', - ); - - // Single pass for average per-file duration (not pre-computed by backend) - let durationSum = 0; - let durationCount = 0; - for (const f of job.files) { - if (f.status === 'completed' && f.upload_duration_seconds) { - durationSum += f.upload_duration_seconds; - durationCount++; - } - } - setText('avg-file-time', durationCount > 0 ? formatDuration(durationSum / durationCount) : '-'); - - // Render first page and wire up pagination - renderCompletionPage(); - wireCompletionPagination(); - - if (failed === 0) { - showNotification('Upload completed successfully!', 'success'); - } else { - showNotification(`Upload completed with ${failed} failed files`, 'error'); - } -} - -/** - * Render the current page of the completion file table. - */ -function renderCompletionPage() { - if (!lastCompletedJob) return; - - const files = lastCompletedJob.files; - const totalPages = Math.max(1, Math.ceil(files.length / COMPLETION_PAGE_SIZE)); - const start = completionPage * COMPLETION_PAGE_SIZE; - const end = Math.min(start + COMPLETION_PAGE_SIZE, files.length); - const pageFiles = files.slice(start, end); - - const tbody = document.getElementById('completion-file-list'); - if (tbody) { - tbody.innerHTML = pageFiles - .map((/** @type {any} */ file) => { - let statusBadge; - let statusClass; - if (file.status === 'completed') { - statusBadge = 'Uploaded'; - statusClass = 'bg-green-100 text-green-800'; - } else if (file.status === 'skipped') { - statusBadge = 'Skipped'; - statusClass = 'bg-yellow-100 text-yellow-800'; - } else if (file.status === 'failed') { - statusBadge = 'Failed'; - statusClass = 'bg-red-100 text-red-800'; - } else { - statusBadge = file.status; - statusClass = 'bg-gray-100 text-gray-800'; - } - - const duration = file.upload_duration_seconds - ? formatDuration(file.upload_duration_seconds) - : '-'; - const speed = file.upload_speed_mbps ? `${file.upload_speed_mbps} Mbps` : '-'; - - return ` - - -
- ${fileIcon('h-4 w-4 text-gray-400 mr-2')} - ${file.filename} -
- - ${file.s3_path || '-'} - ${file.file_size_formatted} - ${duration} - ${speed} - - ${statusBadge} - - - `; - }) - .join(''); - } - - // Update pagination controls - setText('completion-page-info', `Page ${completionPage + 1} of ${totalPages}`); - - const prevBtn = /** @type {HTMLButtonElement | null} */ ( - document.getElementById('completion-prev-btn') - ); - const nextBtn = /** @type {HTMLButtonElement | null} */ ( - document.getElementById('completion-next-btn') - ); - if (prevBtn) prevBtn.disabled = completionPage === 0; - if (nextBtn) nextBtn.disabled = completionPage >= totalPages - 1; -} - -/** - * Wire up pagination button click handlers. - */ -function wireCompletionPagination() { - const prevBtn = document.getElementById('completion-prev-btn'); - const nextBtn = document.getElementById('completion-next-btn'); - - if (prevBtn) { - prevBtn.onclick = () => { - if (completionPage > 0) { - completionPage--; - renderCompletionPage(); - } - }; - } - - if (nextBtn) { - nextBtn.onclick = () => { - if (!lastCompletedJob) return; - const totalPages = Math.ceil(lastCompletedJob.files.length / COMPLETION_PAGE_SIZE); - if (completionPage < totalPages - 1) { - completionPage++; - renderCompletionPage(); - } - }; - } -} - -/** - * Download the upload summary as a CSV file. - */ -export function downloadSummaryCSV() { - if (!lastCompletedJob || !lastCompletedJob.files) { - showNotification('No upload data available', 'error'); - return; - } - - const headers = ['Filename', 'Size', 'S3 Path', 'Status', 'Duration', 'Speed']; - const rows = lastCompletedJob.files.map((/** @type {any} */ file) => { - const duration = file.upload_duration_seconds - ? formatDuration(file.upload_duration_seconds) - : ''; - const speed = file.upload_speed_mbps ? `${file.upload_speed_mbps} Mbps` : ''; - return [ - file.filename, - file.file_size_formatted || '', - file.s3_path || '', - file.status, - duration, - speed, - ]; - }); - - const csvContent = [ - headers.join(','), - ...rows.map((/** @type {string[]} */ row) => - row.map((/** @type {string} */ cell) => `"${String(cell).replace(/"/g, '""')}"`).join(','), - ), - ].join('\n'); - - const blob = new Blob([csvContent], { type: 'text/csv;charset=utf-8;' }); - const url = URL.createObjectURL(blob); - const link = document.createElement('a'); - link.href = url; - const now = new Date(); - const pad = (/** @type {number} */ n) => String(n).padStart(2, '0'); - link.download = `upload-summary-${now.toISOString().slice(0, 10)}-${pad(now.getHours())}${pad(now.getMinutes())}.csv`; - link.click(); - URL.revokeObjectURL(url); -} diff --git a/app/static/js/modules/upload-init.js b/app/static/js/modules/upload-init.js deleted file mode 100644 index 28855ba..0000000 --- a/app/static/js/modules/upload-init.js +++ /dev/null @@ -1,53 +0,0 @@ -import { checkForActiveJob } from './analysis.js'; -import { apiPost } from './api.js'; -import { hideEl, showEl } from './dom.js'; -import { initFolderBrowser, showConfirmModal, startCombinedUpload } from './folder-browser.js'; -/** - * Upload page initialization and event wiring. - */ -import state from './state.js'; -import { setUploadStep } from './stepper.js'; -import { cancelUpload, resetUpload } from './upload-control.js'; -import { downloadSummaryCSV } from './upload-exec.js'; - -export function initUpload() { - const panel = document.getElementById('folder-browser-panel'); - if (!panel) return; - - initFolderBrowser(); - - setUploadStep(1); - checkForActiveJob(); - - document.getElementById('cancel-upload-btn')?.addEventListener('click', cancelUpload); - document.getElementById('upload-more-btn')?.addEventListener('click', resetUpload); - document.getElementById('download-csv-btn')?.addEventListener('click', downloadSummaryCSV); - - document.getElementById('continue-upload-btn')?.addEventListener('click', showConfirmModal); - document.getElementById('confirm-upload-btn')?.addEventListener('click', startCombinedUpload); - - document.getElementById('cancel-scan-btn')?.addEventListener('click', async () => { - // Cancel active scan job if running - if (state.scanJobId) { - try { - await apiPost(`/api/upload/cancel/${state.scanJobId}`); - } catch (_error) { - // Scan may have already finished - } - if (state.eventSource) { - state.eventSource.close(); - state.eventSource = null; - } - state.scanJobId = null; - } - - hideEl('scan-results-section'); - showEl('folder-browser-panel'); - state.selectedFolderPath = null; - state.scanFilePaths = []; - state.scanFileStatuses = []; - state.scanTotalSize = 0; - state.scanFolderPath = null; - setUploadStep(1); - }); -} From 5c4d805e934789684a371277ec0fb51f8d2fcf5c Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Tue, 17 Feb 2026 17:05:37 -0700 Subject: [PATCH 098/173] Refactor: Remove legacy js configuration --- biome.json | 39 --------------------------------------- jsconfig.json | 14 -------------- package.json | 21 --------------------- vitest.config.js | 13 ------------- 4 files changed, 87 deletions(-) delete mode 100644 biome.json delete mode 100644 jsconfig.json delete mode 100644 package.json delete mode 100644 vitest.config.js diff --git a/biome.json b/biome.json deleted file mode 100644 index 0e9c777..0000000 --- a/biome.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "$schema": "https://biomejs.dev/schemas/1.9.0/schema.json", - "organizeImports": { - "enabled": true - }, - "formatter": { - "enabled": true, - "indentStyle": "space", - "indentWidth": 2, - "lineWidth": 100 - }, - "javascript": { - "formatter": { - "quoteStyle": "single", - "semicolons": "always" - } - }, - "linter": { - "enabled": true, - "rules": { - "recommended": true, - "correctness": { - "noUnusedVariables": "warn", - "noUnusedImports": "warn" - }, - "suspicious": { - "noExplicitAny": "off" - }, - "style": { - "noParameterAssign": "off", - "useConst": "error" - } - } - }, - "files": { - "include": ["app/static/js/**/*.js"], - "ignore": ["node_modules/", "tests/"] - } -} diff --git a/jsconfig.json b/jsconfig.json deleted file mode 100644 index 187ef93..0000000 --- a/jsconfig.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "compilerOptions": { - "checkJs": true, - "noEmit": true, - "strict": false, - "target": "ES2022", - "module": "ES2022", - "moduleResolution": "bundler", - "lib": ["ES2022", "DOM", "DOM.Iterable"], - "skipLibCheck": true - }, - "include": ["app/static/js/**/*.js"], - "exclude": ["node_modules", "tests"] -} diff --git a/package.json b/package.json deleted file mode 100644 index 611fc07..0000000 --- a/package.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "private": true, - "name": "modaq-upload", - "description": "Frontend js for modaq_upload flask app", - "scripts": { - "lint": "biome check app/static/js/", - "lint:fix": "biome check --write app/static/js/", - "typecheck": "tsc -p jsconfig.json", - "test": "vitest run", - "test:watch": "vitest", - "test:coverage": "vitest run --coverage", - "check": "biome check app/static/js/ && tsc -p jsconfig.json && vitest run" - }, - "devDependencies": { - "@biomejs/biome": "^1.9.0", - "@vitest/coverage-v8": "^2.1.0", - "jsdom": "^25.0.0", - "typescript": "^5.6.0", - "vitest": "^2.1.0" - } -} diff --git a/vitest.config.js b/vitest.config.js deleted file mode 100644 index 53550f8..0000000 --- a/vitest.config.js +++ /dev/null @@ -1,13 +0,0 @@ -import { defineConfig } from 'vitest/config'; - -export default defineConfig({ - test: { - environment: 'jsdom', - include: ['tests/js/**/*.test.js'], - coverage: { - provider: 'v8', - include: ['app/static/js/**/*.js'], - reportsDirectory: 'htmlcov-js', - }, - }, -}); From e134ed31fb2f63fb615a464be078480deb7683b6 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Wed, 18 Feb 2026 09:27:10 -0700 Subject: [PATCH 099/173] Frontend: Add dev testing packages --- frontend/package-lock.json | 669 +++++++++++++++++++++++++++++++------ frontend/package.json | 3 +- 2 files changed, 569 insertions(+), 103 deletions(-) diff --git a/frontend/package-lock.json b/frontend/package-lock.json index c7aefc7..249a958 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -19,6 +19,7 @@ "devDependencies": { "@eslint/js": "^9.39.1", "@tailwindcss/vite": "^4.1.18", + "@testing-library/dom": "^10.4.0", "@testing-library/jest-dom": "^6.9.1", "@testing-library/react": "^16.3.2", "@testing-library/user-event": "^14.6.1", @@ -26,7 +27,7 @@ "@types/react": "^19.2.7", "@types/react-dom": "^19.2.3", "@vitejs/plugin-react": "^5.1.1", - "@vitest/coverage-v8": "^4.0.18", + "@vitest/coverage-v8": "^3.2.4", "eslint": "^9.39.1", "eslint-plugin-react-hooks": "^7.0.1", "eslint-plugin-react-refresh": "^0.4.24", @@ -46,6 +47,20 @@ "dev": true, "license": "MIT" }, + "node_modules/@ampproject/remapping": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.3.0.tgz", + "integrity": "sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + }, + "engines": { + "node": ">=6.0.0" + } + }, "node_modules/@asamuzakjp/css-color": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/@asamuzakjp/css-color/-/css-color-4.1.2.tgz", @@ -1186,6 +1201,34 @@ "url": "https://github.com/sponsors/nzakas" } }, + "node_modules/@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "dev": true, + "license": "ISC", + "dependencies": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@istanbuljs/schema": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.3.tgz", + "integrity": "sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/@jridgewell/gen-mapping": { "version": "0.3.13", "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", @@ -1236,6 +1279,17 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@pkgjs/parseargs": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "dev": true, + "license": "MIT", + "optional": true, + "engines": { + "node": ">=14" + } + }, "node_modules/@rolldown/pluginutils": { "version": "1.0.0-rc.3", "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-rc.3.tgz", @@ -1925,6 +1979,26 @@ "url": "https://github.com/sponsors/tannerlinsley" } }, + "node_modules/@testing-library/dom": { + "version": "10.4.1", + "resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-10.4.1.tgz", + "integrity": "sha512-o4PXJQidqJl82ckFaXUeoAW+XysPLauYI43Abki5hABd853iMhitooc6znOnczgbTYmEP6U6/y1ZyKAIsvMKGg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.10.4", + "@babel/runtime": "^7.12.5", + "@types/aria-query": "^5.0.1", + "aria-query": "5.3.0", + "dom-accessibility-api": "^0.5.9", + "lz-string": "^1.5.0", + "picocolors": "1.1.1", + "pretty-format": "^27.0.2" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/@testing-library/jest-dom": { "version": "6.9.1", "resolved": "https://registry.npmjs.org/@testing-library/jest-dom/-/jest-dom-6.9.1.tgz", @@ -1994,6 +2068,13 @@ "@testing-library/dom": ">=7.21.4" } }, + "node_modules/@types/aria-query": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/@types/aria-query/-/aria-query-5.0.4.tgz", + "integrity": "sha512-rfT93uj5s0PRL7EzccGMs3brplhcrghnDoV26NqKhCAS1hVo+WdNsPvE/yb6ilfr5hi2MEk6d5EWJTKdxg8jVw==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/babel__core": { "version": "7.20.5", "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", @@ -2085,7 +2166,7 @@ "version": "19.2.14", "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.14.tgz", "integrity": "sha512-ilcTH/UniCkMdtexkoCN0bI7pMcJDvmQFPvuPvmEaYA/NSfFTAgdUSLAoVjaRJm7+6PvcM+q1zYOwS4wTYMF9w==", - "dev": true, + "devOptional": true, "license": "MIT", "dependencies": { "csstype": "^3.2.2" @@ -2405,29 +2486,32 @@ } }, "node_modules/@vitest/coverage-v8": { - "version": "4.0.18", - "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-4.0.18.tgz", - "integrity": "sha512-7i+N2i0+ME+2JFZhfuz7Tg/FqKtilHjGyGvoHYQ6iLV0zahbsJ9sljC9OcFcPDbhYKCet+sG8SsVqlyGvPflZg==", + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-3.2.4.tgz", + "integrity": "sha512-EyF9SXU6kS5Ku/U82E259WSnvg6c8KTjppUncuNdm5QHpe17mwREHnjDzozC8x9MZ0xfBUFSaLkRv4TMA75ALQ==", "dev": true, "license": "MIT", "dependencies": { + "@ampproject/remapping": "^2.3.0", "@bcoe/v8-coverage": "^1.0.2", - "@vitest/utils": "4.0.18", - "ast-v8-to-istanbul": "^0.3.10", + "ast-v8-to-istanbul": "^0.3.3", + "debug": "^4.4.1", "istanbul-lib-coverage": "^3.2.2", "istanbul-lib-report": "^3.0.1", - "istanbul-reports": "^3.2.0", - "magicast": "^0.5.1", - "obug": "^2.1.1", - "std-env": "^3.10.0", - "tinyrainbow": "^3.0.3" + "istanbul-lib-source-maps": "^5.0.6", + "istanbul-reports": "^3.1.7", + "magic-string": "^0.30.17", + "magicast": "^0.3.5", + "std-env": "^3.9.0", + "test-exclude": "^7.0.1", + "tinyrainbow": "^2.0.0" }, "funding": { "url": "https://opencollective.com/vitest" }, "peerDependencies": { - "@vitest/browser": "4.0.18", - "vitest": "4.0.18" + "@vitest/browser": "3.2.4", + "vitest": "3.2.4" }, "peerDependenciesMeta": { "@vitest/browser": { @@ -2480,16 +2564,6 @@ "url": "https://opencollective.com/vitest" } }, - "node_modules/@vitest/expect/node_modules/tinyrainbow": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-2.0.0.tgz", - "integrity": "sha512-op4nsTR47R6p0vMUUoYl/a+ljLFVtlfaXkLQmqfLR1qHma1h/ysYk4hEXZ880bf2CYgTskvTa/e196Vd5dDQXw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=14.0.0" - } - }, "node_modules/@vitest/mocker": { "version": "3.2.4", "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-3.2.4.tgz", @@ -2517,19 +2591,6 @@ } } }, - "node_modules/@vitest/pretty-format": { - "version": "4.0.18", - "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.0.18.tgz", - "integrity": "sha512-P24GK3GulZWC5tz87ux0m8OADrQIUVDPIjjj65vBXYG17ZeU3qD7r+MNZ1RNv4l8CGU2vtTRqixrOi9fYk/yKw==", - "dev": true, - "license": "MIT", - "dependencies": { - "tinyrainbow": "^3.0.3" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, "node_modules/@vitest/runner": { "version": "3.2.4", "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-3.2.4.tgz", @@ -2573,16 +2634,6 @@ "url": "https://opencollective.com/vitest" } }, - "node_modules/@vitest/runner/node_modules/tinyrainbow": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-2.0.0.tgz", - "integrity": "sha512-op4nsTR47R6p0vMUUoYl/a+ljLFVtlfaXkLQmqfLR1qHma1h/ysYk4hEXZ880bf2CYgTskvTa/e196Vd5dDQXw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=14.0.0" - } - }, "node_modules/@vitest/snapshot": { "version": "3.2.4", "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-3.2.4.tgz", @@ -2611,16 +2662,6 @@ "url": "https://opencollective.com/vitest" } }, - "node_modules/@vitest/snapshot/node_modules/tinyrainbow": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-2.0.0.tgz", - "integrity": "sha512-op4nsTR47R6p0vMUUoYl/a+ljLFVtlfaXkLQmqfLR1qHma1h/ysYk4hEXZ880bf2CYgTskvTa/e196Vd5dDQXw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=14.0.0" - } - }, "node_modules/@vitest/spy": { "version": "3.2.4", "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-3.2.4.tgz", @@ -2634,20 +2675,6 @@ "url": "https://opencollective.com/vitest" } }, - "node_modules/@vitest/utils": { - "version": "4.0.18", - "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.0.18.tgz", - "integrity": "sha512-msMRKLMVLWygpK3u2Hybgi4MNjcYJvwTb0Ru09+fOyCXIgT5raYP041DRRdiJiI3k/2U6SEbAETB3YtBrUkCFA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/pretty-format": "4.0.18", - "tinyrainbow": "^3.0.3" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, "node_modules/acorn": { "version": "8.15.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", @@ -2698,6 +2725,16 @@ "url": "https://github.com/sponsors/epoberezkin" } }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/ansi-styles": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", @@ -3030,7 +3067,7 @@ "version": "3.2.3", "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", - "dev": true, + "devOptional": true, "license": "MIT" }, "node_modules/data-urls": { @@ -3119,6 +3156,20 @@ "node": ">=8" } }, + "node_modules/dom-accessibility-api": { + "version": "0.5.16", + "resolved": "https://registry.npmjs.org/dom-accessibility-api/-/dom-accessibility-api-0.5.16.tgz", + "integrity": "sha512-X7BJ2yElsnOJ30pZF4uIIDfBEVgF4XEBxL9Bxhy6dnrm5hkzqmsWHGTiHqRiITNhMyFLyAiWndIJP7Z1NTteDg==", + "dev": true, + "license": "MIT" + }, + "node_modules/eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", + "dev": true, + "license": "MIT" + }, "node_modules/electron-to-chromium": { "version": "1.5.286", "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.286.tgz", @@ -3126,6 +3177,13 @@ "dev": true, "license": "ISC" }, + "node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "dev": true, + "license": "MIT" + }, "node_modules/enhanced-resolve": { "version": "5.19.0", "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.19.0.tgz", @@ -3519,6 +3577,23 @@ "dev": true, "license": "ISC" }, + "node_modules/foreground-child": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz", + "integrity": "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==", + "dev": true, + "license": "ISC", + "dependencies": { + "cross-spawn": "^7.0.6", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/fsevents": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", @@ -3544,6 +3619,28 @@ "node": ">=6.9.0" } }, + "node_modules/glob": { + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.5.0.tgz", + "integrity": "sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==", + "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", + "dev": true, + "license": "ISC", + "dependencies": { + "foreground-child": "^3.1.0", + "jackspeak": "^3.1.2", + "minimatch": "^9.0.4", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^1.11.1" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/glob-parent": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", @@ -3557,6 +3654,32 @@ "node": ">=10.13.0" } }, + "node_modules/glob/node_modules/brace-expansion": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/glob/node_modules/minimatch": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", + "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/globals": { "version": "16.5.0", "resolved": "https://registry.npmjs.org/globals/-/globals-16.5.0.tgz", @@ -3722,6 +3845,16 @@ "node": ">=0.10.0" } }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/is-glob": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", @@ -3774,6 +3907,21 @@ "node": ">=10" } }, + "node_modules/istanbul-lib-source-maps": { + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/istanbul-lib-source-maps/-/istanbul-lib-source-maps-5.0.6.tgz", + "integrity": "sha512-yg2d+Em4KizZC5niWhQaIomgf5WlL4vOOjZ5xGCmF8SnPE/mDWWXgvRExdcpCgh9lLRRa1/fSYp2ymmbJ1pI+A==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.23", + "debug": "^4.1.1", + "istanbul-lib-coverage": "^3.0.0" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/istanbul-reports": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.2.0.tgz", @@ -3788,6 +3936,22 @@ "node": ">=8" } }, + "node_modules/jackspeak": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz", + "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + }, + "optionalDependencies": { + "@pkgjs/parseargs": "^0.11.0" + } + }, "node_modules/jiti": { "version": "2.6.1", "resolved": "https://registry.npmjs.org/jiti/-/jiti-2.6.1.tgz", @@ -4239,6 +4403,16 @@ "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, + "node_modules/lz-string": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/lz-string/-/lz-string-1.5.0.tgz", + "integrity": "sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ==", + "dev": true, + "license": "MIT", + "bin": { + "lz-string": "bin/bin.js" + } + }, "node_modules/magic-string": { "version": "0.30.21", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", @@ -4250,15 +4424,15 @@ } }, "node_modules/magicast": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/magicast/-/magicast-0.5.2.tgz", - "integrity": "sha512-E3ZJh4J3S9KfwdjZhe2afj6R9lGIN5Pher1pF39UGrXRqq/VDaGVIGN13BjHd2u8B61hArAGOnso7nBOouW3TQ==", + "version": "0.3.5", + "resolved": "https://registry.npmjs.org/magicast/-/magicast-0.3.5.tgz", + "integrity": "sha512-L0WhttDl+2BOsybvEOLK7fW3UA0OQ0IQ2d6Zl2x/a6vVRs3bAY0ECOSHHeL5jD+SbOpOCUEi0y1DgHEn9Qn1AQ==", "dev": true, "license": "MIT", "dependencies": { - "@babel/parser": "^7.29.0", - "@babel/types": "^7.29.0", - "source-map-js": "^1.2.1" + "@babel/parser": "^7.25.4", + "@babel/types": "^7.25.4", + "source-map-js": "^1.2.0" } }, "node_modules/make-dir": { @@ -4320,6 +4494,16 @@ "node": "*" } }, + "node_modules/minipass": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", + "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", @@ -4360,17 +4544,6 @@ "dev": true, "license": "MIT" }, - "node_modules/obug": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/obug/-/obug-2.1.1.tgz", - "integrity": "sha512-uTqF9MuPraAQ+IsnPf366RG4cP9RtUi7MLO1N3KEc+wb0a6yKpeL0lmk2IB1jY5KHPAlTc6T/JRdC/YqxHNwkQ==", - "dev": true, - "funding": [ - "https://github.com/sponsors/sxzz", - "https://opencollective.com/debug" - ], - "license": "MIT" - }, "node_modules/optionator": { "version": "0.9.4", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", @@ -4421,6 +4594,13 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/package-json-from-dist": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", + "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", + "dev": true, + "license": "BlueOak-1.0.0" + }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -4467,6 +4647,30 @@ "node": ">=8" } }, + "node_modules/path-scurry": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", + "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^10.2.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + }, + "engines": { + "node": ">=16 || 14 >=14.18" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/path-scurry/node_modules/lru-cache": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", + "dev": true, + "license": "ISC" + }, "node_modules/pathe": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", @@ -4543,6 +4747,34 @@ "node": ">= 0.8.0" } }, + "node_modules/pretty-format": { + "version": "27.5.1", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-27.5.1.tgz", + "integrity": "sha512-Qb1gy5OrP5+zDf2Bvnzdl3jsTf1qXVMazbvCoKhtKqVs4/YK4ozX4gKQJJVyNe+cajNPn0KoC0MC3FUmaHWEmQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1", + "ansi-styles": "^5.0.0", + "react-is": "^17.0.1" + }, + "engines": { + "node": "^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0" + } + }, + "node_modules/pretty-format/node_modules/ansi-styles": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, "node_modules/punycode": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", @@ -4574,6 +4806,13 @@ "react": "^19.2.4" } }, + "node_modules/react-is": { + "version": "17.0.2", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz", + "integrity": "sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==", + "dev": true, + "license": "MIT" + }, "node_modules/react-refresh": { "version": "0.18.0", "resolved": "https://registry.npmjs.org/react-refresh/-/react-refresh-0.18.0.tgz", @@ -4780,6 +5019,19 @@ "dev": true, "license": "ISC" }, + "node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/source-map-js": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", @@ -4804,6 +5056,103 @@ "dev": true, "license": "MIT" }, + "node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/string-width-cjs": { + "name": "string-width", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/string-width-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.2.tgz", + "integrity": "sha512-gmBGslpoQJtgnMAvOVqGZpEz9dyoKTCzy2nfz/n8aIFhN/jCE/rCmcxabB6jOOHV+0WNnylOxaxBQPSvcWklhA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/strip-ansi-cjs": { + "name": "strip-ansi", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi/node_modules/ansi-regex": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", + "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, "node_modules/strip-indent": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/strip-indent/-/strip-indent-3.0.0.tgz", @@ -4891,6 +5240,47 @@ "url": "https://opencollective.com/webpack" } }, + "node_modules/test-exclude": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-7.0.1.tgz", + "integrity": "sha512-pFYqmTw68LXVjeWJMST4+borgQP2AyMNbg1BpZh9LbyhUeNkeaPF9gzfPGUAnSMV3qPYdWUwDIjjCLiSDOl7vg==", + "dev": true, + "license": "ISC", + "dependencies": { + "@istanbuljs/schema": "^0.1.2", + "glob": "^10.4.1", + "minimatch": "^9.0.4" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/test-exclude/node_modules/brace-expansion": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/test-exclude/node_modules/minimatch": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", + "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/tinybench": { "version": "2.9.0", "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", @@ -4933,9 +5323,9 @@ } }, "node_modules/tinyrainbow": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.0.3.tgz", - "integrity": "sha512-PSkbLUoxOFRzJYjjxHJt9xro7D+iilgMX/C9lawzVuYiIdcihh9DXmVibBe8lmcFrRi/VzlPjBxbN7rH24q8/Q==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-2.0.0.tgz", + "integrity": "sha512-op4nsTR47R6p0vMUUoYl/a+ljLFVtlfaXkLQmqfLR1qHma1h/ysYk4hEXZ880bf2CYgTskvTa/e196Vd5dDQXw==", "dev": true, "license": "MIT", "engines": { @@ -5309,16 +5699,6 @@ "url": "https://opencollective.com/vitest" } }, - "node_modules/vitest/node_modules/tinyrainbow": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-2.0.0.tgz", - "integrity": "sha512-op4nsTR47R6p0vMUUoYl/a+ljLFVtlfaXkLQmqfLR1qHma1h/ysYk4hEXZ880bf2CYgTskvTa/e196Vd5dDQXw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=14.0.0" - } - }, "node_modules/w3c-xmlserializer": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-5.0.0.tgz", @@ -5423,6 +5803,91 @@ "node": ">=0.10.0" } }, + "node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs": { + "name": "wrap-ansi", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/wrap-ansi-cjs/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi/node_modules/ansi-styles": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, "node_modules/ws": { "version": "8.19.0", "resolved": "https://registry.npmjs.org/ws/-/ws-8.19.0.tgz", diff --git a/frontend/package.json b/frontend/package.json index ec763b6..44356f3 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -25,6 +25,7 @@ "devDependencies": { "@eslint/js": "^9.39.1", "@tailwindcss/vite": "^4.1.18", + "@testing-library/dom": "^10.4.0", "@testing-library/jest-dom": "^6.9.1", "@testing-library/react": "^16.3.2", "@testing-library/user-event": "^14.6.1", @@ -32,7 +33,7 @@ "@types/react": "^19.2.7", "@types/react-dom": "^19.2.3", "@vitejs/plugin-react": "^5.1.1", - "@vitest/coverage-v8": "^4.0.18", + "@vitest/coverage-v8": "^3.2.4", "eslint": "^9.39.1", "eslint-plugin-react-hooks": "^7.0.1", "eslint-plugin-react-refresh": "^0.4.24", From 199a3855fcad6a0ae1da87b7b8bc7c375faffa11 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Wed, 18 Feb 2026 09:27:27 -0700 Subject: [PATCH 100/173] Frontend: Ignore _* unused variables --- frontend/eslint.config.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/frontend/eslint.config.js b/frontend/eslint.config.js index 5e6b472..427b247 100644 --- a/frontend/eslint.config.js +++ b/frontend/eslint.config.js @@ -19,5 +19,8 @@ export default defineConfig([ ecmaVersion: 2020, globals: globals.browser, }, + rules: { + '@typescript-eslint/no-unused-vars': ['error', { argsIgnorePattern: '^_' }], + }, }, ]) From b8409c77da8b899cb29f398b599ee29b3acaa42d Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Wed, 18 Feb 2026 10:00:01 -0700 Subject: [PATCH 101/173] Delete: Add folder/file exclusion --- app/routes/delete.py | 10 +++++++++- app/services/delete_manager.py | 23 ++++++++++++++++++++++- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/app/routes/delete.py b/app/routes/delete.py index 2f34db7..b392c03 100644 --- a/app/routes/delete.py +++ b/app/routes/delete.py @@ -56,11 +56,19 @@ def scan_folder() -> tuple[Response, int]: if not folder_path.is_dir(): return jsonify({"error": f"Path is not a directory: {folder_path}"}), 400 + excluded_subfolders: list[str] = data.get("excluded_subfolders", []) + excluded_files: list[str] = data.get("excluded_files", []) + settings = get_settings() manager = get_delete_manager() try: - job = manager.scan_folder(str(folder_path.absolute()), settings.s3_bucket) + job = manager.scan_folder( + str(folder_path.absolute()), + settings.s3_bucket, + excluded_subfolders=excluded_subfolders, + excluded_files=excluded_files, + ) except PermissionError as e: return jsonify({"error": f"Permission denied: {e}"}), 403 diff --git a/app/services/delete_manager.py b/app/services/delete_manager.py index 671909e..a80fafe 100644 --- a/app/services/delete_manager.py +++ b/app/services/delete_manager.py @@ -168,12 +168,20 @@ class DeleteManager: def __init__(self) -> None: self.jobs: dict[str, DeleteJob] = {} - def scan_folder(self, folder_path: str, bucket: str) -> DeleteJob: + def scan_folder( + self, + folder_path: str, + bucket: str, + excluded_subfolders: list[str] | None = None, + excluded_files: list[str] | None = None, + ) -> DeleteJob: """Scan a folder for .mcap files and cross-reference with upload cache. Args: folder_path: Local directory to scan bucket: S3 bucket to check against + excluded_subfolders: Subfolder names to skip + excluded_files: Root-level filenames to skip Returns: A new DeleteJob with files matched against the cache @@ -182,11 +190,24 @@ def scan_folder(self, folder_path: str, bucket: str) -> DeleteJob: job = DeleteJob(job_id=job_id) cache = get_cache_service() folder = Path(folder_path) + excluded_subs_set = set(excluded_subfolders or []) + excluded_files_set = set(excluded_files or []) for mcap_path in sorted(folder.rglob("*.mcap")): if not mcap_path.is_file(): continue + rel = mcap_path.relative_to(folder) + parts = rel.parts + + # Skip root-level files that are excluded + if len(parts) == 1 and parts[0] in excluded_files_set: + continue + + # Skip files under excluded subfolders + if len(parts) > 1 and parts[0] in excluded_subs_set: + continue + stat = mcap_path.stat() file_size = stat.st_size filename = mcap_path.name From 34558c64f1b9fc8e52c9f06c1cee57a8ece85f62 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Wed, 18 Feb 2026 10:00:27 -0700 Subject: [PATCH 102/173] Frontent: Wrap refs in use effect --- frontend/src/hooks/useSSE.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/frontend/src/hooks/useSSE.ts b/frontend/src/hooks/useSSE.ts index 3bf7c02..cce1988 100644 --- a/frontend/src/hooks/useSSE.ts +++ b/frontend/src/hooks/useSSE.ts @@ -21,8 +21,11 @@ export function useSSE({ url, onMessage, onError }: UseSSEOptions): void { // because the caller created a new closure. const onMessageRef = useRef(onMessage); const onErrorRef = useRef(onError); - onMessageRef.current = onMessage; - onErrorRef.current = onError; + + useEffect(() => { + onMessageRef.current = onMessage; + onErrorRef.current = onError; + }); useEffect(() => { if (!url) return; From 964eec8a2d5a40d4bbc323e60aacf5900f7a8e3e Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Wed, 18 Feb 2026 10:01:22 -0700 Subject: [PATCH 103/173] Frontend: Add is cancelling state to upload --- frontend/src/hooks/useUploadJob.ts | 54 +++++++++++++++++++----------- 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/frontend/src/hooks/useUploadJob.ts b/frontend/src/hooks/useUploadJob.ts index 4429fb6..f11e71c 100644 --- a/frontend/src/hooks/useUploadJob.ts +++ b/frontend/src/hooks/useUploadJob.ts @@ -7,7 +7,7 @@ * only in the terminal event and is stored in uploadStore.completedJob. */ -import { useCallback, useMemo, useRef, useState } from "react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { apiPost } from "../api/client.ts"; import { useUploadStore } from "../stores/uploadStore.ts"; @@ -27,6 +27,13 @@ interface StatusCounts { failed: number; } +interface UseUploadJobOptions { + /** Called for each file update during analysis/upload. */ + onFileUpdate?: (file: FileUploadState) => void; + /** Called with the full file list on completion. */ + onCompletion?: (files: FileUploadState[]) => void; +} + interface UseUploadJobResult { startUpload: ( filePaths: string[], @@ -40,12 +47,9 @@ interface UseUploadJobResult { progressPercent: number; eta: number | null; isRunning: boolean; + isCancelling: boolean; uploadedBytesFormatted: string; totalBytesFormatted: string; - /** Per-file SSE callback ref — wire to FileStore.updateFile. */ - onFileUpdate: React.MutableRefObject<((file: FileUploadState) => void) | null>; - /** Completion callback ref — wire to FileStore.mergeCompletion. */ - onCompletion: React.MutableRefObject<((files: FileUploadState[]) => void) | null>; } type SSEEvent = @@ -65,7 +69,7 @@ function isFullJobDict(data: Record): data is UploadJob & Recor return "job_id" in data && "total_bytes" in data && !("type" in data); } -export function useUploadJob(): UseUploadJobResult { +export function useUploadJob(options: UseUploadJobOptions = {}): UseUploadJobResult { const [jobId, setJobId] = useState(null); const [isRunning, setIsRunning] = useState(false); const [filesProcessed, setFilesProcessed] = useState(0); @@ -80,12 +84,18 @@ export function useUploadJob(): UseUploadJobResult { const [eta, setEta] = useState(null); const [uploadedBytesFormatted, setUploadedBytesFormatted] = useState(""); const [totalBytesFormatted, setTotalBytesFormatted] = useState(""); + const [isCancelling, setIsCancelling] = useState(false); const { setUploadJobId, setCompletedJob } = useUploadStore(); - // Callback refs for per-file updates — wired by UploadPage to FileStore - const onFileUpdateRef = useRef<((file: FileUploadState) => void) | null>(null); - const onCompletionRef = useRef<((files: FileUploadState[]) => void) | null>(null); + // Callback refs — kept in sync with latest options via effect + const onFileUpdateRef = useRef(options.onFileUpdate); + const onCompletionRef = useRef(options.onCompletion); + + useEffect(() => { + onFileUpdateRef.current = options.onFileUpdate; + onCompletionRef.current = options.onCompletion; + }); /** Handle each SSE event. */ const handleMessage = useCallback( @@ -172,6 +182,7 @@ export function useUploadJob(): UseUploadJobResult { p.status === "cancelled" ) { setIsRunning(false); + setIsCancelling(false); setJobId(null); } return; @@ -196,6 +207,7 @@ export function useUploadJob(): UseUploadJobResult { // Notify unified table with all completion data onCompletionRef.current?.(job.files); setIsRunning(false); + setIsCancelling(false); setJobId(null); return; } @@ -215,6 +227,7 @@ export function useUploadJob(): UseUploadJobResult { // Stream closed — if we're still "running" the server ended it. if (isRunning) { setIsRunning(false); + setIsCancelling(false); setJobId(null); } }, @@ -232,6 +245,7 @@ export function useUploadJob(): UseUploadJobResult { setEta(null); setUploadedBytesFormatted(""); setTotalBytesFormatted(""); + setIsCancelling(false); setIsRunning(true); try { @@ -253,18 +267,19 @@ export function useUploadJob(): UseUploadJobResult { [setUploadJobId], ); - /** Cancel the current upload job. */ + /** Cancel the current upload job. SSE terminal event handles cleanup. */ const cancelUpload = useCallback(async () => { const currentJobId = useUploadStore.getState().uploadJobId; - if (currentJobId) { - try { - await apiPost(`/api/upload/cancel/${currentJobId}`); - } catch { - // Ignore - } + if (!currentJobId) return; + setIsCancelling(true); + try { + await apiPost(`/api/upload/cancel/${currentJobId}`); + } catch { + // If the cancel request itself fails, force-close + setIsRunning(false); + setJobId(null); + setIsCancelling(false); } - setIsRunning(false); - setJobId(null); }, []); return { @@ -277,9 +292,8 @@ export function useUploadJob(): UseUploadJobResult { progressPercent, eta, isRunning, + isCancelling, uploadedBytesFormatted, totalBytesFormatted, - onFileUpdate: onFileUpdateRef, - onCompletion: onCompletionRef, }; } From d8a52f010a94e719c161242323165ed5f2e4cb21 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Wed, 18 Feb 2026 10:01:56 -0700 Subject: [PATCH 104/173] Frontend: Add exclusions to delete scan --- frontend/src/hooks/useDeleteScan.ts | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/frontend/src/hooks/useDeleteScan.ts b/frontend/src/hooks/useDeleteScan.ts index cecdcd6..1bde0c9 100644 --- a/frontend/src/hooks/useDeleteScan.ts +++ b/frontend/src/hooks/useDeleteScan.ts @@ -11,8 +11,13 @@ import { apiPost } from "../api/client.ts"; import { useDeleteStore } from "../stores/deleteStore.ts"; import type { DeleteScanResponse } from "../types/delete.ts"; +interface ScanExclusions { + subfolders: string[]; + files: string[]; +} + interface UseDeleteScanResult { - scan: (folderPath: string) => Promise; + scan: (folderPath: string, exclusions?: ScanExclusions) => Promise; isScanning: boolean; } @@ -21,12 +26,15 @@ export function useDeleteScan(): UseDeleteScanResult { useDeleteStore(); const scan = useCallback( - async (folderPath: string) => { + async (folderPath: string, exclusions?: ScanExclusions) => { setIsScanning(true); try { - const res = await apiPost("/api/delete/scan", { - folder_path: folderPath, - }); + const body: Record = { folder_path: folderPath }; + if (exclusions) { + body.excluded_subfolders = exclusions.subfolders; + body.excluded_files = exclusions.files; + } + const res = await apiPost("/api/delete/scan", body); setDeleteJobId(res.job_id); setScanResults(res.files, res.total_size); } finally { From 84c19d31c44ee27f2092bf3a250d8b3b46198e2b Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Wed, 18 Feb 2026 10:02:23 -0700 Subject: [PATCH 105/173] Frontend: Add isCancelling state to delete page --- frontend/src/pages/DeletePage.tsx | 66 +++++++++++++++++++++++++------ 1 file changed, 54 insertions(+), 12 deletions(-) diff --git a/frontend/src/pages/DeletePage.tsx b/frontend/src/pages/DeletePage.tsx index 56017e5..782dbf1 100644 --- a/frontend/src/pages/DeletePage.tsx +++ b/frontend/src/pages/DeletePage.tsx @@ -11,10 +11,13 @@ import { useCallback, useEffect, useMemo, useState } from "react"; import ProgressBar from "../components/common/ProgressBar.tsx"; +import Spinner from "../components/common/Spinner.tsx"; import StatCard from "../components/common/StatCard.tsx"; import DeleteConfirmation from "../components/delete/DeleteConfirmation.tsx"; import DeleteStepper from "../components/delete/DeleteStepper.tsx"; +import CancelConfirmModal from "../components/upload/CancelConfirmModal.tsx"; import FolderBrowser from "../components/upload/FolderBrowser.tsx"; +import type { FolderExclusions } from "../components/upload/FolderBrowser.tsx"; import { useDeleteJob } from "../hooks/useDeleteJob.ts"; import { useDeleteScan } from "../hooks/useDeleteScan.ts"; import { useAppStore } from "../stores/appStore.ts"; @@ -73,14 +76,16 @@ export default function DeletePage() { // Pagination for file table const [page, setPage] = useState(0); const pageSize = 50; + const [showCancelModal, setShowCancelModal] = useState(false); + const [refreshKey, setRefreshKey] = useState(0); // ── Step transitions ── const handleFolderSelected = useCallback( - async (path: string) => { + async (path: string, exclusions?: FolderExclusions) => { setFolderPath(path); setStep(2); - await scan(path); + await scan(path, exclusions); }, [setFolderPath, setStep, scan], ); @@ -90,6 +95,15 @@ export default function DeletePage() { await deleteJob.startDelete(); }, [setStep, deleteJob]); + const handleCancelClick = useCallback(() => { + setShowCancelModal(true); + }, []); + + const handleConfirmCancel = useCallback(async () => { + setShowCancelModal(false); + await deleteJob.cancelDelete(); + }, [deleteJob]); + // Auto-advance from Step 4 to Step 5 when deletion completes useEffect(() => { if (step === 4 && !deleteJob.isRunning && completedJob) { @@ -98,8 +112,13 @@ export default function DeletePage() { }, [step, deleteJob.isRunning, completedJob, setStep]); const handleStartOver = useCallback(() => { + const previousPath = folderPath; reset(); - }, [reset]); + // Preserve the folder path so the browser reopens at the same location + if (previousPath) setFolderPath(previousPath); + // Bump key to force FolderBrowser to re-mount with fresh data + setRefreshKey((k) => k + 1); + }, [reset, folderPath, setFolderPath]); const handleBack = useCallback(() => { if (step === 2) { @@ -149,6 +168,7 @@ export default function DeletePage() { {/* Step 1: Folder Selection */} {step === 1 && ( - S3 Path + Cloud Path @@ -311,20 +331,42 @@ export default function DeletePage() { percent={progressPercent} label={ deleteJob.jobStatus === "verifying" - ? "Verifying files against S3..." + ? "Verifying files against cloud storage..." : deleteJob.jobStatus === "deleting" ? "Clearing verified files..." : "Processing..." } /> - + {!deleteJob.isCancelling && ( + + )} + + {/* Cancel confirmation modal */} + setShowCancelModal(false)} + onConfirm={handleConfirmCancel} + filesProcessed={deleteJob.filesProcessed} + totalFiles={deleteJob.totalFiles} + /> + + {/* Cancelling overlay */} + {deleteJob.isCancelling && ( +
+
+ +

Cancelling...

+

Waiting for in-progress operations to finish.

+
+
+ )} )} From 4745eceae4b3618f99ae6ee1fbd379090ee6f6d7 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Wed, 18 Feb 2026 10:02:43 -0700 Subject: [PATCH 106/173] Frontend: Add Cancel Confirmation modal --- .../components/upload/CancelConfirmModal.tsx | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 frontend/src/components/upload/CancelConfirmModal.tsx diff --git a/frontend/src/components/upload/CancelConfirmModal.tsx b/frontend/src/components/upload/CancelConfirmModal.tsx new file mode 100644 index 0000000..253149c --- /dev/null +++ b/frontend/src/components/upload/CancelConfirmModal.tsx @@ -0,0 +1,66 @@ +/** + * Confirmation modal shown when the user clicks "Cancel Upload". + */ + +import Modal from "../common/Modal.tsx"; +import { WarningIcon } from "../../utils/icons.tsx"; + +interface CancelConfirmModalProps { + isOpen: boolean; + onClose: () => void; + onConfirm: () => void; + filesProcessed: number; + totalFiles: number; +} + +export default function CancelConfirmModal({ + isOpen, + onClose, + onConfirm, + filesProcessed, + totalFiles, +}: CancelConfirmModalProps) { + const remaining = totalFiles - filesProcessed; + + return ( + + + + + } + > +
+
+ +
+

+ {filesProcessed} of {totalFiles} files + have been processed so far. Cancelling will stop the remaining{" "} + {remaining} file{remaining !== 1 ? "s" : ""} from being uploaded. +

+

+ Files already uploaded will remain in cloud storage. +

+
+
+
+
+ ); +} From 4ec3d5b9921b1c97628d4c9e5e48fd4567f8c1ac Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Wed, 18 Feb 2026 10:04:02 -0700 Subject: [PATCH 107/173] Frontend: Add is cancelling state to upload --- frontend/src/pages/UploadPage.tsx | 111 ++++++++++++++++++++---------- 1 file changed, 76 insertions(+), 35 deletions(-) diff --git a/frontend/src/pages/UploadPage.tsx b/frontend/src/pages/UploadPage.tsx index 146c136..6f27c7d 100644 --- a/frontend/src/pages/UploadPage.tsx +++ b/frontend/src/pages/UploadPage.tsx @@ -6,8 +6,10 @@ * with phase-aware header, toolbar, and footer. */ -import { useCallback, useEffect, useLayoutEffect, useMemo, useState } from "react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import Spinner from "../components/common/Spinner.tsx"; +import CancelConfirmModal from "../components/upload/CancelConfirmModal.tsx"; import ConfirmModal from "../components/upload/ConfirmModal.tsx"; import FolderBrowser from "../components/upload/FolderBrowser.tsx"; import type { FolderExclusions } from "../components/upload/FolderBrowser.tsx"; @@ -48,48 +50,53 @@ export default function UploadPage() { totals, } = useFolderScan(); - const uploadJob = useUploadJob(); - // Unified file store const { files, store } = useFileStore(); + // SSE callback: map each file event to the FileStore + const handleFileUpdate = useCallback((file: FileUploadState) => { + const statusMap: Record = { + analyzing: "in_progress", + uploading: "in_progress", + completed: "completed", + skipped: "skipped", + failed: "failed", + cancelled: "failed", + }; + store.updateFile(file.local_path, { + status: statusMap[file.status] ?? "queued", + progressPercent: file.progress_percent, + s3Path: file.s3_path || undefined, + error: file.error_message || undefined, + duration: file.upload_duration_seconds, + speed: file.upload_speed_mbps, + }); + }, [store]); + + // SSE callback: merge full completion data into the FileStore + const handleCompletion = useCallback((completionFiles: FileUploadState[]) => { + store.mergeCompletion(completionFiles); + }, [store]); + + const uploadJob = useUploadJob({ + onFileUpdate: handleFileUpdate, + onCompletion: handleCompletion, + }); + // Local state const [showConfirmModal, setShowConfirmModal] = useState(false); + const [showCancelModal, setShowCancelModal] = useState(false); const [pendingSelectedPaths, setPendingSelectedPaths] = useState([]); const [selectedPaths, setSelectedPaths] = useState>(new Set()); // Derive phase from step const phase: UploadPhase = step <= 2 ? "review" : step === 3 ? "uploading" : "summary"; - // ── Wire SSE callbacks to FileStore (useLayoutEffect to avoid race) ── - - useLayoutEffect(() => { - uploadJob.onFileUpdate.current = (file: FileUploadState) => { - const statusMap: Record = { - analyzing: "in_progress", - uploading: "in_progress", - completed: "completed", - skipped: "skipped", - failed: "failed", - cancelled: "failed", - }; - store.updateFile(file.local_path, { - status: statusMap[file.status] ?? "queued", - progressPercent: file.progress_percent, - s3Path: file.s3_path || undefined, - error: file.error_message || undefined, - duration: file.upload_duration_seconds, - speed: file.upload_speed_mbps, - }); - }; - uploadJob.onCompletion.current = (completionFiles: FileUploadState[]) => { - store.mergeCompletion(completionFiles); - }; - }); - // ── Sync FileStore from scan data as folders arrive ── - useEffect(() => { + const prevFoldersRef = useRef(folders); + if (folders !== prevFoldersRef.current) { + prevFoldersRef.current = folders; if (folders.length > 0 && step >= 2) { store.buildFromScan(folders); // Auto-select new files @@ -103,7 +110,7 @@ export default function UploadPage() { } setSelectedPaths(newSelected); } - }, [folders, step, store]); + } // ── Freeze/unfreeze sort on phase transitions ── @@ -149,7 +156,18 @@ export default function UploadPage() { [pendingSelectedPaths, setStep, uploadJob, store], ); - /** Step 3 -> 4: Upload finished, advance to completion. */ + /** Cancel button → show confirmation modal. */ + const handleCancelClick = useCallback(() => { + setShowCancelModal(true); + }, []); + + /** Cancel confirmed → send cancel to backend, overlay locks the screen. */ + const handleConfirmCancel = useCallback(async () => { + setShowCancelModal(false); + await uploadJob.cancelUpload(); + }, [uploadJob]); + + /** Step 3 -> 4: Upload finished (or cancelled), advance to completion. */ useEffect(() => { if (step === 3 && !uploadJob.isRunning && completedJob) { setStep(4); @@ -194,7 +212,10 @@ export default function UploadPage() { }); }, []); - const allFiles = useMemo(() => Array.from(store.getAllRows().values()), [files]); + // `files` is listed as a dep to re-derive when the store snapshot changes + // (store itself is a stable singleton). + // eslint-disable-next-line react-hooks/exhaustive-deps + const allFiles = useMemo(() => Array.from(store.getAllRows().values()), [store, files]); const toggleAllFiltered = useCallback(() => { const filteredPaths = files.map((f) => f.path); @@ -371,8 +392,8 @@ export default function UploadPage() { onBack={handleBack} onStartUpload={handleStartUploadClick} selectedNewCount={selectedNewCount} - onCancel={uploadJob.cancelUpload} - isRunning={uploadJob.isRunning} + onCancel={handleCancelClick} + isRunning={uploadJob.isRunning && !uploadJob.isCancelling} onDownloadCsv={handleDownloadCsv} onUploadMore={handleUploadMore} failedCount={uploadJob.statusCounts.failed} @@ -389,6 +410,26 @@ export default function UploadPage() { totalSize={selectedTotals.totalSize} /> )} + + {/* Cancel confirmation modal */} + setShowCancelModal(false)} + onConfirm={handleConfirmCancel} + filesProcessed={uploadJob.filesProcessed} + totalFiles={uploadJob.totalFiles} + /> + + {/* Cancelling overlay — locks the screen while backend winds down */} + {uploadJob.isCancelling && ( +
+
+ +

Cancelling upload...

+

Waiting for in-progress files to finish.

+
+
+ )} )} From 174448f0e8724e0552a6cb0f676645405e5b86bf Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Wed, 18 Feb 2026 10:04:37 -0700 Subject: [PATCH 108/173] Frontend: Add refresh button to FolderBrowser --- .../src/components/upload/FolderBrowser.tsx | 51 +++++++++++++++++-- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/frontend/src/components/upload/FolderBrowser.tsx b/frontend/src/components/upload/FolderBrowser.tsx index 79b770d..51b9876 100644 --- a/frontend/src/components/upload/FolderBrowser.tsx +++ b/frontend/src/components/upload/FolderBrowser.tsx @@ -23,6 +23,7 @@ import { ChevronRightIcon, CheckIcon, PlusIcon, + RefreshIcon, UploadIcon, } from "../../utils/icons.tsx"; @@ -124,6 +125,15 @@ export default function FolderBrowser({ navigate(initialPath); }, [navigate, initialPath]); + /** Re-fetch the current directory without resetting to initialPath. */ + const refresh = useCallback(() => { + if (data) { + navigate(data.current_path); + } else { + navigate(initialPath); + } + }, [data, navigate, initialPath]); + // Reset checked state and search when data changes (new directory loaded) useEffect(() => { if (!data) return; @@ -151,6 +161,30 @@ export default function FolderBrowser({ const checkedCount = checkedFolders.size + checkedFiles.size; const allChecked = totalItems > 0 && checkedCount === totalItems; + // Count actionable files among the user's current selection + const selectedActionableCount = useMemo(() => { + if (!data) return 0; + let count = 0; + // Checked folders: sum their actionable file counts + for (const folder of data.folders) { + if (checkedFolders.has(folder.name)) { + count += mode === "upload" + ? folder.mcap_count - folder.already_uploaded + : folder.already_uploaded; + } + } + // Checked loose files: count actionable ones + for (const file of data.files) { + if (checkedFiles.has(file.name)) { + const isUploaded = file.already_uploaded ?? false; + if (mode === "upload" ? !isUploaded : isUploaded) { + count += 1; + } + } + } + return count; + }, [data, checkedFolders, checkedFiles, mode]); + const selectAll = useCallback(() => { if (!data) return; setCheckedFolders(new Set(data.folders.map((f) => f.name))); @@ -306,9 +340,18 @@ export default function FolderBrowser({ {/* Main content area */}
- {/* Breadcrumbs */} -
+ {/* Breadcrumbs + refresh */} +
+
{/* Summary bar — always visible when we have data and MCAP files exist */} @@ -359,7 +402,7 @@ export default function FolderBrowser({ ${cfg.buttonColor} disabled:bg-gray-300 disabled:text-gray-500 disabled:cursor-not-allowed`} > - {cfg.buttonLabel(actionableCount)} + {cfg.buttonLabel(selectedActionableCount)}
@@ -581,7 +624,7 @@ function FolderList({ {/* Upload status indicator */} {allUploaded && ( - + {folder.mcap_count}/{folder.mcap_count} uploaded )} From 377259290c0ce932da2eb7224217653e08be0f77 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Wed, 18 Feb 2026 10:05:03 -0700 Subject: [PATCH 109/173] Frontend: Add more context to uploaded success messages --- .../src/components/upload/UploadFooter.tsx | 44 +++++++++++-------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/frontend/src/components/upload/UploadFooter.tsx b/frontend/src/components/upload/UploadFooter.tsx index d39c786..fe60a94 100644 --- a/frontend/src/components/upload/UploadFooter.tsx +++ b/frontend/src/components/upload/UploadFooter.tsx @@ -3,9 +3,10 @@ * * - Review: Back + Start Upload * - Upload: Cancel Upload - * - Summary: Download CSV + Upload More + * - Summary: info blurb + Download CSV + Upload More */ +import { Link } from "react-router-dom"; import type { UploadPhase } from "../../types/upload.ts"; import { ChevronRightIcon, XCircleIcon, DownloadIcon, UploadIcon } from "../../utils/icons.tsx"; @@ -81,23 +82,30 @@ export default function UploadFooter({ // Summary return ( -
- - +
+

+ These results have been saved and can be reviewed anytime on the{" "} + History page. +

+
+ + +
); } From e5f133c5ce58a3a20603ee7c429c7750986b1fca Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Wed, 18 Feb 2026 10:05:42 -0700 Subject: [PATCH 110/173] Frontend: Ignore useVirtualizer type error --- frontend/src/components/upload/UnifiedFileTable.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/src/components/upload/UnifiedFileTable.tsx b/frontend/src/components/upload/UnifiedFileTable.tsx index ccf10fb..82485e3 100644 --- a/frontend/src/components/upload/UnifiedFileTable.tsx +++ b/frontend/src/components/upload/UnifiedFileTable.tsx @@ -56,6 +56,7 @@ export default function UnifiedFileTable({ }: UnifiedFileTableProps) { const scrollContainerRef = useRef(null); + // eslint-disable-next-line react-hooks/incompatible-library const rowVirtualizer = useVirtualizer({ count: files.length, getScrollElement: () => scrollContainerRef.current, From 77b93c63e4fad0cbf0069c960916e35a891e2b6b Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Wed, 18 Feb 2026 10:06:06 -0700 Subject: [PATCH 111/173] Frontend: Use darker green for success --- frontend/src/components/common/AlertBanner.tsx | 4 ++-- frontend/src/components/common/Stepper.tsx | 10 +++++----- frontend/src/components/upload/UnifiedFileTable.tsx | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/frontend/src/components/common/AlertBanner.tsx b/frontend/src/components/common/AlertBanner.tsx index 9013a4f..5b802c4 100644 --- a/frontend/src/components/common/AlertBanner.tsx +++ b/frontend/src/components/common/AlertBanner.tsx @@ -51,8 +51,8 @@ const iconColorStyles: Record = { info: "text-blue-500", warning: "text-yellow-500", error: "text-red-500", - success: "text-green-500", - shield: "text-green-500", + success: "text-green-700", + shield: "text-green-700", }; export default function AlertBanner({ diff --git a/frontend/src/components/common/Stepper.tsx b/frontend/src/components/common/Stepper.tsx index 7ba4754..0a9b244 100644 --- a/frontend/src/components/common/Stepper.tsx +++ b/frontend/src/components/common/Stepper.tsx @@ -56,8 +56,8 @@ export default function Stepper({ let circleClass = "w-9 h-9 rounded-full flex items-center justify-center text-sm font-bold transition-colors"; if (isDone) { - circleClass += " bg-green-500 text-white"; - if (canClick) circleClass += " cursor-pointer hover:bg-green-600"; + circleClass += " bg-green-700 text-white"; + if (canClick) circleClass += " cursor-pointer hover:bg-green-800"; } else if (isActive) { circleClass += " bg-nlr-blue text-white"; } else { @@ -79,12 +79,12 @@ export default function Stepper({ aria-label={`Step ${step.number}: ${step.label}${isDone ? " (completed)" : ""}${isActive ? " (current)" : ""}`} data-testid={`${testIdPrefix}-${step.number}`} > - {isDone ? : step.number} + {isDone ? : step.number} )} diff --git a/frontend/src/components/upload/UnifiedFileTable.tsx b/frontend/src/components/upload/UnifiedFileTable.tsx index 82485e3..9d27915 100644 --- a/frontend/src/components/upload/UnifiedFileTable.tsx +++ b/frontend/src/components/upload/UnifiedFileTable.tsx @@ -294,7 +294,7 @@ function StatusIcon({ status }: { status: UnifiedStatus }) { case "in_progress": return ; case "completed": - return ; + return ; case "failed": return ; case "skipped": From 40a2ba81d8c791a50aef9d1de205970d1976b50c Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Wed, 18 Feb 2026 10:07:00 -0700 Subject: [PATCH 112/173] Frontend: Add link to uploaded files in delete screen --- .../components/delete/DeleteConfirmation.tsx | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/frontend/src/components/delete/DeleteConfirmation.tsx b/frontend/src/components/delete/DeleteConfirmation.tsx index a973ac2..5479c1d 100644 --- a/frontend/src/components/delete/DeleteConfirmation.tsx +++ b/frontend/src/components/delete/DeleteConfirmation.tsx @@ -6,6 +6,7 @@ */ import { useState } from "react"; +import { Link } from "react-router-dom"; import { formatBytes } from "../../utils/format/bytes.ts"; import AlertBanner from "../common/AlertBanner.tsx"; @@ -43,11 +44,19 @@ export default function DeleteConfirmation({ } /> - {/* S3 reassurance */} + {/* Cloud reassurance */} + Only local copies are removed. Your files in{" "} + + cloud storage + + {" "}remain safe and unchanged. Each file is verified against the cloud upload before removal. + + } /> {/* Type-to-confirm */} @@ -79,7 +88,7 @@ export default function DeleteConfirmation({ /> I understand that cleared files cannot be recovered from this drive - and that the S3 copies have been verified. + and that the cloud uploads have been verified. From 5723e0bd9cff5dad861923e63771cf5cbf6b6f52 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Wed, 18 Feb 2026 10:07:20 -0700 Subject: [PATCH 113/173] Frontend: Add is delete hooks for cancelling state --- frontend/src/hooks/useDeleteJob.ts | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/frontend/src/hooks/useDeleteJob.ts b/frontend/src/hooks/useDeleteJob.ts index e02823d..49e4a80 100644 --- a/frontend/src/hooks/useDeleteJob.ts +++ b/frontend/src/hooks/useDeleteJob.ts @@ -32,6 +32,7 @@ interface UseDeleteJobResult { statusCounts: StatusCounts; totalDeletedSize: number; isRunning: boolean; + isCancelling: boolean; jobStatus: string; } @@ -72,6 +73,7 @@ export function useDeleteJob(): UseDeleteJobResult { }); const [totalDeletedSize, setTotalDeletedSize] = useState(0); const [jobStatus, setJobStatus] = useState("pending"); + const [isCancelling, setIsCancelling] = useState(false); const [activeJobId, setActiveJobId] = useState(null); const { deleteJobId, setCompletedJob, setIsDeleting } = useDeleteStore(); @@ -99,6 +101,7 @@ export function useDeleteJob(): UseDeleteJobResult { setTotalDeletedSize(c.total_deleted_size); setJobStatus(c.status); setIsRunning(false); + setIsCancelling(false); setIsDeleting(false); setActiveJobId(null); setCompletedJob(c as unknown as DeleteJobResult); @@ -122,6 +125,7 @@ export function useDeleteJob(): UseDeleteJobResult { onError: () => { if (isRunning) { setIsRunning(false); + setIsCancelling(false); setIsDeleting(false); setActiveJobId(null); } @@ -141,6 +145,7 @@ export function useDeleteJob(): UseDeleteJobResult { }); setTotalDeletedSize(0); setJobStatus("verifying"); + setIsCancelling(false); setIsRunning(true); setIsDeleting(true); @@ -153,18 +158,20 @@ export function useDeleteJob(): UseDeleteJobResult { } }, [deleteJobId, setIsDeleting]); + /** Cancel the delete job. SSE terminal event handles cleanup. */ const cancelDelete = useCallback(async () => { const jobId = useDeleteStore.getState().deleteJobId; - if (jobId) { - try { - await apiPost(`/api/delete/cancel/${jobId}`); - } catch { - // Ignore - } + if (!jobId) return; + setIsCancelling(true); + try { + await apiPost(`/api/delete/cancel/${jobId}`); + } catch { + // If cancel request fails, force-close + setIsRunning(false); + setIsCancelling(false); + setIsDeleting(false); + setActiveJobId(null); } - setIsRunning(false); - setIsDeleting(false); - setActiveJobId(null); }, [setIsDeleting]); return { @@ -175,6 +182,7 @@ export function useDeleteJob(): UseDeleteJobResult { statusCounts, totalDeletedSize, isRunning, + isCancelling, jobStatus, }; } From 6ff0f9bb99368f68defd9db6e69e9ca47ea24fdb Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Wed, 18 Feb 2026 10:07:42 -0700 Subject: [PATCH 114/173] Frontend: Remove unused icon sizes const --- frontend/src/utils/icons.tsx | 9 --------- 1 file changed, 9 deletions(-) diff --git a/frontend/src/utils/icons.tsx b/frontend/src/utils/icons.tsx index 6ca0e21..d917a5d 100644 --- a/frontend/src/utils/icons.tsx +++ b/frontend/src/utils/icons.tsx @@ -63,12 +63,3 @@ export const CircleIcon = Circle; // Export type for icon props export type { LucideProps as IconProps }; - -// Default icon size classes -export const iconSizes = { - xs: "w-3 h-3", - sm: "w-4 h-4", - md: "w-5 h-5", - lg: "w-6 h-6", - xl: "w-8 h-8", -} as const; From 14c8f2a845197e6ee09ad4891af7dc0e02b39be8 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Wed, 18 Feb 2026 10:08:06 -0700 Subject: [PATCH 115/173] Frontend: Add basic react/dom tests --- frontend/tests/components/S3Browser.test.tsx | 117 ++++++++++ frontend/tests/components/Stepper.test.tsx | 135 +++++++++++ frontend/tests/components/common.test.tsx | 225 +++++++++++++++++++ frontend/tests/components/layout.test.tsx | 112 +++++++++ frontend/tests/components/settings.test.tsx | 82 +++++++ frontend/tests/hooks/useDebounce.test.ts | 91 ++++++++ frontend/tests/hooks/usePagination.test.ts | 121 ++++++++++ frontend/tests/hooks/useSSE.test.ts | 177 +++++++++++++++ frontend/tests/setup.ts | 1 + frontend/tests/utils/formatters.test.ts | 110 +++++++++ frontend/vitest.config.ts | 12 + 11 files changed, 1183 insertions(+) create mode 100644 frontend/tests/components/S3Browser.test.tsx create mode 100644 frontend/tests/components/Stepper.test.tsx create mode 100644 frontend/tests/components/common.test.tsx create mode 100644 frontend/tests/components/layout.test.tsx create mode 100644 frontend/tests/components/settings.test.tsx create mode 100644 frontend/tests/hooks/useDebounce.test.ts create mode 100644 frontend/tests/hooks/usePagination.test.ts create mode 100644 frontend/tests/hooks/useSSE.test.ts create mode 100644 frontend/tests/setup.ts create mode 100644 frontend/tests/utils/formatters.test.ts create mode 100644 frontend/vitest.config.ts diff --git a/frontend/tests/components/S3Browser.test.tsx b/frontend/tests/components/S3Browser.test.tsx new file mode 100644 index 0000000..497ce3b --- /dev/null +++ b/frontend/tests/components/S3Browser.test.tsx @@ -0,0 +1,117 @@ +import { render, screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import S3Browser from "../../src/components/files/S3Browser.tsx"; + +// Mock the API client +vi.mock("../../src/api/client.ts", () => ({ + apiGet: vi.fn(), +})); + +import { apiGet } from "../../src/api/client.ts"; + +const mockApiGet = vi.mocked(apiGet); + +const MOCK_LIST_RESPONSE = { + success: true, + folders: [ + { name: "year=2024", prefix: "year=2024/" }, + { name: "year=2025", prefix: "year=2025/" }, + ], + files: [ + { name: "test.mcap", key: "test.mcap", size: 1024, last_modified: "2024-01-15T10:30:00Z" }, + ], + breadcrumbs: [], +}; + +const MOCK_SUBFOLDER_RESPONSE = { + success: true, + folders: [ + { name: "month=01", prefix: "year=2024/month=01/" }, + { name: "month=02", prefix: "year=2024/month=02/" }, + ], + files: [], + breadcrumbs: [{ name: "year=2024", prefix: "year=2024/" }], +}; + +describe("S3Browser", () => { + beforeEach(() => { + mockApiGet.mockReset(); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("shows loading spinner initially", () => { + mockApiGet.mockReturnValue(new Promise(() => {})); // Never resolves + render(); + expect(screen.getByText("Loading files...")).toBeInTheDocument(); + }); + + it("renders bucket name and region", async () => { + mockApiGet.mockResolvedValue(MOCK_LIST_RESPONSE); + render(); + + await waitFor(() => { + expect(screen.getAllByText("my-bucket").length).toBeGreaterThanOrEqual(1); + }); + expect(screen.getByText("(us-west-2)")).toBeInTheDocument(); + }); + + it("renders folders and files from the API response", async () => { + mockApiGet.mockResolvedValue(MOCK_LIST_RESPONSE); + render(); + + await waitFor(() => { + expect(screen.getByText("year=2024")).toBeInTheDocument(); + }); + expect(screen.getByText("year=2025")).toBeInTheDocument(); + expect(screen.getByText("test.mcap")).toBeInTheDocument(); + expect(screen.getByText("1.0 KB")).toBeInTheDocument(); + }); + + it("navigates into a folder when clicked", async () => { + const user = userEvent.setup(); + mockApiGet + .mockResolvedValueOnce(MOCK_LIST_RESPONSE) // Initial load + .mockResolvedValueOnce(MOCK_SUBFOLDER_RESPONSE); // After click + + render(); + + await waitFor(() => { + expect(screen.getByText("year=2024")).toBeInTheDocument(); + }); + + await user.click(screen.getByText("year=2024")); + + await waitFor(() => { + expect(screen.getByText("month=01")).toBeInTheDocument(); + }); + expect(screen.getByText("month=02")).toBeInTheDocument(); + }); + + it("shows an error message and retry button on API failure", async () => { + mockApiGet.mockRejectedValue(new Error("Network error")); + render(); + + await waitFor(() => { + expect(screen.getByText("Network error")).toBeInTheDocument(); + }); + expect(screen.getByText("Retry")).toBeInTheDocument(); + }); + + it("shows empty state when no files or folders", async () => { + mockApiGet.mockResolvedValue({ + success: true, + folders: [], + files: [], + breadcrumbs: [], + }); + render(); + + await waitFor(() => { + expect(screen.getByText("No files or folders found at this location.")).toBeInTheDocument(); + }); + }); +}); diff --git a/frontend/tests/components/Stepper.test.tsx b/frontend/tests/components/Stepper.test.tsx new file mode 100644 index 0000000..c554c35 --- /dev/null +++ b/frontend/tests/components/Stepper.test.tsx @@ -0,0 +1,135 @@ +/** + * Tests for the Stepper component. + */ + +import { render, screen, cleanup } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { describe, it, expect, vi, afterEach } from "vitest"; + +import Stepper from "../../src/components/upload/Stepper.tsx"; + +afterEach(() => { + cleanup(); +}); + +describe("Stepper", () => { + it("renders all four steps", () => { + render(); + + expect(screen.getByTestId("step-1")).toBeInTheDocument(); + expect(screen.getByTestId("step-2")).toBeInTheDocument(); + expect(screen.getByTestId("step-3")).toBeInTheDocument(); + expect(screen.getByTestId("step-4")).toBeInTheDocument(); + }); + + it("displays step labels", () => { + render(); + + expect(screen.getByText("Select")).toBeInTheDocument(); + expect(screen.getByText("Review")).toBeInTheDocument(); + expect(screen.getByText("Upload")).toBeInTheDocument(); + expect(screen.getByText("Complete")).toBeInTheDocument(); + }); + + it("marks the current step as active", () => { + render(); + + const step2 = screen.getByTestId("step-2"); + expect(step2).toHaveAttribute("aria-current", "step"); + }); + + it("shows checkmark icon for completed steps", () => { + render(); + + // Steps 1 and 2 should have checkmarks + const checkmarks = screen.getAllByTestId("checkmark-icon"); + expect(checkmarks).toHaveLength(2); + }); + + it("shows numbers for active and future steps", () => { + render(); + + // Step 2 (active) should show "2" + expect(screen.getByTestId("step-2")).toHaveTextContent("2"); + // Step 3 (future) should show "3" + expect(screen.getByTestId("step-3")).toHaveTextContent("3"); + // Step 4 (future) should show "4" + expect(screen.getByTestId("step-4")).toHaveTextContent("4"); + }); + + it("allows clicking completed steps 1 and 2 when not uploading", async () => { + const user = userEvent.setup(); + const onStepClick = vi.fn(); + + render( + , + ); + + // Step 1 (completed) should be clickable + await user.click(screen.getByTestId("step-1")); + expect(onStepClick).toHaveBeenCalledWith(1); + + // Step 2 (completed) should be clickable + await user.click(screen.getByTestId("step-2")); + expect(onStepClick).toHaveBeenCalledWith(2); + }); + + it("disables clicking completed steps when uploading", async () => { + const user = userEvent.setup(); + const onStepClick = vi.fn(); + + render( + , + ); + + // Step 1 should be disabled + const step1 = screen.getByTestId("step-1"); + expect(step1).toBeDisabled(); + + await user.click(step1); + expect(onStepClick).not.toHaveBeenCalled(); + }); + + it("does not allow clicking future steps", async () => { + const user = userEvent.setup(); + const onStepClick = vi.fn(); + + render( + , + ); + + // Step 3 (future) should be disabled + const step3 = screen.getByTestId("step-3"); + expect(step3).toBeDisabled(); + + await user.click(step3); + expect(onStepClick).not.toHaveBeenCalled(); + }); + + it("does not allow clicking the active step", async () => { + const user = userEvent.setup(); + const onStepClick = vi.fn(); + + render( + , + ); + + const step2 = screen.getByTestId("step-2"); + expect(step2).toBeDisabled(); + + await user.click(step2); + expect(onStepClick).not.toHaveBeenCalled(); + }); + + it("step 4 renders at correct position", () => { + render(); + + // All four previous steps should have checkmarks + const checkmarks = screen.getAllByTestId("checkmark-icon"); + expect(checkmarks).toHaveLength(3); // Steps 1, 2, 3 + + // Step 4 is active + const step4 = screen.getByTestId("step-4"); + expect(step4).toHaveAttribute("aria-current", "step"); + }); +}); diff --git a/frontend/tests/components/common.test.tsx b/frontend/tests/components/common.test.tsx new file mode 100644 index 0000000..eac956a --- /dev/null +++ b/frontend/tests/components/common.test.tsx @@ -0,0 +1,225 @@ +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { describe, expect, it, vi } from "vitest"; +import Modal from "../../src/components/common/Modal.tsx"; +import ProgressBar from "../../src/components/common/ProgressBar.tsx"; +import StatCard from "../../src/components/common/StatCard.tsx"; +import Breadcrumb from "../../src/components/common/Breadcrumb.tsx"; +import Spinner from "../../src/components/common/Spinner.tsx"; +import SortableHeader from "../../src/components/common/SortableHeader.tsx"; + +describe("Modal", () => { + it("renders nothing when isOpen is false", () => { + render( + {}} title="Test"> +

body

+
, + ); + expect(screen.queryByText("Test")).not.toBeInTheDocument(); + }); + + it("renders title, body, and footer when open", () => { + render( + {}} title="My Modal" footer={}> +

Hello world

+
, + ); + expect(screen.getByText("My Modal")).toBeInTheDocument(); + expect(screen.getByText("Hello world")).toBeInTheDocument(); + expect(screen.getByText("OK")).toBeInTheDocument(); + }); + + it("calls onClose when Escape is pressed", async () => { + const user = userEvent.setup(); + const handleClose = vi.fn(); + render( + +

content

+
, + ); + + await user.keyboard("{Escape}"); + expect(handleClose).toHaveBeenCalledOnce(); + }); + + it("calls onClose when backdrop is clicked", async () => { + const user = userEvent.setup(); + const handleClose = vi.fn(); + render( + +

content

+
, + ); + + await user.click(screen.getByTestId("modal-backdrop")); + expect(handleClose).toHaveBeenCalledOnce(); + }); + + it("does not call onClose when modal content is clicked", async () => { + const user = userEvent.setup(); + const handleClose = vi.fn(); + render( + +

content

+
, + ); + + await user.click(screen.getByText("content")); + expect(handleClose).not.toHaveBeenCalled(); + }); + + it("calls onClose when close button (X) is clicked", async () => { + const user = userEvent.setup(); + const handleClose = vi.fn(); + render( + +

content

+
, + ); + + await user.click(screen.getByLabelText("Close modal")); + expect(handleClose).toHaveBeenCalledOnce(); + }); +}); + +describe("StatCard", () => { + it("renders value and label", () => { + render(); + expect(screen.getByText("42")).toBeInTheDocument(); + expect(screen.getByText("Total Files")).toBeInTheDocument(); + }); + + it("renders string values", () => { + render(); + expect(screen.getByText("1.5 GB")).toBeInTheDocument(); + expect(screen.getByText("Total Size")).toBeInTheDocument(); + }); + + it("applies default text color to value", () => { + render(); + const value = screen.getByText("0"); + expect(value.className).toContain("text-nlr-blue"); + }); + + it("applies custom text color to value", () => { + render(); + const value = screen.getByText("0"); + expect(value.className).toContain("text-red-500"); + }); +}); + +describe("ProgressBar", () => { + it("shows correct width style", () => { + render(); + const bar = screen.getByRole("progressbar"); + expect(bar).toHaveStyle({ width: "65%" }); + }); + + it("shows percentage text when label is provided", () => { + render(); + expect(screen.getByText("Uploading")).toBeInTheDocument(); + expect(screen.getByText("42%")).toBeInTheDocument(); + }); + + it("clamps percent to 0-100 range", () => { + render(); + const bar = screen.getByRole("progressbar"); + expect(bar).toHaveStyle({ width: "100%" }); + expect(bar.getAttribute("aria-valuenow")).toBe("100"); + }); + + it("clamps negative percent to 0", () => { + render(); + const bar = screen.getByRole("progressbar"); + expect(bar).toHaveStyle({ width: "0%" }); + expect(bar.getAttribute("aria-valuenow")).toBe("0"); + }); + + it("applies default color", () => { + render(); + const bar = screen.getByRole("progressbar"); + expect(bar.className).toContain("bg-nlr-blue"); + }); +}); + +describe("Breadcrumb", () => { + it("renders all items", () => { + render( + {} }, + { label: "Files", onClick: () => {} }, + { label: "Current" }, + ]} + />, + ); + expect(screen.getByText("Home")).toBeInTheDocument(); + expect(screen.getByText("Files")).toBeInTheDocument(); + expect(screen.getByText("Current")).toBeInTheDocument(); + }); + + it("makes the last item non-clickable", () => { + render( + {} }, + { label: "Current" }, + ]} + />, + ); + // Last item should be a span, not a button + expect(screen.getByText("Current").tagName).toBe("SPAN"); + // First item should be a button + expect(screen.getByText("Home").tagName).toBe("BUTTON"); + }); + + it("calls onClick when a breadcrumb item is clicked", async () => { + const user = userEvent.setup(); + const handleClick = vi.fn(); + render( + , + ); + + await user.click(screen.getByText("Home")); + expect(handleClick).toHaveBeenCalledOnce(); + }); +}); + +describe("Spinner", () => { + it("renders without a message", () => { + render(); + expect(screen.getByTestId("spinner")).toBeInTheDocument(); + }); + + it("renders with a message", () => { + render(); + expect(screen.getByText("Loading data...")).toBeInTheDocument(); + }); +}); + +describe("SortableHeader", () => { + it("renders label and calls onSort when clicked", async () => { + const user = userEvent.setup(); + const handleSort = vi.fn(); + const { container } = render( + + + + + + +
, + ); + expect(screen.getByText("Name")).toBeInTheDocument(); + + const th = container.querySelector("th"); + expect(th).not.toBeNull(); + await user.click(th!); + expect(handleSort).toHaveBeenCalledOnce(); + }); +}); diff --git a/frontend/tests/components/layout.test.tsx b/frontend/tests/components/layout.test.tsx new file mode 100644 index 0000000..cdafec8 --- /dev/null +++ b/frontend/tests/components/layout.test.tsx @@ -0,0 +1,112 @@ +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { MemoryRouter } from "react-router-dom"; +import { describe, expect, it, vi, beforeEach } from "vitest"; +import Layout from "../../src/components/layout/Layout.tsx"; +import NavBar from "../../src/components/layout/NavBar.tsx"; +import { useAppStore } from "../../src/stores/appStore.ts"; + +// Mock react-router-dom's Outlet so Layout renders without child routes +vi.mock("react-router-dom", async () => { + const actual = await vi.importActual("react-router-dom"); + return { + ...actual, + Outlet: () =>
page content
, + }; +}); + +function renderWithRouter(ui: React.ReactElement) { + return render({ui}); +} + +describe("Layout", () => { + beforeEach(() => { + useAppStore.setState({ + settings: null, + version: { version: "1.2.3", commit: "abc1234def5678", branch: "main", dirty: false }, + notifications: [], + }); + }); + + it("renders Header, NavBar, Footer, and Outlet", () => { + renderWithRouter(); + + // Header - title defaults to MODAQ Upload when settings are null + expect(screen.getByText("MODAQ Upload")).toBeInTheDocument(); + + // NavBar links + expect(screen.getByText("Upload")).toBeInTheDocument(); + expect(screen.getByText("Browse Uploaded Files")).toBeInTheDocument(); + expect(screen.getByText("History")).toBeInTheDocument(); + expect(screen.getByText("Settings")).toBeInTheDocument(); + + // Footer + expect(screen.getByText("National Laboratory of the Rockies")).toBeInTheDocument(); + + // Outlet (mocked) + expect(screen.getByTestId("outlet")).toBeInTheDocument(); + }); + + it("opens AboutModal when version badge is clicked", async () => { + const user = userEvent.setup(); + renderWithRouter(); + + const badge = screen.getByText("v1.2.3"); + await user.click(badge); + + // Modal should now be open - check for the modal backdrop and version info + expect(screen.getByTestId("modal-backdrop")).toBeInTheDocument(); + expect(screen.getByRole("heading", { name: "About" })).toBeInTheDocument(); + expect(screen.getByText("1.2.3")).toBeInTheDocument(); + expect(screen.getByText("abc1234")).toBeInTheDocument(); + }); + + it("closes AboutModal when close button is clicked", async () => { + const user = userEvent.setup(); + renderWithRouter(); + + await user.click(screen.getByText("v1.2.3")); + expect(screen.getByTestId("modal-backdrop")).toBeInTheDocument(); + + // Click the footer Close button + const closeButtons = screen.getAllByText("Close"); + await user.click(closeButtons[0]); + expect(screen.queryByTestId("modal-backdrop")).not.toBeInTheDocument(); + }); +}); + +describe("NavBar", () => { + beforeEach(() => { + useAppStore.setState({ + version: { version: "2.0.0", commit: "deadbeef", branch: "develop", dirty: false }, + }); + }); + + it("shows all 4 nav links", () => { + renderWithRouter( {}} />); + + expect(screen.getByText("Upload")).toBeInTheDocument(); + expect(screen.getByText("Browse Uploaded Files")).toBeInTheDocument(); + expect(screen.getByText("History")).toBeInTheDocument(); + expect(screen.getByText("Settings")).toBeInTheDocument(); + }); + + it("shows version badge and calls onAboutClick when clicked", async () => { + const user = userEvent.setup(); + const handleAboutClick = vi.fn(); + renderWithRouter(); + + const badge = screen.getByText("v2.0.0"); + expect(badge).toBeInTheDocument(); + + await user.click(badge); + expect(handleAboutClick).toHaveBeenCalledOnce(); + }); + + it("hides version badge when version is not loaded", () => { + useAppStore.setState({ version: null }); + renderWithRouter( {}} />); + + expect(screen.queryByText(/^v/)).not.toBeInTheDocument(); + }); +}); diff --git a/frontend/tests/components/settings.test.tsx b/frontend/tests/components/settings.test.tsx new file mode 100644 index 0000000..62330c5 --- /dev/null +++ b/frontend/tests/components/settings.test.tsx @@ -0,0 +1,82 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { render, screen, waitFor } from "@testing-library/react"; +import SettingsForm from "../../src/components/settings/SettingsForm.tsx"; +import { useAppStore } from "../../src/stores/appStore.ts"; + +// Mock the API client +vi.mock("../../src/api/client.ts", () => ({ + apiGet: vi.fn(), + apiPost: vi.fn(), + apiPut: vi.fn(), +})); + +// Import mocked functions for control +import { apiGet } from "../../src/api/client.ts"; + +const mockApiGet = vi.mocked(apiGet); + +describe("SettingsForm", () => { + beforeEach(() => { + vi.clearAllMocks(); + + // Set up store with settings + useAppStore.setState({ + settings: { + aws_profile: "default", + aws_region: "us-west-2", + s3_bucket: "test-bucket", + default_upload_folder: "/data/mcap", + display_name: "Test App", + log_directory: "logs", + }, + settingsLoading: false, + }); + + // Mock profiles endpoint + mockApiGet.mockResolvedValue({ profiles: ["default", "production"] }); + }); + + it("renders all form fields", async () => { + render(); + + await waitFor(() => { + expect(screen.getByLabelText("AWS Profile")).toBeInTheDocument(); + }); + + expect(screen.getByLabelText("AWS Region")).toBeInTheDocument(); + expect(screen.getByLabelText("S3 Bucket")).toBeInTheDocument(); + expect(screen.getByLabelText("Default Upload Folder")).toBeInTheDocument(); + expect(screen.getByLabelText("Display Name")).toBeInTheDocument(); + expect(screen.getByLabelText("Log Directory")).toBeInTheDocument(); + }); + + it("shows Test Connection and Save Settings buttons", async () => { + render(); + + await waitFor(() => { + expect(screen.getByText("Test Connection")).toBeInTheDocument(); + }); + + expect(screen.getByText("Save Settings")).toBeInTheDocument(); + }); + + it("populates form fields from settings", async () => { + render(); + + await waitFor(() => { + expect(screen.getByLabelText("S3 Bucket")).toHaveValue("test-bucket"); + }); + + expect(screen.getByLabelText("Default Upload Folder")).toHaveValue("/data/mcap"); + expect(screen.getByLabelText("Display Name")).toHaveValue("Test App"); + expect(screen.getByLabelText("Log Directory")).toHaveValue("logs"); + }); + + it("fetches profiles on mount", async () => { + render(); + + await waitFor(() => { + expect(mockApiGet).toHaveBeenCalledWith("/api/settings/profiles"); + }); + }); +}); diff --git a/frontend/tests/hooks/useDebounce.test.ts b/frontend/tests/hooks/useDebounce.test.ts new file mode 100644 index 0000000..ac9ee75 --- /dev/null +++ b/frontend/tests/hooks/useDebounce.test.ts @@ -0,0 +1,91 @@ +import { act, renderHook } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { useDebounce } from "../../src/hooks/useDebounce.ts"; + +describe("useDebounce", () => { + beforeEach(() => { + vi.useFakeTimers(); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it("returns the initial value immediately", () => { + const { result } = renderHook(() => useDebounce("hello", 300)); + expect(result.current).toBe("hello"); + }); + + it("does not update the debounced value before the delay", () => { + const { result, rerender } = renderHook(({ value }) => useDebounce(value, 300), { + initialProps: { value: "hello" }, + }); + + rerender({ value: "world" }); + + act(() => { + vi.advanceTimersByTime(100); + }); + + expect(result.current).toBe("hello"); + }); + + it("updates the debounced value after the delay", () => { + const { result, rerender } = renderHook(({ value }) => useDebounce(value, 300), { + initialProps: { value: "hello" }, + }); + + rerender({ value: "world" }); + + act(() => { + vi.advanceTimersByTime(300); + }); + + expect(result.current).toBe("world"); + }); + + it("uses the default delay of 300ms", () => { + const { result, rerender } = renderHook(({ value }) => useDebounce(value), { + initialProps: { value: "a" }, + }); + + rerender({ value: "b" }); + + act(() => { + vi.advanceTimersByTime(299); + }); + expect(result.current).toBe("a"); + + act(() => { + vi.advanceTimersByTime(1); + }); + expect(result.current).toBe("b"); + }); + + it("resets the timer on rapid changes and only takes the last value", () => { + const { result, rerender } = renderHook(({ value }) => useDebounce(value, 200), { + initialProps: { value: "a" }, + }); + + rerender({ value: "b" }); + act(() => { + vi.advanceTimersByTime(100); + }); + + rerender({ value: "c" }); + act(() => { + vi.advanceTimersByTime(100); + }); + + // Still the initial value since each change reset the timer + expect(result.current).toBe("a"); + + rerender({ value: "d" }); + act(() => { + vi.advanceTimersByTime(200); + }); + + // Now it should be the latest value + expect(result.current).toBe("d"); + }); +}); diff --git a/frontend/tests/hooks/usePagination.test.ts b/frontend/tests/hooks/usePagination.test.ts new file mode 100644 index 0000000..c14eb94 --- /dev/null +++ b/frontend/tests/hooks/usePagination.test.ts @@ -0,0 +1,121 @@ +import { act, renderHook } from "@testing-library/react"; +import { describe, expect, it } from "vitest"; +import { usePagination } from "../../src/hooks/usePagination.ts"; + +describe("usePagination", () => { + it("starts at page 1 with default limit of 100", () => { + const { result } = renderHook(() => usePagination()); + expect(result.current.currentPage).toBe(1); + expect(result.current.limit).toBe(100); + expect(result.current.offset).toBe(0); + expect(result.current.totalPages).toBe(1); + }); + + it("respects custom page size", () => { + const { result } = renderHook(() => usePagination(25)); + expect(result.current.limit).toBe(25); + }); + + it("computes totalPages when setTotal is called", () => { + const { result } = renderHook(() => usePagination(50)); + act(() => { + result.current.setTotal(120); + }); + expect(result.current.totalPages).toBe(3); // ceil(120/50) + }); + + it("navigates with nextPage and prevPage", () => { + const { result } = renderHook(() => usePagination(10)); + act(() => { + result.current.setTotal(50); + }); + expect(result.current.totalPages).toBe(5); + + act(() => { + result.current.nextPage(); + }); + expect(result.current.currentPage).toBe(2); + expect(result.current.offset).toBe(10); + + act(() => { + result.current.nextPage(); + }); + expect(result.current.currentPage).toBe(3); + expect(result.current.offset).toBe(20); + + act(() => { + result.current.prevPage(); + }); + expect(result.current.currentPage).toBe(2); + expect(result.current.offset).toBe(10); + }); + + it("does not go below page 1", () => { + const { result } = renderHook(() => usePagination(10)); + act(() => { + result.current.setTotal(30); + }); + act(() => { + result.current.prevPage(); + }); + expect(result.current.currentPage).toBe(1); + }); + + it("does not go above totalPages", () => { + const { result } = renderHook(() => usePagination(10)); + act(() => { + result.current.setTotal(20); + }); + // totalPages = 2 + act(() => { + result.current.goToPage(5); + }); + expect(result.current.currentPage).toBe(2); + }); + + it("goToPage navigates to the correct page", () => { + const { result } = renderHook(() => usePagination(10)); + act(() => { + result.current.setTotal(100); + }); + act(() => { + result.current.goToPage(7); + }); + expect(result.current.currentPage).toBe(7); + expect(result.current.offset).toBe(60); + }); + + it("clamps current page when total shrinks", () => { + const { result } = renderHook(() => usePagination(10)); + act(() => { + result.current.setTotal(100); + }); + act(() => { + result.current.goToPage(10); + }); + expect(result.current.currentPage).toBe(10); + + act(() => { + result.current.setTotal(30); + }); + // totalPages = 3, so page should clamp to 3 + expect(result.current.currentPage).toBe(3); + }); + + it("reset goes back to page 1", () => { + const { result } = renderHook(() => usePagination(10)); + act(() => { + result.current.setTotal(50); + }); + act(() => { + result.current.goToPage(4); + }); + expect(result.current.currentPage).toBe(4); + + act(() => { + result.current.reset(); + }); + expect(result.current.currentPage).toBe(1); + expect(result.current.offset).toBe(0); + }); +}); diff --git a/frontend/tests/hooks/useSSE.test.ts b/frontend/tests/hooks/useSSE.test.ts new file mode 100644 index 0000000..ffe78b6 --- /dev/null +++ b/frontend/tests/hooks/useSSE.test.ts @@ -0,0 +1,177 @@ +/** + * Tests for the useSSE hook. + */ + +import { renderHook, act, cleanup } from "@testing-library/react"; +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; + +import { useSSE } from "../../src/hooks/useSSE.ts"; + +// Mock EventSource +class MockEventSource { + url: string; + onmessage: ((event: MessageEvent) => void) | null = null; + onerror: ((event: Event) => void) | null = null; + readyState = 0; + closed = false; + + constructor(url: string) { + this.url = url; + MockEventSource.instances.push(this); + } + + close() { + this.closed = true; + this.readyState = 2; + } + + // Simulate a message from the server + simulateMessage(data: unknown) { + if (this.onmessage) { + this.onmessage(new MessageEvent("message", { data: JSON.stringify(data) })); + } + } + + // Simulate an error + simulateError() { + if (this.onerror) { + this.onerror(new Event("error")); + } + } + + static instances: MockEventSource[] = []; + static clear() { + MockEventSource.instances = []; + } +} + +// Install mock +beforeEach(() => { + MockEventSource.clear(); + vi.stubGlobal("EventSource", MockEventSource); +}); + +afterEach(() => { + cleanup(); + vi.restoreAllMocks(); +}); + +describe("useSSE", () => { + it("does not create an EventSource when url is null", () => { + const onMessage = vi.fn(); + renderHook(() => useSSE({ url: null, onMessage })); + + expect(MockEventSource.instances).toHaveLength(0); + }); + + it("creates an EventSource when url is provided", () => { + const onMessage = vi.fn(); + renderHook(() => + useSSE({ url: "/api/upload/progress/test-id", onMessage }), + ); + + expect(MockEventSource.instances).toHaveLength(1); + expect(MockEventSource.instances[0]!.url).toBe( + "/api/upload/progress/test-id", + ); + }); + + it("calls onMessage with parsed JSON data", () => { + const onMessage = vi.fn(); + renderHook(() => + useSSE({ url: "/api/upload/progress/test-id", onMessage }), + ); + + const es = MockEventSource.instances[0]!; + act(() => { + es.simulateMessage({ type: "scan_started", folders_total: 5 }); + }); + + expect(onMessage).toHaveBeenCalledTimes(1); + expect(onMessage).toHaveBeenCalledWith({ + type: "scan_started", + folders_total: 5, + }); + }); + + it("ignores non-JSON messages without throwing", () => { + const onMessage = vi.fn(); + renderHook(() => + useSSE({ url: "/api/upload/progress/test-id", onMessage }), + ); + + const es = MockEventSource.instances[0]!; + // Send raw non-JSON string + act(() => { + if (es.onmessage) { + es.onmessage(new MessageEvent("message", { data: "not-json" })); + } + }); + + expect(onMessage).not.toHaveBeenCalled(); + }); + + it("calls onError and closes on error", () => { + const onMessage = vi.fn(); + const onError = vi.fn(); + renderHook(() => + useSSE({ url: "/api/upload/progress/test-id", onMessage, onError }), + ); + + const es = MockEventSource.instances[0]!; + act(() => { + es.simulateError(); + }); + + expect(onError).toHaveBeenCalledTimes(1); + expect(es.closed).toBe(true); + }); + + it("closes the EventSource on unmount", () => { + const onMessage = vi.fn(); + const { unmount } = renderHook(() => + useSSE({ url: "/api/upload/progress/test-id", onMessage }), + ); + + const es = MockEventSource.instances[0]!; + expect(es.closed).toBe(false); + + unmount(); + + expect(es.closed).toBe(true); + }); + + it("closes old EventSource and opens new one when url changes", () => { + const onMessage = vi.fn(); + const { rerender } = renderHook( + ({ url }: { url: string | null }) => useSSE({ url, onMessage }), + { initialProps: { url: "/api/upload/progress/id-1" } }, + ); + + expect(MockEventSource.instances).toHaveLength(1); + const first = MockEventSource.instances[0]!; + + rerender({ url: "/api/upload/progress/id-2" }); + + expect(first.closed).toBe(true); + expect(MockEventSource.instances).toHaveLength(2); + expect(MockEventSource.instances[1]!.url).toBe( + "/api/upload/progress/id-2", + ); + }); + + it("closes EventSource when url changes to null", () => { + const onMessage = vi.fn(); + const { rerender } = renderHook( + ({ url }: { url: string | null }) => useSSE({ url, onMessage }), + { initialProps: { url: "/api/upload/progress/id-1" as string | null } }, + ); + + const es = MockEventSource.instances[0]!; + expect(es.closed).toBe(false); + + rerender({ url: null }); + + expect(es.closed).toBe(true); + }); +}); diff --git a/frontend/tests/setup.ts b/frontend/tests/setup.ts new file mode 100644 index 0000000..f149f27 --- /dev/null +++ b/frontend/tests/setup.ts @@ -0,0 +1 @@ +import "@testing-library/jest-dom/vitest"; diff --git a/frontend/tests/utils/formatters.test.ts b/frontend/tests/utils/formatters.test.ts new file mode 100644 index 0000000..5d4b4c8 --- /dev/null +++ b/frontend/tests/utils/formatters.test.ts @@ -0,0 +1,110 @@ +import { describe, it, expect } from "vitest"; +import { formatBytes } from "../../src/utils/format/bytes.ts"; +import { formatDate, formatDateTime } from "../../src/utils/format/date.ts"; +import { formatDuration, formatEta } from "../../src/utils/format/time.ts"; + +describe("formatBytes", () => { + it('returns "0 B" for zero bytes', () => { + expect(formatBytes(0)).toBe("0 B"); + }); + + it("formats bytes correctly", () => { + expect(formatBytes(500)).toBe("500 B"); + }); + + it("formats kilobytes", () => { + expect(formatBytes(1024)).toBe("1.00 KB"); + expect(formatBytes(1536)).toBe("1.50 KB"); + }); + + it("formats megabytes", () => { + expect(formatBytes(1048576)).toBe("1.00 MB"); + expect(formatBytes(1572864)).toBe("1.50 MB"); + }); + + it("formats gigabytes", () => { + expect(formatBytes(1073741824)).toBe("1.00 GB"); + }); + + it("formats terabytes", () => { + expect(formatBytes(1099511627776)).toBe("1.00 TB"); + }); +}); + +describe("formatEta", () => { + it('returns "--" for null/undefined', () => { + expect(formatEta(null)).toBe("--"); + expect(formatEta(undefined)).toBe("--"); + }); + + it('returns "--" for negative values', () => { + expect(formatEta(-5)).toBe("--"); + }); + + it("formats seconds", () => { + expect(formatEta(30)).toBe("30s"); + expect(formatEta(1)).toBe("1s"); + }); + + it("formats minutes and seconds", () => { + expect(formatEta(90)).toBe("1m 30s"); + expect(formatEta(125)).toBe("2m 5s"); + }); + + it("formats hours and minutes", () => { + expect(formatEta(3661)).toBe("1h 1m"); + expect(formatEta(7200)).toBe("2h 0m"); + }); +}); + +describe("formatDuration", () => { + it("formats sub-second durations as milliseconds", () => { + expect(formatDuration(0.5)).toBe("500ms"); + expect(formatDuration(0.001)).toBe("1ms"); + }); + + it("formats seconds with one decimal", () => { + expect(formatDuration(5.3)).toBe("5.3s"); + expect(formatDuration(30.0)).toBe("30.0s"); + }); + + it("formats minutes and seconds", () => { + expect(formatDuration(90)).toBe("1m 30s"); + expect(formatDuration(125)).toBe("2m 5s"); + }); +}); + +describe("formatDate", () => { + it("formats a Unix epoch into a locale date string (date only)", () => { + // 2024-01-15 10:30:00 UTC = 1705311000 + const result = formatDate(1705311000); + // Should contain date components (locale-dependent formatting) + expect(result).toContain("Jan"); + expect(result).toContain("15"); + expect(result).toContain("2024"); + // Should NOT contain time + expect(result).not.toContain(":"); + }); +}); + +describe("formatDateTime", () => { + it('returns "-" for null/undefined', () => { + expect(formatDateTime(null)).toBe("-"); + expect(formatDateTime(undefined)).toBe("-"); + }); + + it('returns "-" for zero', () => { + expect(formatDateTime(0)).toBe("-"); + }); + + it("formats a Unix epoch into a locale datetime string", () => { + // 2024-01-15 10:30:00 UTC = 1705311000 + const result = formatDateTime(1705311000); + // Should contain date components (locale-dependent formatting) + expect(result).toContain("Jan"); + expect(result).toContain("15"); + expect(result).toContain("2024"); + // Should also contain time + expect(result).toContain(":"); + }); +}); diff --git a/frontend/vitest.config.ts b/frontend/vitest.config.ts new file mode 100644 index 0000000..738362e --- /dev/null +++ b/frontend/vitest.config.ts @@ -0,0 +1,12 @@ +import react from "@vitejs/plugin-react"; +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + plugins: [react()], + test: { + globals: true, + environment: "jsdom", + setupFiles: ["./tests/setup.ts"], + css: true, + }, +}); From 8fc4c63dc995fbbe2fec34de965690a90f62b1f5 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Wed, 18 Feb 2026 10:33:27 -0700 Subject: [PATCH 116/173] Ignore: Don't ignore nested logs dirs --- .gitignore | 2 +- frontend/.gitignore | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 447eb83..0716996 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,7 @@ settings.json modaq_upload_cache.db # Log files -logs/ +/logs/ # Python __pycache__/ diff --git a/frontend/.gitignore b/frontend/.gitignore index 982baa9..be7b770 100644 --- a/frontend/.gitignore +++ b/frontend/.gitignore @@ -1,5 +1,5 @@ # Logs -logs +/logs *.log npm-debug.log* yarn-debug.log* From 74d17f79ae4ddcecf4055b9aa1d63a1aa4729508 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Wed, 18 Feb 2026 10:35:10 -0700 Subject: [PATCH 117/173] Frontend: Add CSV Preview component --- frontend/src/components/logs/CsvPreview.tsx | 182 ++++++++++++++++++++ 1 file changed, 182 insertions(+) create mode 100644 frontend/src/components/logs/CsvPreview.tsx diff --git a/frontend/src/components/logs/CsvPreview.tsx b/frontend/src/components/logs/CsvPreview.tsx new file mode 100644 index 0000000..523e188 --- /dev/null +++ b/frontend/src/components/logs/CsvPreview.tsx @@ -0,0 +1,182 @@ +import { useCallback, useState } from "react"; +import { apiGet } from "../../api/client.ts"; +import type { CsvFileInfo, CsvPreviewResponse } from "../../types/api.ts"; + +function formatBytes(bytes: number): string { + if (bytes === 0) return "0 B"; + const units = ["B", "KB", "MB", "GB", "TB"]; + const i = Math.floor(Math.log(bytes) / Math.log(1024)); + return `${(bytes / 1024 ** i).toFixed(i === 0 ? 0 : 1)} ${units[i]}`; +} + +interface CsvPreviewProps { + csvFiles: CsvFileInfo[]; +} + +export default function CsvPreview({ csvFiles }: CsvPreviewProps) { + const [expanded, setExpanded] = useState(false); + const [previewPath, setPreviewPath] = useState(null); + const [previewData, setPreviewData] = useState(null); + const [previewLoading, setPreviewLoading] = useState(false); + const [previewError, setPreviewError] = useState(null); + + const loadPreview = useCallback(async (path: string) => { + // Toggle off if clicking same file + if (previewPath === path) { + setPreviewPath(null); + setPreviewData(null); + return; + } + + setPreviewPath(path); + setPreviewLoading(true); + setPreviewError(null); + try { + const data = await apiGet("/api/logs/csv-preview", { path }); + setPreviewData(data); + } catch (err) { + setPreviewError(err instanceof Error ? err.message : "Failed to load preview"); + } finally { + setPreviewLoading(false); + } + }, [previewPath]); + + if (csvFiles.length === 0) { + return null; + } + + return ( +
+ {/* Collapsible header */} + + + {expanded && ( +
+ {/* CSV file list */} +
+ {csvFiles.map((file) => ( +
+
+ + + +
+

{file.filename}

+

+ {file.date} -- {formatBytes(file.size)} +

+
+
+ + + Download + +
+
+ + {/* Inline preview */} + {previewPath === file.path && ( +
+ {previewLoading && ( +
+ Loading preview... +
+ )} + {previewError && ( +
{previewError}
+ )} + {previewData && !previewLoading && ( +
+ + + + {previewData.columns.map((col) => ( + + ))} + + + + {previewData.rows.map((row, i) => ( + + {previewData.columns.map((col) => ( + + ))} + + ))} + +
+ {col} +
+ {row[col] ?? ""} +
+
+ )} +
+ )} +
+ ))} +
+
+ )} +
+ ); +} From 8669060414c8ac949df2b025e59ae50eede302e5 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Wed, 18 Feb 2026 10:35:27 -0700 Subject: [PATCH 118/173] Frontend: Add logs filter bar component --- frontend/src/components/logs/FilterBar.tsx | 128 +++++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 frontend/src/components/logs/FilterBar.tsx diff --git a/frontend/src/components/logs/FilterBar.tsx b/frontend/src/components/logs/FilterBar.tsx new file mode 100644 index 0000000..013bfcb --- /dev/null +++ b/frontend/src/components/logs/FilterBar.tsx @@ -0,0 +1,128 @@ +import { useCallback, useEffect, useState } from "react"; +import { useDebounce } from "../../hooks/useDebounce.ts"; + +export interface LogFilters { + date: string; + level: string; + category: string; + search: string; +} + +const EMPTY_FILTERS: LogFilters = { + date: "", + level: "", + category: "", + search: "", +}; + +const LEVELS = ["All", "INFO", "WARNING", "ERROR"] as const; +const CATEGORIES = ["All", "upload", "analysis", "settings", "app", "sync"] as const; + +interface FilterBarProps { + onFilterChange: (filters: LogFilters) => void; +} + +export default function FilterBar({ onFilterChange }: FilterBarProps) { + const [filters, setFilters] = useState(EMPTY_FILTERS); + const debouncedSearch = useDebounce(filters.search, 300); + + // Notify parent when any filter changes (debounced for search) + const { date, level, category } = filters; + useEffect(() => { + onFilterChange({ date, level, category, search: debouncedSearch }); + }, [date, level, category, debouncedSearch, onFilterChange]); + + const updateFilter = useCallback((key: K, value: LogFilters[K]) => { + setFilters((prev) => ({ ...prev, [key]: value })); + }, []); + + const clearFilters = () => { + setFilters(EMPTY_FILTERS); + }; + + const hasActiveFilters = + filters.date !== "" || filters.level !== "" || filters.category !== "" || filters.search !== ""; + + return ( +
+
+ {/* Date picker */} +
+ + updateFilter("date", e.target.value)} + className="px-3 py-2 border border-gray-300 rounded text-sm focus:outline-none focus:ring-2 focus:ring-nlr-blue focus:border-transparent" + /> +
+ + {/* Level dropdown */} +
+ + +
+ + {/* Category dropdown */} +
+ + +
+ + {/* Search input */} +
+ + updateFilter("search", e.target.value)} + placeholder="Search messages..." + className="px-3 py-2 border border-gray-300 rounded text-sm focus:outline-none focus:ring-2 focus:ring-nlr-blue focus:border-transparent" + /> +
+ + {/* Clear button */} + {hasActiveFilters && ( + + )} +
+
+ ); +} From d69e04c8df0962f161c0717e09669924cadb2430 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Wed, 18 Feb 2026 10:35:49 -0700 Subject: [PATCH 119/173] Frontend: Add logs stats bar component --- frontend/src/components/logs/LogStatsBar.tsx | 72 ++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 frontend/src/components/logs/LogStatsBar.tsx diff --git a/frontend/src/components/logs/LogStatsBar.tsx b/frontend/src/components/logs/LogStatsBar.tsx new file mode 100644 index 0000000..a48e7c5 --- /dev/null +++ b/frontend/src/components/logs/LogStatsBar.tsx @@ -0,0 +1,72 @@ +import { useEffect, useState } from "react"; +import { apiGet } from "../../api/client.ts"; +import type { LogStats } from "../../types/api.ts"; +import StatCard from "../common/StatCard.tsx"; + +interface LogStatsBarProps { + onStatsLoaded?: (stats: LogStats) => void; +} + +export default function LogStatsBar({ onStatsLoaded }: LogStatsBarProps) { + const [stats, setStats] = useState(null); + const [error, setError] = useState(null); + + useEffect(() => { + async function fetchStats() { + try { + const data = await apiGet("/api/logs/stats"); + setStats(data); + onStatsLoaded?.(data); + } catch (err) { + setError(err instanceof Error ? err.message : "Failed to load stats"); + } + } + void fetchStats(); + }, [onStatsLoaded]); + + if (error) { + return ( +
+ {error} +
+ ); + } + + if (!stats) { + return ( +
+ {Array.from({ length: 4 }).map((_, i) => ( +
+
+
+
+ ))} +
+ ); + } + + return ( +
+ + + + +
+ ); +} From 68e6d906e666df0109cd3c830476e46e7a32334b Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Wed, 18 Feb 2026 10:36:16 -0700 Subject: [PATCH 120/173] Frontend: Add logs table component --- frontend/src/components/logs/LogTable.tsx | 265 ++++++++++++++++++++++ 1 file changed, 265 insertions(+) create mode 100644 frontend/src/components/logs/LogTable.tsx diff --git a/frontend/src/components/logs/LogTable.tsx b/frontend/src/components/logs/LogTable.tsx new file mode 100644 index 0000000..a9d6d50 --- /dev/null +++ b/frontend/src/components/logs/LogTable.tsx @@ -0,0 +1,265 @@ +import { useCallback, useEffect, useState } from "react"; +import { apiGet } from "../../api/client.ts"; +import { usePagination } from "../../hooks/usePagination.ts"; +import type { LogEntriesResponse, LogEntry } from "../../types/api.ts"; +import SortableHeader from "../common/SortableHeader.tsx"; +import type { LogFilters } from "./FilterBar.tsx"; + +const LEVEL_BADGES: Record = { + INFO: "bg-blue-100 text-blue-800", + WARNING: "bg-yellow-100 text-yellow-800", + ERROR: "bg-red-100 text-red-800", +}; + +function LevelBadge({ level }: { level: string }) { + const classes = LEVEL_BADGES[level] ?? "bg-gray-100 text-gray-800"; + return ( + + {level} + + ); +} + +function formatTimestamp(iso: string): string { + const d = new Date(iso); + return d.toLocaleString(undefined, { + month: "short", + day: "numeric", + hour: "2-digit", + minute: "2-digit", + second: "2-digit", + }); +} + +type SortColumn = "timestamp" | "level" | "category" | "event"; + +interface LogTableProps { + filters: LogFilters; +} + +export default function LogTable({ filters }: LogTableProps) { + const [entries, setEntries] = useState([]); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + const [expandedId, setExpandedId] = useState(null); + const [sortColumn, setSortColumn] = useState("timestamp"); + const [ascending, setAscending] = useState(false); + + const pagination = usePagination(50); + const { offset, limit, setTotal, reset: paginationReset } = pagination; + + const toggleSort = useCallback( + (column: SortColumn) => { + if (column === sortColumn) { + setAscending((prev) => !prev); + } else { + setSortColumn(column); + setAscending(column === "timestamp" ? false : true); + } + }, + [sortColumn], + ); + + const fetchEntries = useCallback(async () => { + setLoading(true); + setError(null); + try { + const params: Record = { + offset: String(offset), + limit: String(limit), + }; + if (filters.date) params.date = filters.date; + if (filters.level) params.level = filters.level; + if (filters.category) params.category = filters.category; + if (filters.search) params.search = filters.search; + + const data = await apiGet("/api/logs/entries", params); + setEntries(data.entries); + setTotal(data.total); + } catch (err) { + setError(err instanceof Error ? err.message : "Failed to load log entries"); + } finally { + setLoading(false); + } + }, [filters, offset, limit, setTotal]); + + // Reset to page 1 when filters change + useEffect(() => { + paginationReset(); + }, [filters.date, filters.level, filters.category, filters.search, paginationReset]); + + useEffect(() => { + void fetchEntries(); + }, [fetchEntries]); + + // Client-side sort of the current page + const sortedEntries = [...entries].sort((a, b) => { + const aVal = a[sortColumn]; + const bVal = b[sortColumn]; + let cmp = 0; + if (typeof aVal === "string" && typeof bVal === "string") { + cmp = aVal.localeCompare(bVal); + } + return ascending ? cmp : -cmp; + }); + + const toggleExpand = (id: number) => { + setExpandedId((prev) => (prev === id ? null : id)); + }; + + return ( +
+ {/* Table */} +
+ + + + toggleSort("timestamp")} + /> + toggleSort("level")} + /> + toggleSort("category")} + /> + toggleSort("event")} + /> + + + + + {loading && entries.length === 0 ? ( + + + + ) : error ? ( + + + + ) : sortedEntries.length === 0 ? ( + + + + ) : ( + sortedEntries.map((entry) => ( + toggleExpand(entry.id)} + /> + )) + )} + +
+ Message +
+ Loading log entries... +
+ {error} +
+ No log entries found. +
+
+ + {/* Pagination */} + {pagination.totalPages > 1 && ( +
+ + Page {pagination.currentPage} of {pagination.totalPages} + +
+ + +
+
+ )} +
+ ); +} + +/** Single log row with expandable metadata */ +function LogRow({ + entry, + expanded, + onToggle, +}: { + entry: LogEntry; + expanded: boolean; + onToggle: () => void; +}) { + const hasMetadata = entry.metadata && Object.keys(entry.metadata).length > 0; + + return ( + <> + + + {formatTimestamp(entry.timestamp)} + + + + + {entry.category} + {entry.event} + +
+ {entry.message} + {hasMetadata && ( + + + + )} +
+ + + {expanded && hasMetadata && ( + + +
+              {JSON.stringify(entry.metadata, null, 2)}
+            
+ + + )} + + ); +} From 93a26a42ae4524bf64d7b9138a1bfb43b857539a Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Wed, 18 Feb 2026 10:36:31 -0700 Subject: [PATCH 121/173] Frontend: Add Upload List component --- .../src/components/logs/UploadSessionList.tsx | 180 ++++++++++++++++++ 1 file changed, 180 insertions(+) create mode 100644 frontend/src/components/logs/UploadSessionList.tsx diff --git a/frontend/src/components/logs/UploadSessionList.tsx b/frontend/src/components/logs/UploadSessionList.tsx new file mode 100644 index 0000000..6275f3c --- /dev/null +++ b/frontend/src/components/logs/UploadSessionList.tsx @@ -0,0 +1,180 @@ +import { useState } from "react"; +import type { UploadSession, UploadSessionFile } from "../../types/api.ts"; + +interface UploadSessionListProps { + sessions: UploadSession[]; +} + +function StatusBadge({ status }: { status: string }) { + const colors: Record = { + completed: "bg-green-100 text-green-700", + skipped: "bg-gray-100 text-gray-600", + failed: "bg-red-100 text-red-700", + }; + return ( + + {status} + + ); +} + +function SessionStatusSummary({ session }: { session: UploadSession }) { + const parts: { text: string; color: string }[] = []; + if (session.completed > 0) parts.push({ text: `${session.completed} completed`, color: "text-green-600" }); + if (session.failed > 0) parts.push({ text: `${session.failed} failed`, color: "text-red-600" }); + if (session.skipped > 0) parts.push({ text: `${session.skipped} skipped`, color: "text-gray-500" }); + + return ( + + {parts.map((p, i) => ( + + {i > 0 && , } + {p.text} + + ))} + + ); +} + +function formatDuration(seconds: number): string { + if (seconds < 60) return `${Math.round(seconds)}s`; + const m = Math.floor(seconds / 60); + const s = Math.round(seconds % 60); + return s > 0 ? `${m}m ${s}s` : `${m}m`; +} + +function FileDetailTable({ files }: { files: UploadSessionFile[] }) { + return ( +
+ + + + + + + + + + + + {files.map((file) => ( + + + + + + + + ))} + {/* Show error rows separately for failed files */} + {files + .filter((f) => f.status === "failed" && f.error_message) + .map((f) => ( + + + + ))} + +
FilenameSizeStatusSpeedS3 Path
{file.filename}{file.file_size_formatted} + {file.upload_speed_mbps ? `${file.upload_speed_mbps} Mbps` : "-"} + + {file.s3_path || "-"} +
+ {f.filename}: {f.error_message} +
+
+ ); +} + +export default function UploadSessionList({ sessions }: UploadSessionListProps) { + const [expandedIndex, setExpandedIndex] = useState(null); + + if (sessions.length === 0) { + return ( +
+ No upload sessions found. Upload some files to see history here. +
+ ); + } + + return ( +
+ {sessions.map((session, i) => { + const isExpanded = expandedIndex === i; + return ( +
+ {/* Session header row */} + + + {/* Expanded file detail */} + {isExpanded && ( +
+ +
+ )} +
+ ); + })} +
+ ); +} From 877ea27a7646489334bbccdd4f3f628d1e8b51e2 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Wed, 18 Feb 2026 10:36:49 -0700 Subject: [PATCH 122/173] Frontend: Add upload stats bar --- .../src/components/logs/UploadStatsBar.tsx | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 frontend/src/components/logs/UploadStatsBar.tsx diff --git a/frontend/src/components/logs/UploadStatsBar.tsx b/frontend/src/components/logs/UploadStatsBar.tsx new file mode 100644 index 0000000..1b434c1 --- /dev/null +++ b/frontend/src/components/logs/UploadStatsBar.tsx @@ -0,0 +1,72 @@ +import { useEffect, useState } from "react"; +import { apiGet } from "../../api/client.ts"; +import type { UploadStatsResponse } from "../../types/api.ts"; +import StatCard from "../common/StatCard.tsx"; + +interface UploadStatsBarProps { + onDataLoaded?: (data: UploadStatsResponse) => void; +} + +export default function UploadStatsBar({ onDataLoaded }: UploadStatsBarProps) { + const [stats, setStats] = useState(null); + const [error, setError] = useState(null); + + useEffect(() => { + async function fetchStats() { + try { + const data = await apiGet("/api/logs/upload-stats"); + setStats(data); + onDataLoaded?.(data); + } catch (err) { + setError(err instanceof Error ? err.message : "Failed to load upload stats"); + } + } + void fetchStats(); + }, [onDataLoaded]); + + if (error) { + return ( +
+ {error} +
+ ); + } + + if (!stats) { + return ( +
+ {Array.from({ length: 4 }).map((_, i) => ( +
+
+
+
+ ))} +
+ ); + } + + return ( +
+ + + + +
+ ); +} From df4e0741987af24453d6ed90a60ae58a2e783538 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Wed, 18 Feb 2026 10:45:28 -0700 Subject: [PATCH 123/173] Backend: Use os.walk for error tolerant file tree walking --- app/routes/files.py | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/app/routes/files.py b/app/routes/files.py index 61b9d78..f7efeef 100644 --- a/app/routes/files.py +++ b/app/routes/files.py @@ -1,5 +1,6 @@ """File browsing API routes for modaq_upload""" +import os from pathlib import Path from flask import Blueprint, Response, g, jsonify, request @@ -142,7 +143,9 @@ def browse_local() -> tuple[Response, int]: if not path.is_dir(): return jsonify({"error": f"Not a directory: {path}"}), 400 - # Build response — single-pass rglob for recursive MCAP counts + cache checks + # Build response — single-pass walk for recursive MCAP counts + cache checks. + # os.walk with onerror skips unreadable subdirectories instead of aborting, + # which is important on Linux where permission errors are common. cache = get_cache_service() bucket = g.settings.s3_bucket @@ -152,23 +155,30 @@ def browse_local() -> tuple[Response, int]: mcap_count = 0 direct_uploaded = 0 - try: - for mcap_path in path.rglob("*.mcap"): - # Skip hidden paths (any component starting with .) - if any(part.startswith(".") for part in mcap_path.relative_to(path).parts): + def _walk_error(err: OSError) -> None: + pass # Skip unreadable directories silently + + for dirpath, dirnames, filenames in os.walk(str(path), onerror=_walk_error): + # Skip hidden directories in-place so os.walk won't descend into them + dirnames[:] = [d for d in dirnames if not d.startswith(".")] + + for fname in filenames: + if not fname.endswith(".mcap") or fname.startswith("."): continue - # Check cache for already-uploaded status + mcap_path = Path(dirpath) / fname + rel = mcap_path.relative_to(path) + parts = rel.parts + try: file_stat = mcap_path.stat() - uploaded = cache.check_exists_by_filename( - bucket, mcap_path.name, file_stat.st_size - ) is True - except PermissionError: + except OSError: continue - rel = mcap_path.relative_to(path) - parts = rel.parts + uploaded = cache.check_exists_by_filename( + bucket, mcap_path.name, file_stat.st_size + ) is True + if len(parts) == 1: # Direct child MCAP file mcap_count += 1 @@ -191,8 +201,6 @@ def browse_local() -> tuple[Response, int]: folder_uploaded_counts[folder_name] = ( folder_uploaded_counts.get(folder_name, 0) + 1 ) - except PermissionError: - return jsonify({"error": f"Permission denied: {path}"}), 403 # Build folder list from direct children (non-hidden directories) folders: list[dict[str, str | int]] = [] @@ -237,7 +245,7 @@ def browse_local() -> tuple[Response, int]: volumes_path = Path("/Volumes") if volumes_path.exists(): try: - for vol in volumes_path.iterdir(): + for vol in sorted(volumes_path.iterdir(), key=lambda x: x.name.lower()): if vol.is_dir() and not vol.name.startswith("."): quick_links.append({"name": vol.name, "path": str(vol)}) except PermissionError: From 270b5fda562a6763c1b0a53f149b4efe1b2c79eb Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Wed, 18 Feb 2026 10:45:54 -0700 Subject: [PATCH 124/173] Backend: Add /media/m2 and /media to preset paths --- app/routes/files.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/app/routes/files.py b/app/routes/files.py index f7efeef..067e784 100644 --- a/app/routes/files.py +++ b/app/routes/files.py @@ -251,6 +251,40 @@ def _walk_error(err: OSError) -> None: except PermissionError: pass + # Add /media/m2 as a priority quick link on Linux if it exists + media_m2 = Path("/media/m2") + if media_m2.exists() and media_m2.is_dir(): + quick_links.append({"name": "m2", "path": str(media_m2)}) + + # Add /media itself and its subdirectories on Linux (removable drives, USB, etc.) + media_path = Path("/media") + if media_path.exists() and not volumes_path.exists(): + quick_links.append({"name": "media", "path": str(media_path)}) + try: + for entry in sorted(media_path.iterdir(), key=lambda x: x.name.lower()): + if not entry.is_dir() or entry.name.startswith("."): + continue + entry_str = str(entry) + # Already added /media/m2 above + if entry_str == str(media_m2): + continue + # If it's a user directory (e.g. /media/username), list its children + try: + children = [ + c + for c in entry.iterdir() + if c.is_dir() and not c.name.startswith(".") + ] + except PermissionError: + children = [] + if children: + for child in sorted(children, key=lambda x: x.name.lower()): + quick_links.append({"name": child.name, "path": str(child)}) + else: + quick_links.append({"name": entry.name, "path": entry_str}) + except PermissionError: + pass + return jsonify( { "success": True, From 1b63c27db808fe37cda2fa3e1f98ef3403952c20 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Wed, 18 Feb 2026 11:31:14 -0700 Subject: [PATCH 125/173] Refactor: Validate and upload file by file --- app/routes/upload.py | 73 ++--- app/services/upload_manager.py | 502 +++++++++++++++++++++++++---- frontend/src/hooks/useUploadJob.ts | 7 +- 3 files changed, 479 insertions(+), 103 deletions(-) diff --git a/app/routes/upload.py b/app/routes/upload.py index f736be9..e93daa4 100644 --- a/app/routes/upload.py +++ b/app/routes/upload.py @@ -483,10 +483,12 @@ def run_scan() -> None: thread = threading.Thread(target=run_scan, daemon=True) thread.start() - return jsonify({ - "job_id": scan_job.job_id, - "status": "scanning", - }), 202 + return jsonify( + { + "job_id": scan_job.job_id, + "status": "scanning", + } + ), 202 @upload_bp.route("/bulk-analyze", methods=["POST"]) @@ -551,50 +553,41 @@ def upload_progress_callback(job: UploadJob) -> None: else: send_sse_event(job.job_id, job.to_progress_dict()) - # Start analysis in background thread - def run_bulk_analysis() -> None: - manager.analyze_job_async( - job.job_id, - settings.aws_profile, - settings.aws_region, - settings.s3_bucket, - progress_callback=analysis_progress_callback, - ) - - # Send analysis complete event - final_job = manager.get_job(job.job_id) - if final_job: - send_sse_event( + # Start in background thread + def run_bulk_job() -> None: + if auto_upload: + # Pipeline: analyze each file and upload immediately as it's ready. + # Uploads start flowing while remaining files are still being parsed. + manager.analyze_and_upload_pipeline( job.job_id, - { - "type": "analysis_complete", - "job": final_job.to_dict(), - "auto_upload": final_job.auto_upload, - }, + settings.aws_profile, + settings.aws_region, + settings.s3_bucket, + skip_duplicates=skip_duplicates, + analysis_callback=analysis_progress_callback, + upload_callback=upload_progress_callback, ) - - # Auto-upload if enabled and analysis succeeded - if final_job.auto_upload and final_job.status == UploadStatus.READY: + else: + # Analysis only — user will review results before starting upload. + manager.analyze_job_async( + job.job_id, + settings.aws_profile, + settings.aws_region, + settings.s3_bucket, + progress_callback=analysis_progress_callback, + ) + final_job = manager.get_job(job.job_id) + if final_job: send_sse_event( job.job_id, { - "type": "auto_upload_starting", - "job_id": job.job_id, + "type": "analysis_complete", + "job": final_job.to_dict(), + "auto_upload": False, }, ) - manager.start_upload( - job.job_id, - settings.aws_profile, - settings.aws_region, - settings.s3_bucket, - skip_duplicates=skip_duplicates, - progress_callback=upload_progress_callback, - ) - elif final_job.auto_upload: - # All files failed analysis — send terminal status so frontend doesn't hang - send_sse_event(job.job_id, final_job.to_dict()) - thread = threading.Thread(target=run_bulk_analysis, daemon=True) + thread = threading.Thread(target=run_bulk_job, daemon=True) thread.start() return jsonify( diff --git a/app/services/upload_manager.py b/app/services/upload_manager.py index 0057666..b98eb0a 100644 --- a/app/services/upload_manager.py +++ b/app/services/upload_manager.py @@ -442,9 +442,7 @@ def _check_duplicate( if cache_result is not None: file_state.is_duplicate = cache_result else: - file_state.is_duplicate = s3_service.check_file_exists( - s3_client, s3_bucket, s3_path - ) + file_state.is_duplicate = s3_service.check_file_exists(s3_client, s3_bucket, s3_path) if use_cache: cache = get_cache_service() cache.update_cache( @@ -606,9 +604,7 @@ def analyze_job_async( with ProcessPoolExecutor(max_workers=cpu_workers) as proc_executor: parse_futures = { - proc_executor.submit( - _extract_start_time_worker, file_state.local_path - ): file_state + proc_executor.submit(_extract_start_time_worker, file_state.local_path): file_state for file_state in job.files } for future in as_completed(parse_futures): @@ -931,9 +927,357 @@ def byte_callback(uploaded: int, total: int) -> None: # Save per-job JSONL summary try: completed_at = job.completed_at or datetime.now(UTC) - log.save_job_jsonl(job_id, { - "timestamp": completed_at.isoformat(), - "event": "upload_job_completed", + log.save_job_jsonl( + job_id, + { + "timestamp": completed_at.isoformat(), + "event": "upload_job_completed", + "job_id": job_id, + "status": job.status.value, + "uploaded": uploaded_count, + "skipped": skipped_count, + "failed": failed_count, + "total_bytes_uploaded": job.successfully_uploaded_bytes, + "duration_seconds": job.total_upload_duration_seconds, + "avg_speed_mbps": job.average_upload_speed_mbps, + "files": file_summary, + }, + completed_at, + ) + except Exception: + logger.warning("Failed to save job JSONL summary", exc_info=True) + + # Save upload summary CSV + try: + log.save_job_csv(job_id, job, completed_at) + except Exception: + logger.warning("Failed to save job CSV summary", exc_info=True) + + # Auto-sync logs to S3 after job completion + try: + log.sync_logs_to_s3(s3_client, s3_bucket) + except Exception: + logger.debug("Log sync to S3 failed", exc_info=True) + + if progress_callback: + progress_callback(job) + + def analyze_and_upload_pipeline( + self, + job_id: str, + aws_profile: str, + aws_region: str, + s3_bucket: str, + skip_duplicates: bool = True, + analysis_callback: Callable[["UploadJob", FileUploadState], None] | None = None, + upload_callback: Callable[["UploadJob"], None] | None = None, + use_cache: bool = True, + ) -> None: + """Analyze each file and upload it immediately — pipeline approach. + + Instead of analyzing all files first and then uploading, this processes + files through a pipeline: MCAP parsing runs in a ProcessPoolExecutor, + and as each parse completes the file is immediately checked for duplicates + and submitted to a ThreadPoolExecutor for upload. + + Args: + job_id: The job ID to process + aws_profile: AWS profile to use + aws_region: AWS region + s3_bucket: S3 bucket to upload to + skip_duplicates: Whether to skip files that already exist + analysis_callback: Called after each file is analyzed + upload_callback: Called for upload progress updates + use_cache: Whether to use cache for duplicate checking + """ + log = get_log_service() + job = self.get_job(job_id) + if not job: + return + + job.status = UploadStatus.UPLOADING + job.started_at = datetime.now(UTC) + + log.info( + "upload", + "pipeline_started", + f"Starting analyze-and-upload pipeline for {len(job.files)} files", + {"job_id": job_id, "total_files": len(job.files)}, + ) + + # Create S3 client + try: + s3_client = s3_service.create_s3_client(aws_profile, aws_region) + except Exception as e: + job.status = UploadStatus.FAILED + for file_state in job.files: + file_state.status = UploadStatus.FAILED + file_state.error_message = f"Failed to create S3 client: {e}" + if analysis_callback: + analysis_callback(job, file_state) + if upload_callback: + upload_callback(job) + return + + cpu_workers = max(1, (os.cpu_count() or 4) - 1) + upload_executor = ThreadPoolExecutor(max_workers=self.max_workers) + + # Set all files to ANALYZING + for fs in job.files: + fs.status = UploadStatus.ANALYZING + + try: + # Submit all files for MCAP parsing (CPU-bound, true parallelism) + with ProcessPoolExecutor(max_workers=cpu_workers) as proc_executor: + parse_futures = { + proc_executor.submit(_extract_start_time_worker, fs.local_path): fs + for fs in job.files + } + + for future in as_completed(parse_futures): + if job.cancelled: + break + + fs = parse_futures[future] + result = future.result() + + if isinstance(result, str): + # Parse failed + fs.status = UploadStatus.FAILED + fs.error_message = result + log.error( + "analysis", + "file_analysis_failed", + f"Failed to analyze {fs.filename}: {result}", + {"job_id": job_id, "filename": fs.filename, "error": result}, + ) + if analysis_callback: + analysis_callback(job, fs) + continue + + # Parse succeeded — set timestamp and generate S3 path + fs.start_time = result + naive_start = mcap_service.to_naive_utc(result) + fs.is_valid = naive_start >= EPOCH_CUTOFF.replace(tzinfo=None) + fs.s3_path = mcap_service.generate_s3_path(result, fs.filename) + + # Check duplicate (I/O but fast — cache lookup or S3 HEAD) + self._check_duplicate(fs, s3_client, s3_bucket, use_cache) + fs.status = UploadStatus.READY + + log.info( + "analysis", + "file_analysis_completed", + f"Analyzed {fs.filename}", + { + "job_id": job_id, + "filename": fs.filename, + "file_size": fs.file_size, + "s3_path": fs.s3_path, + "is_duplicate": fs.is_duplicate, + "is_valid": fs.is_valid, + }, + ) + + # Notify frontend of analysis result + if analysis_callback: + analysis_callback(job, fs) + + # Decide: skip or upload? + if not fs.is_valid: + fs.status = UploadStatus.SKIPPED + fs.error_message = "Invalid timestamp (pre-1980)" + log.warning( + "upload", + "file_upload_skipped", + f"Skipped invalid timestamp: {fs.filename}", + { + "job_id": job_id, + "filename": fs.filename, + "reason": "invalid_timestamp", + }, + ) + if upload_callback: + upload_callback(job) + continue + + if skip_duplicates and fs.is_duplicate: + fs.status = UploadStatus.SKIPPED + fs.bytes_uploaded = fs.file_size + log.info( + "upload", + "file_upload_skipped", + f"Skipped duplicate: {fs.filename}", + { + "job_id": job_id, + "filename": fs.filename, + "reason": "duplicate", + }, + ) + if upload_callback: + upload_callback(job) + continue + + # Submit for upload immediately + def make_upload_task( + file_state: FileUploadState, + ) -> Callable[[], Any]: + def upload_task() -> Any: + if job.cancelled: + with job.lock: + file_state.status = UploadStatus.CANCELLED + return None + + try: + with job.lock: + file_state.status = UploadStatus.UPLOADING + file_state.upload_started_at = datetime.now(UTC) + log.info( + "upload", + "file_upload_started", + f"Uploading {file_state.filename}", + { + "job_id": job_id, + "filename": file_state.filename, + "file_size": file_state.file_size, + "s3_path": file_state.s3_path, + }, + ) + if upload_callback: + upload_callback(job) + + def byte_callback(uploaded: int, total: int) -> None: + with job.lock: + file_state.bytes_uploaded = uploaded + if upload_callback: + upload_callback(job) + + upload_result = s3_service.upload_file_with_progress( + s3_client, + file_state.local_path, + s3_bucket, + file_state.s3_path, + byte_callback, + ) + + # Handle completion inline + file_state.upload_completed_at = datetime.now(UTC) + if upload_result["success"]: + file_state.status = UploadStatus.COMPLETED + file_state.bytes_uploaded = file_state.file_size + log.info( + "upload", + "file_upload_completed", + f"Uploaded {file_state.filename}", + { + "job_id": job_id, + "filename": file_state.filename, + "file_size": file_state.file_size, + "upload_duration_seconds": ( + file_state.upload_duration_seconds + ), + "s3_path": file_state.s3_path, + }, + ) + try: + cache = get_cache_service() + cache.update_cache( + s3_bucket, + file_state.s3_path, + exists=True, + filename=file_state.filename, + file_size=file_state.file_size, + ) + except Exception: + logger.debug( + "Cache update failed after upload", + exc_info=True, + ) + else: + file_state.status = UploadStatus.FAILED + file_state.error_message = upload_result.get( + "error", "Unknown error" + ) + log.error( + "upload", + "file_upload_failed", + f"Failed to upload {file_state.filename}: " + f"{file_state.error_message}", + { + "job_id": job_id, + "filename": file_state.filename, + "error": file_state.error_message, + }, + ) + except Exception as e: + file_state.upload_completed_at = datetime.now(UTC) + file_state.status = UploadStatus.FAILED + file_state.error_message = str(e) + log.error( + "upload", + "file_upload_failed", + f"Failed to upload {file_state.filename}: {e}", + { + "job_id": job_id, + "filename": file_state.filename, + "error": str(e), + }, + ) + + if upload_callback: + upload_callback(job) + return None + + return upload_task + + upload_executor.submit(make_upload_task(fs)) + + except Exception as e: + log.error( + "upload", + "pipeline_error", + f"Pipeline error: {e}", + {"job_id": job_id, "error": str(e)}, + ) + finally: + # Wait for all in-flight uploads to complete + upload_executor.shutdown(wait=True) + + # Final job status + job.completed_at = datetime.now(UTC) + if job.cancelled: + job.status = UploadStatus.CANCELLED + elif all(f.status in (UploadStatus.COMPLETED, UploadStatus.SKIPPED) for f in job.files): + job.status = UploadStatus.COMPLETED + elif any(f.status == UploadStatus.COMPLETED for f in job.files): + job.status = UploadStatus.COMPLETED # Partial success + else: + job.status = UploadStatus.FAILED + + # Clean up temp directory + self.cleanup_temp_dir(job_id) + + uploaded_count = sum(1 for f in job.files if f.status == UploadStatus.COMPLETED) + skipped_count = sum(1 for f in job.files if f.status == UploadStatus.SKIPPED) + failed_count = sum(1 for f in job.files if f.status == UploadStatus.FAILED) + + file_summary = [ + { + "filename": f.filename, + "s3_path": f.s3_path, + "status": f.status.value, + "file_size": f.file_size, + "duration_seconds": f.upload_duration_seconds, + } + for f in job.files + ] + + log.info( + "upload", + "upload_job_completed", + f"Upload job completed: {uploaded_count} uploaded, " + f"{skipped_count} skipped, {failed_count} failed", + { "job_id": job_id, "status": job.status.value, "uploaded": uploaded_count, @@ -943,7 +1287,29 @@ def byte_callback(uploaded: int, total: int) -> None: "duration_seconds": job.total_upload_duration_seconds, "avg_speed_mbps": job.average_upload_speed_mbps, "files": file_summary, - }, completed_at) + }, + ) + + # Save per-job JSONL summary + completed_at = job.completed_at or datetime.now(UTC) + try: + log.save_job_jsonl( + job_id, + { + "timestamp": completed_at.isoformat(), + "event": "upload_job_completed", + "job_id": job_id, + "status": job.status.value, + "uploaded": uploaded_count, + "skipped": skipped_count, + "failed": failed_count, + "total_bytes_uploaded": job.successfully_uploaded_bytes, + "duration_seconds": job.total_upload_duration_seconds, + "avg_speed_mbps": job.average_upload_speed_mbps, + "files": file_summary, + }, + completed_at, + ) except Exception: logger.warning("Failed to save job JSONL summary", exc_info=True) @@ -953,14 +1319,14 @@ def byte_callback(uploaded: int, total: int) -> None: except Exception: logger.warning("Failed to save job CSV summary", exc_info=True) - # Auto-sync logs to S3 after job completion + # Auto-sync logs to S3 try: log.sync_logs_to_s3(s3_client, s3_bucket) except Exception: logger.debug("Log sync to S3 failed", exc_info=True) - if progress_callback: - progress_callback(job) + if upload_callback: + upload_callback(job) def cancel_job(self, job_id: str) -> bool: """Cancel an upload job. @@ -1071,9 +1437,7 @@ def pre_filter_files( } # First: check cache by filename+size (works regardless of timestamp source) - filename_result = cache.check_exists_by_filename( - s3_bucket, path.name, stat.st_size - ) + filename_result = cache.check_exists_by_filename(s3_bucket, path.name, stat.st_size) if filename_result is True: stats["cache_hits"] += 1 stats["cache_skipped"] += 1 @@ -1135,9 +1499,7 @@ def check_s3(s3_path: str) -> bool: ): # Update cache with result fs = file_statuses[idx] - cache.update_cache( - s3_bucket, s3_path, exists, fs["filename"], fs["size"] - ) + cache.update_cache(s3_bucket, s3_path, exists, fs["filename"], fs["size"]) if exists: stats["s3_hits"] += 1 fs["already_uploaded"] = True @@ -1250,20 +1612,26 @@ def scan_folder_async( if scan_job.cancelled: if progress_callback: - progress_callback(job_id, { - "type": "scan_complete", - "status": "cancelled", - }) + progress_callback( + job_id, + { + "type": "scan_complete", + "status": "cancelled", + }, + ) return scan_job.folders_total = len(folder_map) if progress_callback: - progress_callback(job_id, { - "type": "scan_started", - "folders_total": scan_job.folders_total, - "root_folder": scan_job.root_folder, - }) + progress_callback( + job_id, + { + "type": "scan_started", + "folders_total": scan_job.folders_total, + "root_folder": scan_job.root_folder, + }, + ) # Phase 2: Process each subfolder for folder_path_str, mcap_paths in sorted(folder_map.items()): @@ -1283,17 +1651,22 @@ def scan_folder_async( stat = mcap_path.stat() file_paths.append(str(mcap_path)) folder_size += stat.st_size - files_info.append({ - "path": str(mcap_path), - "filename": mcap_path.name, - "size": stat.st_size, - "mtime": stat.st_mtime, - "relative_path": str(mcap_path.relative_to(root)), - }) + files_info.append( + { + "path": str(mcap_path), + "filename": mcap_path.name, + "size": stat.st_size, + "mtime": stat.st_mtime, + "relative_path": str(mcap_path.relative_to(root)), + } + ) # Pre-filter this batch for duplicates _, pre_stats = self.pre_filter_files( - file_paths, s3_bucket, aws_profile, aws_region, + file_paths, + s3_bucket, + aws_profile, + aws_region, cache_only=cache_only, ) @@ -1353,17 +1726,20 @@ def scan_folder_async( scan_job.scanned_folders.append(folder_dict) if progress_callback: - progress_callback(job_id, { - "type": "scan_folder_complete", - "folder": folder_dict, - "folders_scanned": scan_job.folders_scanned, - "folders_total": scan_job.folders_total, - "running_totals": { - "total_files_found": scan_job.total_files_found, - "total_already_uploaded": scan_job.total_already_uploaded, - "total_size": scan_job.total_size, + progress_callback( + job_id, + { + "type": "scan_folder_complete", + "folder": folder_dict, + "folders_scanned": scan_job.folders_scanned, + "folders_total": scan_job.folders_total, + "running_totals": { + "total_files_found": scan_job.total_files_found, + "total_already_uploaded": scan_job.total_already_uploaded, + "total_size": scan_job.total_size, + }, }, - }) + ) # Terminal event with scan_job.lock: @@ -1373,15 +1749,18 @@ def scan_folder_async( scan_job.status = "completed" if progress_callback: - progress_callback(job_id, { - "type": "scan_complete", - "status": scan_job.status, - "folders_scanned": scan_job.folders_scanned, - "folders_total": scan_job.folders_total, - "total_files_found": scan_job.total_files_found, - "total_already_uploaded": scan_job.total_already_uploaded, - "total_size": scan_job.total_size, - }) + progress_callback( + job_id, + { + "type": "scan_complete", + "status": scan_job.status, + "folders_scanned": scan_job.folders_scanned, + "folders_total": scan_job.folders_total, + "total_files_found": scan_job.total_files_found, + "total_already_uploaded": scan_job.total_already_uploaded, + "total_size": scan_job.total_size, + }, + ) except Exception as e: scan_job.status = "failed" @@ -1392,11 +1771,14 @@ def scan_folder_async( {"job_id": job_id, "error": str(e)}, ) if progress_callback: - progress_callback(job_id, { - "type": "scan_complete", - "status": "failed", - "error": str(e), - }) + progress_callback( + job_id, + { + "type": "scan_complete", + "status": "failed", + "error": str(e), + }, + ) def get_active_jobs(self) -> list[UploadJob]: """Get all currently active (non-terminal) jobs.""" diff --git a/frontend/src/hooks/useUploadJob.ts b/frontend/src/hooks/useUploadJob.ts index f11e71c..381ba59 100644 --- a/frontend/src/hooks/useUploadJob.ts +++ b/frontend/src/hooks/useUploadJob.ts @@ -130,14 +130,15 @@ export function useUploadJob(options: UseUploadJobOptions = {}): UseUploadJobRes case "analysis_complete": { const evt = data as AnalysisCompleteEvent; setTotalFiles(evt.job.total_files); - // If not auto-uploading, this is terminal. if (!evt.auto_upload) { + // Analysis-only mode — this is terminal. setIsRunning(false); setJobId(null); setCompletedJob(evt.job); + setActiveFiles([]); } - // Clear active files from analysis phase. - setActiveFiles([]); + // When auto_upload is true, don't clear activeFiles — uploads + // may already be in progress via the pipeline. break; } From 452df173518bf92c224bbcc338ec2e699121f1ba Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Wed, 18 Feb 2026 13:30:24 -0700 Subject: [PATCH 126/173] Backend: Send completion messages immediately during upload --- app/services/upload_manager.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/app/services/upload_manager.py b/app/services/upload_manager.py index b98eb0a..08c2ac8 100644 --- a/app/services/upload_manager.py +++ b/app/services/upload_manager.py @@ -890,6 +890,11 @@ def byte_callback(uploaded: int, total: int) -> None: # Clean up temp directory when upload completes self.cleanup_temp_dir(job_id) + # Send terminal event IMMEDIATELY so the frontend unblocks. + # Heavy I/O (logging, CSV, S3 sync) follows below. + if progress_callback: + progress_callback(job) + uploaded_count = sum(1 for f in job.files if f.status == UploadStatus.COMPLETED) skipped_count = sum(1 for f in job.files if f.status == UploadStatus.SKIPPED) failed_count = sum(1 for f in job.files if f.status == UploadStatus.FAILED) @@ -959,9 +964,6 @@ def byte_callback(uploaded: int, total: int) -> None: except Exception: logger.debug("Log sync to S3 failed", exc_info=True) - if progress_callback: - progress_callback(job) - def analyze_and_upload_pipeline( self, job_id: str, @@ -1257,6 +1259,11 @@ def byte_callback(uploaded: int, total: int) -> None: # Clean up temp directory self.cleanup_temp_dir(job_id) + # Send terminal event IMMEDIATELY so the frontend unblocks. + # Heavy I/O (logging, CSV, S3 sync) follows below. + if upload_callback: + upload_callback(job) + uploaded_count = sum(1 for f in job.files if f.status == UploadStatus.COMPLETED) skipped_count = sum(1 for f in job.files if f.status == UploadStatus.SKIPPED) failed_count = sum(1 for f in job.files if f.status == UploadStatus.FAILED) @@ -1325,9 +1332,6 @@ def byte_callback(uploaded: int, total: int) -> None: except Exception: logger.debug("Log sync to S3 failed", exc_info=True) - if upload_callback: - upload_callback(job) - def cancel_job(self, job_id: str) -> bool: """Cancel an upload job. From 925550b34ac9fad8fe43bce32d627eae16004d2d Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Wed, 18 Feb 2026 13:30:49 -0700 Subject: [PATCH 127/173] Frontend: Improve UI/functionality of upload page --- .../components/upload/UnifiedFileTable.tsx | 103 ++++++++++++------ 1 file changed, 71 insertions(+), 32 deletions(-) diff --git a/frontend/src/components/upload/UnifiedFileTable.tsx b/frontend/src/components/upload/UnifiedFileTable.tsx index 9d27915..86b897e 100644 --- a/frontend/src/components/upload/UnifiedFileTable.tsx +++ b/frontend/src/components/upload/UnifiedFileTable.tsx @@ -1,16 +1,16 @@ /** * Unified file table that persists across Review / Upload / Summary phases. * - * Fixed 6-column grid — column content adapts per phase, but layout is stable. + * Fixed 7-column grid — column content adapts per phase, but layout is stable. * Uses @tanstack/react-virtual for 20K+ file handling. */ -import { memo, useCallback, useEffect, useRef } from "react"; +import { memo, useEffect, useRef, useState } from "react"; import { useVirtualizer } from "@tanstack/react-virtual"; import { formatBytes } from "../../utils/format/bytes.ts"; import { formatDate } from "../../utils/format/date.ts"; -import { SpinnerIcon, CheckIcon, XIcon, MinusIcon, CircleIcon } from "../../utils/icons.tsx"; +import { SpinnerIcon, CheckIcon, XIcon, MinusIcon, CircleIcon, ChevronDownIcon } from "../../utils/icons.tsx"; import type { SortDir, SortKey, @@ -20,9 +20,17 @@ import type { } from "../../types/upload.ts"; // ── Grid template (fixed across all phases) ── -const GRID_COLS = "grid-cols-[36px_1fr_1fr_80px_120px_1fr]"; +const GRID_COLS = "grid-cols-[42px_36px_1fr_1fr_80px_120px_1fr]"; const ROW_HEIGHT = 40; +// Row background colors by status (uploading + summary phases only) +const STATUS_ROW_BG: Record = { + completed: "bg-green-50/70", + failed: "bg-red-50/70", + skipped: "bg-yellow-50/50", + in_progress: "bg-blue-50/60", +}; + // ── Props ── interface UnifiedFileTableProps { @@ -73,17 +81,31 @@ export default function UnifiedFileTable({ prevCountRef.current = files.length; }, [files.length, rowVirtualizer]); - const scrollToActive = useCallback(() => { - const idx = files.findIndex( - (f) => f.status === "in_progress", - ); + // ── Auto-scroll: follow the active file during upload ── + + const [autoFollow, setAutoFollow] = useState(true); + const lastActivePathRef = useRef(null); + + // Reset auto-follow when entering upload phase + const prevPhaseRef = useRef(phase); + if (phase === "uploading" && prevPhaseRef.current !== "uploading") { + setAutoFollow(true); + lastActivePathRef.current = null; + } + prevPhaseRef.current = phase; + + useEffect(() => { + if (phase !== "uploading" || !autoFollow) return; + + const activeFile = files.find((f) => f.status === "in_progress"); + if (!activeFile || activeFile.path === lastActivePathRef.current) return; + + lastActivePathRef.current = activeFile.path; + const idx = files.indexOf(activeFile); if (idx >= 0) { rowVirtualizer.scrollToIndex(idx, { align: "center" }); } - }, [files, rowVirtualizer]); - - // Count active files for the "jump to active" button - const hasActiveFiles = phase === "uploading" && files.some((f) => f.status === "in_progress"); + }, [phase, autoFollow, files, rowVirtualizer]); return (
@@ -91,7 +113,10 @@ export default function UnifiedFileTable({
- {/* Col 1: Checkbox / Status icon */} + {/* Col 1: Row number */} +
#
+ + {/* Col 2: Checkbox / Status icon */}
{phase === "review" ? ( - {/* Col 2: Filename */} + {/* Col 3: Filename */} - {/* Col 3: Folder */} + {/* Col 4: Folder */} - {/* Col 4: Size */} + {/* Col 5: Size */} - {/* Col 5: Status */} + {/* Col 6: Status */} - {/* Col 6: Detail (context-dependent) */} + {/* Col 7: Detail (context-dependent) */}
{phase === "review" && "Modified"} {phase === "uploading" && ""} @@ -178,6 +203,7 @@ export default function UnifiedFileTable({ - {/* Jump to active button (upload phase only) */} - {hasActiveFiles && ( + {/* Auto-scroll toggle (upload phase only) */} + {phase === "uploading" && ( )}
@@ -221,6 +251,7 @@ export default function UnifiedFileTable({ interface FileRowProps { row: UnifiedFileRow; + rowNumber: number; phase: UploadPhase; isSelected: boolean; onToggle: (path: string) => void; @@ -229,17 +260,25 @@ interface FileRowProps { const FileRow = memo(function FileRow({ row, + rowNumber, phase, isSelected, onToggle, style, }: FileRowProps) { + const statusBg = phase !== "review" ? (STATUS_ROW_BG[row.status] ?? "") : ""; + return (
- {/* Col 1: Checkbox / Status icon */} + {/* Col 1: Row number */} + + {rowNumber} + + + {/* Col 2: Checkbox / Status icon */}
{phase === "review" ? ( - {/* Col 2: Filename */} + {/* Col 3: Filename */} {row.filename} - {/* Col 3: Folder */} + {/* Col 4: Folder */} {row.folder} - {/* Col 4: Size */} + {/* Col 5: Size */} {formatBytes(row.size)} - {/* Col 5: Status */} + {/* Col 6: Status */}
- {/* Col 6: Detail */} + {/* Col 7: Detail */}
{phase === "review" && formatDate(row.mtime)} {phase === "uploading" && row.status === "in_progress" && ( From 37f98977befbe4ba9cbc07445c5d417b8a74cc1a Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Wed, 18 Feb 2026 13:46:48 -0700 Subject: [PATCH 128/173] Fix: Show individual rows as completed after they are done uploading --- app/services/upload_manager.py | 6 ++++++ frontend/src/components/upload/UploadHeader.tsx | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/app/services/upload_manager.py b/app/services/upload_manager.py index 08c2ac8..01689e3 100644 --- a/app/services/upload_manager.py +++ b/app/services/upload_manager.py @@ -1226,6 +1226,12 @@ def byte_callback(uploaded: int, total: int) -> None: }, ) + # Notify per-file status so the frontend + # updates this row immediately (the progress + # dict only includes active files, so without + # this the row would keep spinning). + if analysis_callback: + analysis_callback(job, file_state) if upload_callback: upload_callback(job) return None diff --git a/frontend/src/components/upload/UploadHeader.tsx b/frontend/src/components/upload/UploadHeader.tsx index 1c35a7d..ab9a9ef 100644 --- a/frontend/src/components/upload/UploadHeader.tsx +++ b/frontend/src/components/upload/UploadHeader.tsx @@ -144,9 +144,9 @@ function UploadingHeader({

- {isRunning ? "Uploading..." : "Upload Complete"} + {isRunning && filesProcessed < totalFiles ? "Uploading..." : "Upload Complete"}

- {isRunning && } + {isRunning && filesProcessed < totalFiles && }
Date: Wed, 18 Feb 2026 13:54:02 -0700 Subject: [PATCH 129/173] Linux: Add exfat4 mount permissions script --- setup-mount-permissions.sh | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100755 setup-mount-permissions.sh diff --git a/setup-mount-permissions.sh b/setup-mount-permissions.sh new file mode 100755 index 0000000..84b1c73 --- /dev/null +++ b/setup-mount-permissions.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# setup-mount-permissions.sh +# +# Configures udisks2 to mount exFAT drives with read/write permissions +# for the current user, so the MODAQ Uploader can delete local files. +# +# Usage: sudo ./setup-mount-permissions.sh [username] + +set -euo pipefail + +USER="${1:-${SUDO_USER:-}}" + +if [ -z "$USER" ]; then + echo "Error: Could not determine target user." + echo "Usage: sudo $0 [username]" + exit 1 +fi + +UID_NUM=$(id -u "$USER") +GID_NUM=$(id -g "$USER") + +CONF="/etc/udisks2/mount_options.conf" + +mkdir -p /etc/udisks2 + +cat > "$CONF" << EOF +# MODAQ Uploader — mount exFAT drives with rw for $USER +[defaults] +exfat_defaults=uid=$UID_NUM,gid=$GID_NUM,dmask=0022,fmask=0133 +EOF + +echo "Written: $CONF" +echo "exFAT drives will now mount with rw for $USER (uid=$UID_NUM)." +echo "" +echo "Unplug and re-plug the drive to apply." From 3cf268057434251d95bf5026fd274da9577b0cfa Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Wed, 18 Feb 2026 14:16:51 -0700 Subject: [PATCH 130/173] Linux: Point to correct modaq logo location --- modaq-upload.desktop.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modaq-upload.desktop.template b/modaq-upload.desktop.template index 6294a0d..5cf5435 100644 --- a/modaq-upload.desktop.template +++ b/modaq-upload.desktop.template @@ -4,7 +4,7 @@ Type=Application Name=MODAQ Upload Comment=MODAQ File Uploader Exec={{PROJECT_DIR}}/venv/bin/python {{PROJECT_DIR}}/launch.py -Icon={{PROJECT_DIR}}/app/static/images/modaq-logo.png +Icon={{PROJECT_DIR}}/frontend/public/images/modaq-logo.png Terminal=true Categories=Science;Utility; StartupNotify=true From 4c2f802bac539a67725ed379cead5e88a9608976 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Wed, 18 Feb 2026 14:19:48 -0700 Subject: [PATCH 131/173] Backend: Add delete fix permissions end point --- app/routes/delete.py | 74 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/app/routes/delete.py b/app/routes/delete.py index b392c03..8c1eb2f 100644 --- a/app/routes/delete.py +++ b/app/routes/delete.py @@ -1,6 +1,8 @@ """Delete API routes for local file cleanup after S3 upload.""" +import getpass import json +import subprocess import threading import time from collections import deque @@ -73,6 +75,7 @@ def scan_folder() -> tuple[Response, int]: return jsonify({"error": f"Permission denied: {e}"}), 403 total_size = sum(f.file_size for f in job.files) + has_permission_issues = any(not f.writable for f in job.files) return jsonify({ "success": True, @@ -81,6 +84,7 @@ def scan_folder() -> tuple[Response, int]: "files": [f.to_dict() for f in job.files], "total_files": len(job.files), "total_size": total_size, + "permission_warning": has_permission_issues, }), 200 @@ -229,3 +233,73 @@ def cancel_delete(job_id: str) -> tuple[Response, int]: }), 200 return jsonify({"error": "Job not found"}), 404 + + +@delete_bp.route("/fix-permissions", methods=["POST"]) +def fix_permissions() -> tuple[Response, int]: + """Fix file permissions on an ext4 external drive using sudo chown. + + Runs ``sudo -S chown -R `` with the + supplied password piped via stdin (never logged or stored). + + Request body: + folder_path: Directory whose ownership should be fixed + password: The user's sudo password + + Returns: + JSON with success status or error details + """ + if not request.is_json: + return jsonify({"error": "JSON body required"}), 400 + + data = request.get_json() + if not data or "folder_path" not in data or "password" not in data: + return jsonify({"error": "folder_path and password are required"}), 400 + + folder_path = Path(data["folder_path"]) + password: str = data["password"] + + if not folder_path.exists(): + return jsonify({"error": f"Folder not found: {folder_path}"}), 404 + + if not folder_path.is_dir(): + return jsonify({"error": f"Path is not a directory: {folder_path}"}), 400 + + # Security: only allow paths under /media/ to prevent abuse + try: + resolved = folder_path.resolve() + if not str(resolved).startswith("/media/"): + return jsonify( + {"error": "Permission fix is only allowed for paths under /media/"} + ), 403 + except (OSError, ValueError): + return jsonify({"error": "Invalid path"}), 400 + + current_user = getpass.getuser() + + try: + result = subprocess.run( # noqa: S603 + ["sudo", "-S", "chown", "-R", current_user, str(resolved)], + input=f"{password}\n", + capture_output=True, + text=True, + timeout=60, + ) + + if result.returncode != 0: + stderr = result.stderr.strip() + # Strip sudo password prompt from error output + error_lines = [ + line + for line in stderr.splitlines() + if not line.startswith("[sudo]") and "password" not in line.lower() + ] + error_msg = "\n".join(error_lines).strip() or "Permission fix failed" + return jsonify({"error": error_msg}), 500 + + return jsonify({"success": True}), 200 + + except subprocess.TimeoutExpired: + return jsonify({"error": "Operation timed out"}), 500 + except Exception as e: + return jsonify({"error": str(e)}), 500 From 4de3600a3b5c1bc23ab094c7571b05b137439c9c Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Wed, 18 Feb 2026 14:20:12 -0700 Subject: [PATCH 132/173] Backend: Send writeable status to the frontend --- app/services/delete_manager.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/services/delete_manager.py b/app/services/delete_manager.py index a80fafe..f5d3b21 100644 --- a/app/services/delete_manager.py +++ b/app/services/delete_manager.py @@ -40,6 +40,7 @@ class FileDeleteState: file_size: int s3_path: str s3_bucket: str + writable: bool = True status: DeleteStatus = DeleteStatus.PENDING local_md5: str = "" s3_etag: str = "" @@ -55,6 +56,7 @@ def to_dict(self) -> dict[str, Any]: "file_size": self.file_size, "s3_path": self.s3_path, "s3_bucket": self.s3_bucket, + "writable": self.writable, "status": self.status.value, "local_md5": self.local_md5, "s3_etag": self.s3_etag, @@ -222,6 +224,7 @@ def scan_folder( file_size=file_size, s3_path=str(cache_info["s3_path"]), s3_bucket=bucket, + writable=os.access(str(mcap_path), os.W_OK), ) job.files.append(file_state) From ef9c338c34e1cf4b97895a3f53532a68d70f0b03 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Wed, 18 Feb 2026 14:20:40 -0700 Subject: [PATCH 133/173] Frontend: Add Lock icon --- frontend/src/utils/icons.tsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frontend/src/utils/icons.tsx b/frontend/src/utils/icons.tsx index d917a5d..f52d8ae 100644 --- a/frontend/src/utils/icons.tsx +++ b/frontend/src/utils/icons.tsx @@ -30,6 +30,7 @@ import { Minus, Cloud, Circle, + Lock, type LucideProps, } from "lucide-react"; @@ -60,6 +61,7 @@ export const PlusIcon = Plus; export const MinusIcon = Minus; export const CloudIcon = Cloud; export const CircleIcon = Circle; +export const LockIcon = Lock; // Export type for icon props export type { LucideProps as IconProps }; From 1b4bb36cc007d3c7d4bd7fa3958fc2cacf8676e2 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Wed, 18 Feb 2026 14:21:01 -0700 Subject: [PATCH 134/173] Frontend: Add Fix permissions flow for delete --- .../components/delete/FixPermissionsModal.tsx | 148 ++++++++++++++++++ frontend/src/hooks/useDeleteScan.ts | 2 +- frontend/src/pages/DeletePage.tsx | 48 +++++- frontend/src/stores/deleteStore.ts | 14 +- frontend/src/types/delete.ts | 2 + 5 files changed, 208 insertions(+), 6 deletions(-) create mode 100644 frontend/src/components/delete/FixPermissionsModal.tsx diff --git a/frontend/src/components/delete/FixPermissionsModal.tsx b/frontend/src/components/delete/FixPermissionsModal.tsx new file mode 100644 index 0000000..f1437ba --- /dev/null +++ b/frontend/src/components/delete/FixPermissionsModal.tsx @@ -0,0 +1,148 @@ +import { useCallback, useState } from "react"; + +import { apiPost } from "../../api/client.ts"; +import { LockIcon, SpinnerIcon, SuccessIcon, WarningIcon } from "../../utils/icons.tsx"; +import Modal from "../common/Modal.tsx"; + +interface FixPermissionsModalProps { + isOpen: boolean; + onClose: () => void; + folderPath: string; + onFixed: () => void; +} + +export default function FixPermissionsModal({ + isOpen, + onClose, + folderPath, + onFixed, +}: FixPermissionsModalProps) { + const [password, setPassword] = useState(""); + const [isLoading, setIsLoading] = useState(false); + const [error, setError] = useState(null); + const [success, setSuccess] = useState(false); + + const handleSubmit = useCallback( + async (e: React.FormEvent) => { + e.preventDefault(); + if (!password.trim()) return; + + setIsLoading(true); + setError(null); + + try { + await apiPost("/api/delete/fix-permissions", { + folder_path: folderPath, + password, + }); + setSuccess(true); + setPassword(""); + onFixed(); + } catch (err) { + const msg = + err instanceof Error ? err.message : "Failed to fix permissions"; + setError(msg); + } finally { + setIsLoading(false); + } + }, + [password, folderPath, onFixed], + ); + + const handleClose = useCallback(() => { + setPassword(""); + setError(null); + setSuccess(false); + onClose(); + }, [onClose]); + + return ( + + + +
+ ) : ( + + ) + } + > + {success ? ( +
+ +

+ Permissions fixed successfully. The scan results will be refreshed. +

+
+ ) : ( +
+
+ +
+

+ The files on this drive are owned by a different user. Your sudo + password is needed to change ownership so files can be deleted. +

+

+ Your password is sent directly to the system and is never stored + or logged. +

+
+
+ +
+ + setPassword(e.target.value)} + placeholder="Enter your password" + autoFocus + className="w-full px-3 py-2 border border-gray-300 rounded-md text-sm focus:outline-none focus:ring-2 focus:ring-nlr-blue focus:border-transparent" + /> +
+ + {error && ( +
+ +

{error}

+
+ )} +
+ )} + + ); +} diff --git a/frontend/src/hooks/useDeleteScan.ts b/frontend/src/hooks/useDeleteScan.ts index 1bde0c9..c9e8435 100644 --- a/frontend/src/hooks/useDeleteScan.ts +++ b/frontend/src/hooks/useDeleteScan.ts @@ -36,7 +36,7 @@ export function useDeleteScan(): UseDeleteScanResult { } const res = await apiPost("/api/delete/scan", body); setDeleteJobId(res.job_id); - setScanResults(res.files, res.total_size); + setScanResults(res.files, res.total_size, res.permission_warning); } finally { setIsScanning(false); } diff --git a/frontend/src/pages/DeletePage.tsx b/frontend/src/pages/DeletePage.tsx index 782dbf1..992074f 100644 --- a/frontend/src/pages/DeletePage.tsx +++ b/frontend/src/pages/DeletePage.tsx @@ -8,13 +8,14 @@ * Step 5: Summary with results */ -import { useCallback, useEffect, useMemo, useState } from "react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import ProgressBar from "../components/common/ProgressBar.tsx"; import Spinner from "../components/common/Spinner.tsx"; import StatCard from "../components/common/StatCard.tsx"; import DeleteConfirmation from "../components/delete/DeleteConfirmation.tsx"; import DeleteStepper from "../components/delete/DeleteStepper.tsx"; +import FixPermissionsModal from "../components/delete/FixPermissionsModal.tsx"; import CancelConfirmModal from "../components/upload/CancelConfirmModal.tsx"; import FolderBrowser from "../components/upload/FolderBrowser.tsx"; import type { FolderExclusions } from "../components/upload/FolderBrowser.tsx"; @@ -23,6 +24,7 @@ import { useDeleteScan } from "../hooks/useDeleteScan.ts"; import { useAppStore } from "../stores/appStore.ts"; import { useDeleteStore, type DeleteStep } from "../stores/deleteStore.ts"; import type { DeleteScanFile } from "../types/delete.ts"; +import { LockIcon, WarningIcon } from "../utils/icons.tsx"; import { formatBytes } from "../utils/format/bytes.ts"; /** Status badge colors for file table. */ @@ -61,6 +63,7 @@ export default function DeletePage() { setFolderPath, scanResults, scanTotalSize, + permissionWarning, completedJob, isDeleting, reset, @@ -77,12 +80,17 @@ export default function DeletePage() { const [page, setPage] = useState(0); const pageSize = 50; const [showCancelModal, setShowCancelModal] = useState(false); + const [showFixPermModal, setShowFixPermModal] = useState(false); const [refreshKey, setRefreshKey] = useState(0); + // Track exclusions so re-scan after permission fix uses the same filters + const lastExclusions = useRef(undefined); + // ── Step transitions ── const handleFolderSelected = useCallback( async (path: string, exclusions?: FolderExclusions) => { + lastExclusions.current = exclusions; setFolderPath(path); setStep(2); await scan(path, exclusions); @@ -90,6 +98,12 @@ export default function DeletePage() { [setFolderPath, setStep, scan], ); + const handlePermissionsFixed = useCallback(async () => { + if (folderPath) { + await scan(folderPath, lastExclusions.current); + } + }, [folderPath, scan]); + const handleConfirmDelete = useCallback(async () => { setStep(4); await deleteJob.startDelete(); @@ -196,6 +210,29 @@ export default function DeletePage() { />
+ {permissionWarning && ( +
+ +
+

+ Permission issues detected +

+

+ Some files on this drive are owned by a different user and + cannot be deleted without fixing permissions first. +

+ +
+
+ )} + {scanResults.length === 0 && !isScanning && (
No uploaded MCAP files found in this folder. Only files previously @@ -286,13 +323,20 @@ export default function DeletePage() { )}
+ + setShowFixPermModal(false)} + folderPath={folderPath} + onFixed={handlePermissionsFixed} + />
)} diff --git a/frontend/src/stores/deleteStore.ts b/frontend/src/stores/deleteStore.ts index 44c1bfa..09ddcb9 100644 --- a/frontend/src/stores/deleteStore.ts +++ b/frontend/src/stores/deleteStore.ts @@ -15,8 +15,14 @@ interface DeleteState { // Scan results scanResults: DeleteScanFile[]; scanTotalSize: number; + permissionWarning: boolean; isScanning: boolean; - setScanResults: (files: DeleteScanFile[], totalSize: number) => void; + setScanResults: ( + files: DeleteScanFile[], + totalSize: number, + permissionWarning: boolean, + ) => void; + setPermissionWarning: (warning: boolean) => void; setIsScanning: (scanning: boolean) => void; // Delete job @@ -36,6 +42,7 @@ const initialState = { folderPath: "", scanResults: [] as DeleteScanFile[], scanTotalSize: 0, + permissionWarning: false, isScanning: false, deleteJobId: null, completedJob: null, @@ -48,8 +55,9 @@ export const useDeleteStore = create((set) => ({ setStep: (step) => set({ step }), setFolderPath: (folderPath) => set({ folderPath }), - setScanResults: (scanResults, scanTotalSize) => - set({ scanResults, scanTotalSize }), + setScanResults: (scanResults, scanTotalSize, permissionWarning) => + set({ scanResults, scanTotalSize, permissionWarning }), + setPermissionWarning: (permissionWarning) => set({ permissionWarning }), setIsScanning: (isScanning) => set({ isScanning }), setDeleteJobId: (deleteJobId) => set({ deleteJobId }), diff --git a/frontend/src/types/delete.ts b/frontend/src/types/delete.ts index 0547bcf..7f1e21d 100644 --- a/frontend/src/types/delete.ts +++ b/frontend/src/types/delete.ts @@ -21,6 +21,7 @@ export interface DeleteScanFile { file_size: number; s3_path: string; s3_bucket: string; + writable: boolean; status: DeleteFileStatus; local_md5: string; s3_etag: string; @@ -36,6 +37,7 @@ export interface DeleteScanResponse { files: DeleteScanFile[]; total_files: number; total_size: number; + permission_warning: boolean; } // ── SSE event types ── From 3359dc7d41172fece718219d488a08cc960aa45b Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Wed, 18 Feb 2026 14:41:05 -0700 Subject: [PATCH 135/173] Fix: Actually stop upload when user requests cancellation --- app/routes/files.py | 8 +++++ app/services/s3_service.py | 23 +++++++++++-- app/services/upload_manager.py | 61 +++++++++++++++++++++++++++++----- 3 files changed, 82 insertions(+), 10 deletions(-) diff --git a/app/routes/files.py b/app/routes/files.py index 067e784..fddc02d 100644 --- a/app/routes/files.py +++ b/app/routes/files.py @@ -241,6 +241,11 @@ def _walk_error(err: OSError) -> None: {"name": "Home", "path": str(Path.home())}, ] + # Add SURFWEC SSD as the top priority quick link + surfwec_ssd = Path("/media/m2/SURFWEC_SSD") + if surfwec_ssd.exists() and surfwec_ssd.is_dir(): + quick_links.append({"name": "SURFWEC_SSD", "path": str(surfwec_ssd)}) + # Add Volumes on macOS volumes_path = Path("/Volumes") if volumes_path.exists(): @@ -279,6 +284,9 @@ def _walk_error(err: OSError) -> None: children = [] if children: for child in sorted(children, key=lambda x: x.name.lower()): + # Already added SURFWEC_SSD above + if child == surfwec_ssd: + continue quick_links.append({"name": child.name, "path": str(child)}) else: quick_links.append({"name": entry.name, "path": entry_str}) diff --git a/app/services/s3_service.py b/app/services/s3_service.py index 5b80a27..362e3a1 100644 --- a/app/services/s3_service.py +++ b/app/services/s3_service.py @@ -10,6 +10,10 @@ from botocore.exceptions import ClientError, NoCredentialsError from mypy_boto3_s3 import S3Client + +class UploadCancelledError(Exception): + """Raised when an upload is cancelled mid-transfer.""" + # Multipart threshold: files below this size are uploaded as a single PUT request, # which produces a simple MD5 ETag. Files above use multipart upload, which produces # a composite ETag (md5_of_part_md5s-part_count) that can't be compared to a local MD5. @@ -101,6 +105,7 @@ def upload_file_with_progress( bucket: str, key: str, callback: Callable[[int, int], None] | None = None, + cancel_check: Callable[[], bool] | None = None, ) -> dict[str, Any]: """Upload a file to S3 with progress tracking. @@ -110,9 +115,15 @@ def upload_file_with_progress( bucket: S3 bucket name key: S3 object key callback: Progress callback function (bytes_uploaded, total_bytes) + cancel_check: Callable that returns True if the upload should be cancelled. + Checked on every progress callback (each chunk). When True, raises + UploadCancelledError to abort the boto3 transfer immediately. Returns: Dictionary with upload result information + + Raises: + UploadCancelledError: If cancel_check returns True during upload """ file_path = Path(path) file_size = file_path.stat().st_size @@ -121,18 +132,24 @@ class ProgressCallback: """Callback class for tracking upload progress.""" def __init__( - self, total_size: int, user_callback: Callable[[int, int], None] | None + self, + total_size: int, + user_callback: Callable[[int, int], None] | None, + should_cancel: Callable[[], bool] | None, ) -> None: self.total_size = total_size self.uploaded = 0 self.user_callback = user_callback + self.should_cancel = should_cancel def __call__(self, bytes_amount: int) -> None: + if self.should_cancel and self.should_cancel(): + raise UploadCancelledError(f"Upload cancelled for {key}") self.uploaded += bytes_amount if self.user_callback: self.user_callback(self.uploaded, self.total_size) - progress = ProgressCallback(file_size, callback) + progress = ProgressCallback(file_size, callback, cancel_check) try: client.upload_file( @@ -150,6 +167,8 @@ def __call__(self, bytes_amount: int) -> None: "size": file_size, "error": None, } + except UploadCancelledError: + raise except ClientError as e: return { "success": False, diff --git a/app/services/upload_manager.py b/app/services/upload_manager.py index 01689e3..b8e7fd9 100644 --- a/app/services/upload_manager.py +++ b/app/services/upload_manager.py @@ -16,6 +16,7 @@ from app.services import mcap_service, s3_service from app.services.cache_service import get_cache_service from app.services.log_service import get_log_service +from app.services.s3_service import UploadCancelledError from app.services.utils import format_file_size logger = logging.getLogger(__name__) @@ -608,6 +609,11 @@ def analyze_job_async( for file_state in job.files } for future in as_completed(parse_futures): + if job.cancelled: + for pending_future in parse_futures: + pending_future.cancel() + break + file_state = parse_futures[future] result = future.result() if isinstance(result, str): @@ -776,6 +782,11 @@ def make_upload_task( fs: FileUploadState, ) -> Callable[[], Any]: def upload_task() -> Any: + if job.cancelled: + with job.lock: + fs.status = UploadStatus.CANCELLED + return None + # Mark UPLOADING inside the worker so files stay READY until picked up with job.lock: fs.status = UploadStatus.UPLOADING @@ -806,6 +817,7 @@ def byte_callback(uploaded: int, total: int) -> None: s3_bucket, fs.s3_path, byte_callback, + cancel_check=lambda: job.cancelled, ) return upload_task @@ -815,12 +827,12 @@ def byte_callback(uploaded: int, total: int) -> None: # Process results as they complete for future in as_completed(futures): - if job.cancelled: - break - file_state = futures[future] try: result = future.result() + if result is None: + # Task was cancelled before starting + continue file_state.upload_completed_at = datetime.now(UTC) if result["success"]: file_state.status = UploadStatus.COMPLETED @@ -862,6 +874,10 @@ def byte_callback(uploaded: int, total: int) -> None: "error": file_state.error_message, }, ) + except UploadCancelledError: + with job.lock: + file_state.status = UploadStatus.CANCELLED + file_state.upload_completed_at = datetime.now(UTC) except Exception as e: file_state.upload_completed_at = datetime.now(UTC) file_state.status = UploadStatus.FAILED @@ -1038,6 +1054,9 @@ def analyze_and_upload_pipeline( for future in as_completed(parse_futures): if job.cancelled: + # Cancel remaining parse futures that haven't started yet + for pending_future in parse_futures: + pending_future.cancel() break fs = parse_futures[future] @@ -1128,6 +1147,10 @@ def upload_task() -> Any: if job.cancelled: with job.lock: file_state.status = UploadStatus.CANCELLED + if analysis_callback: + analysis_callback(job, file_state) + if upload_callback: + upload_callback(job) return None try: @@ -1160,6 +1183,7 @@ def byte_callback(uploaded: int, total: int) -> None: s3_bucket, file_state.s3_path, byte_callback, + cancel_check=lambda: job.cancelled, ) # Handle completion inline @@ -1211,6 +1235,10 @@ def byte_callback(uploaded: int, total: int) -> None: "error": file_state.error_message, }, ) + except UploadCancelledError: + with job.lock: + file_state.status = UploadStatus.CANCELLED + file_state.upload_completed_at = datetime.now(UTC) except Exception as e: file_state.upload_completed_at = datetime.now(UTC) file_state.status = UploadStatus.FAILED @@ -1248,8 +1276,20 @@ def byte_callback(uploaded: int, total: int) -> None: {"job_id": job_id, "error": str(e)}, ) finally: - # Wait for all in-flight uploads to complete - upload_executor.shutdown(wait=True) + # Wait for in-flight uploads; cancel_work_items prevents queued tasks from starting + upload_executor.shutdown(wait=True, cancel_futures=True) + + # Mark any files still in non-terminal states as cancelled + if job.cancelled: + with job.lock: + for fs in job.files: + if fs.status in ( + UploadStatus.PENDING, + UploadStatus.READY, + UploadStatus.ANALYZING, + UploadStatus.UPLOADING, + ): + fs.status = UploadStatus.CANCELLED # Final job status job.completed_at = datetime.now(UTC) @@ -1352,9 +1392,14 @@ def cancel_job(self, job_id: str) -> bool: return False job.cancelled = True - for file_state in job.files: - if file_state.status in (UploadStatus.PENDING, UploadStatus.READY): - file_state.status = UploadStatus.CANCELLED + with job.lock: + for file_state in job.files: + if file_state.status in ( + UploadStatus.PENDING, + UploadStatus.READY, + UploadStatus.ANALYZING, + ): + file_state.status = UploadStatus.CANCELLED # Clean up temp directory when job is cancelled self.cleanup_temp_dir(job_id) From b85851b3fb125c1d2422a97574b08083eefd71f0 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Wed, 18 Feb 2026 14:47:18 -0700 Subject: [PATCH 136/173] Feature: Add graceful shutdown button --- app/routes/settings.py | 28 +++++++++++++ .../src/components/settings/DangerZone.tsx | 39 +++++++++++++++++++ frontend/src/utils/icons.tsx | 2 + 3 files changed, 69 insertions(+) diff --git a/app/routes/settings.py b/app/routes/settings.py index 486f72c..2f46c47 100644 --- a/app/routes/settings.py +++ b/app/routes/settings.py @@ -1,5 +1,9 @@ """Settings API routes for modaq_upload""" +import os +import signal +import threading + from flask import Blueprint, Response, jsonify, request from app.config import get_package_version, get_settings, get_updater @@ -247,6 +251,30 @@ def invalidate_cache() -> tuple[Response, int]: ), 200 +@settings_bp.route("/shutdown", methods=["POST"]) +def shutdown_server() -> tuple[Response, int]: + """Gracefully shut down the application server. + + Sends SIGINT to the current process after a brief delay so the + HTTP response can be returned to the client first. + + Returns: + JSON response confirming shutdown was initiated + """ + log = get_log_service() + log.info("app", "shutdown_requested", "Graceful shutdown requested via settings UI") + + def _shutdown() -> None: + os.kill(os.getpid(), signal.SIGINT) + + # Delay slightly so the response reaches the client + timer = threading.Timer(0.5, _shutdown) + timer.daemon = True + timer.start() + + return jsonify({"success": True, "message": "Server is shutting down..."}), 200 + + @settings_bp.route("/cache/sync", methods=["POST"]) def sync_cache_with_s3() -> tuple[Response, int]: """Sync cache with S3, marking deleted files as non-existent. diff --git a/frontend/src/components/settings/DangerZone.tsx b/frontend/src/components/settings/DangerZone.tsx index 8a5ba39..9f61bbd 100644 --- a/frontend/src/components/settings/DangerZone.tsx +++ b/frontend/src/components/settings/DangerZone.tsx @@ -1,6 +1,7 @@ import { useState } from "react"; import { apiPost } from "../../api/client.ts"; import { useAppStore } from "../../stores/appStore.ts"; +import { PowerIcon } from "../../utils/icons.tsx"; const DEFAULT_SETTINGS = { aws_profile: "default", @@ -16,6 +17,7 @@ export default function DangerZone() { const [clearing, setClearing] = useState(false); const [resetting, setResetting] = useState(false); + const [shuttingDown, setShuttingDown] = useState(false); async function handleClearCache() { if (!window.confirm("Are you sure you want to clear the upload cache? This cannot be undone.")) { @@ -59,6 +61,25 @@ export default function DangerZone() { } } + async function handleShutdown() { + if ( + !window.confirm( + "Are you sure you want to shut down the server? You will need to restart it manually.", + ) + ) { + return; + } + + setShuttingDown(true); + try { + await apiPost<{ success: boolean; message: string }>("/api/settings/shutdown"); + addNotification("info", "Server is shutting down..."); + } catch { + addNotification("error", "Failed to shut down server"); + setShuttingDown(false); + } + } + return (

Danger Zone

@@ -100,6 +121,24 @@ export default function DangerZone() { {resetting ? "Resetting..." : "Reset Settings"}
+ +
+
+
Shutdown Server
+
+ Gracefully shuts down the application. You will need to restart it manually. +
+
+ +
); diff --git a/frontend/src/utils/icons.tsx b/frontend/src/utils/icons.tsx index f52d8ae..4a5a2c2 100644 --- a/frontend/src/utils/icons.tsx +++ b/frontend/src/utils/icons.tsx @@ -31,6 +31,7 @@ import { Cloud, Circle, Lock, + Power, type LucideProps, } from "lucide-react"; @@ -62,6 +63,7 @@ export const MinusIcon = Minus; export const CloudIcon = Cloud; export const CircleIcon = Circle; export const LockIcon = Lock; +export const PowerIcon = Power; // Export type for icon props export type { LucideProps as IconProps }; From 42d5faa1fd11b42e7b9cb85019aa4a9319e79b15 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Wed, 18 Feb 2026 14:51:53 -0700 Subject: [PATCH 137/173] Fix: Make graceful shutdown work when ran via gunicorn --- app/routes/settings.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/app/routes/settings.py b/app/routes/settings.py index 2f46c47..7f2d2d0 100644 --- a/app/routes/settings.py +++ b/app/routes/settings.py @@ -2,6 +2,7 @@ import os import signal +import sys import threading from flask import Blueprint, Response, jsonify, request @@ -255,8 +256,13 @@ def invalidate_cache() -> tuple[Response, int]: def shutdown_server() -> tuple[Response, int]: """Gracefully shut down the application server. - Sends SIGINT to the current process after a brief delay so the - HTTP response can be returned to the client first. + Detects whether we're running under gunicorn or the Flask dev server + and sends the appropriate signal after a brief delay so the HTTP + response can be returned to the client first. + + - Gunicorn: sends SIGTERM to the master (parent) process, which + triggers a graceful shutdown of all workers. + - Flask dev server: sends SIGINT to the current process. Returns: JSON response confirming shutdown was initiated @@ -265,7 +271,12 @@ def shutdown_server() -> tuple[Response, int]: log.info("app", "shutdown_requested", "Graceful shutdown requested via settings UI") def _shutdown() -> None: - os.kill(os.getpid(), signal.SIGINT) + if "gunicorn" in sys.modules: + # Under gunicorn, the worker's parent is the master process. + # SIGTERM tells the master to finish active requests and exit. + os.kill(os.getppid(), signal.SIGTERM) + else: + os.kill(os.getpid(), signal.SIGINT) # Delay slightly so the response reaches the client timer = threading.Timer(0.5, _shutdown) From 45bdd016e14ce0853908b72090d7311f5bec9845 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Mon, 2 Mar 2026 06:59:12 -0700 Subject: [PATCH 138/173] Backend: Refactor to use batch processing --- app/services/batch_processor.py | 433 ++++++++++++++++++++++++++++++ app/services/job_storage.py | 455 ++++++++++++++++++++++++++++++++ 2 files changed, 888 insertions(+) create mode 100644 app/services/batch_processor.py create mode 100644 app/services/job_storage.py diff --git a/app/services/batch_processor.py b/app/services/batch_processor.py new file mode 100644 index 0000000..a2c161e --- /dev/null +++ b/app/services/batch_processor.py @@ -0,0 +1,433 @@ +"""Batch processing infrastructure for handling large upload/delete operations.""" + +import logging +import os +import time +from collections.abc import Callable +from dataclasses import dataclass +from datetime import UTC, datetime +from enum import Enum +from typing import Any + +import psutil + +logger = logging.getLogger(__name__) + + +@dataclass +class BatchConfig: + """Configuration for batch processing behavior.""" + + enabled: bool = True + batch_size: int = 100 + auto_tune_workers: bool = True + max_workers: int = 4 + target_cpu_percent: float = 70.0 + skip_mcap_validation: bool = False + use_database_for_large_jobs: bool = True + large_job_threshold: int = 1000 + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> "BatchConfig": + """Create BatchConfig from dictionary.""" + return cls( + enabled=data.get("enabled", True), + batch_size=data.get("batch_size", 100), + auto_tune_workers=data.get("auto_tune_workers", True), + max_workers=data.get("max_workers", 4), + target_cpu_percent=data.get("target_cpu_percent", 70.0), + skip_mcap_validation=data.get("skip_mcap_validation", False), + use_database_for_large_jobs=data.get("use_database_for_large_jobs", True), + large_job_threshold=data.get("large_job_threshold", 1000), + ) + + +class BatchStatus(Enum): + """Status of a batch within a job.""" + + PENDING = "pending" + PROCESSING = "processing" + COMPLETED = "completed" + FAILED = "failed" + CANCELLED = "cancelled" + + +@dataclass +class BatchState: + """State tracking for an individual batch within a job.""" + + batch_id: int + total_batches: int + files_in_batch: int + status: BatchStatus = BatchStatus.PENDING + files_processed: int = 0 + files_uploaded: int = 0 + files_failed: int = 0 + bytes_uploaded: int = 0 + started_at: datetime | None = None + completed_at: datetime | None = None + error_message: str = "" + + @property + def duration_seconds(self) -> float | None: + """Calculate batch processing duration in seconds.""" + if self.started_at and self.completed_at: + return (self.completed_at - self.started_at).total_seconds() + return None + + def to_dict(self) -> dict[str, Any]: + """Convert to dictionary for JSON serialization.""" + return { + "batch_id": self.batch_id, + "total_batches": self.total_batches, + "files_in_batch": self.files_in_batch, + "status": self.status.value, + "files_processed": self.files_processed, + "files_uploaded": self.files_uploaded, + "files_failed": self.files_failed, + "bytes_uploaded": self.bytes_uploaded, + "started_at": self.started_at.isoformat() if self.started_at else None, + "completed_at": self.completed_at.isoformat() if self.completed_at else None, + "duration_seconds": self.duration_seconds, + "error_message": self.error_message, + } + + +class WorkerAutoTuner: + """Monitors system resources and adjusts worker count dynamically.""" + + def __init__( + self, + target_cpu_percent: float = 70.0, + check_interval_seconds: float = 30.0, + max_workers: int = 16, + ) -> None: + """Initialize the auto-tuner. + + Args: + target_cpu_percent: Target CPU utilization (0-100) + check_interval_seconds: Time between adjustment checks + max_workers: Maximum allowed workers (hard ceiling) + """ + self.target_cpu_percent = target_cpu_percent + self.check_interval_seconds = check_interval_seconds + self.max_workers = max_workers + self.cpu_count = os.cpu_count() or 4 + self.last_check_time = 0.0 + self.history: list[dict[str, Any]] = [] + + def should_check(self) -> bool: + """Check if enough time has passed for another adjustment.""" + now = time.time() + return (now - self.last_check_time) >= self.check_interval_seconds + + def adjust_if_needed(self, current_workers: int) -> int: + """Adjust worker count based on current system utilization. + + Args: + current_workers: Current number of workers + + Returns: + Recommended worker count (may be same as current) + """ + if not self.should_check(): + return current_workers + + self.last_check_time = time.time() + + # Measure CPU and memory + try: + cpu_percent = psutil.cpu_percent(interval=1.0) + memory_info = psutil.virtual_memory() + memory_percent = memory_info.percent + + # Record metrics + self.history.append( + { + "timestamp": time.time(), + "cpu_percent": cpu_percent, + "memory_percent": memory_percent, + "workers": current_workers, + } + ) + + # Keep only last 10 measurements + if len(self.history) > 10: + self.history = self.history[-10:] + + logger.info( + f"Auto-tuner: CPU={cpu_percent:.1f}%, Memory={memory_percent:.1f}%, " + f"Workers={current_workers}" + ) + + # Start conservative on first check + if len(self.history) == 1: + recommended = min(4, self.cpu_count - 1, self.max_workers) + logger.info(f"Auto-tuner: Initial worker count: {recommended}") + return max(2, recommended) + + # Decrease if overloaded + if cpu_percent > self.target_cpu_percent + 10 or memory_percent > 85: + new_workers = max(2, current_workers - 1) + logger.info( + f"Auto-tuner: Decreasing workers {current_workers} → {new_workers} " + f"(CPU overload or low memory)" + ) + return new_workers + + # Increase if underutilized + if cpu_percent < self.target_cpu_percent - 10: + new_workers = min( + current_workers + 2, + self.cpu_count, + self.max_workers, + ) + if new_workers > current_workers: + logger.info( + f"Auto-tuner: Increasing workers {current_workers} → {new_workers} " + f"(CPU underutilized)" + ) + return new_workers + + # No change needed + return current_workers + + except Exception as e: + logger.warning(f"Auto-tuner: Error checking system resources: {e}") + return current_workers + + def get_metrics(self) -> dict[str, Any]: + """Get current tuning metrics.""" + if not self.history: + return { + "cpu_percent": None, + "memory_percent": None, + "workers": None, + "history_size": 0, + } + + latest = self.history[-1] + return { + "cpu_percent": latest["cpu_percent"], + "memory_percent": latest["memory_percent"], + "workers": latest["workers"], + "history_size": len(self.history), + "target_cpu_percent": self.target_cpu_percent, + } + + +def split_into_batches(items: list[Any], batch_size: int) -> list[list[Any]]: + """Split a list of items into batches of specified size. + + Args: + items: List of items to split + batch_size: Maximum items per batch + + Returns: + List of batches (each batch is a list of items) + + Example: + >>> split_into_batches([1, 2, 3, 4, 5], 2) + [[1, 2], [3, 4], [5]] + """ + if batch_size <= 0: + raise ValueError("batch_size must be positive") + + batches = [] + for i in range(0, len(items), batch_size): + batches.append(items[i : i + batch_size]) + + return batches + + +class BatchProcessor: + """Orchestrates batch-by-batch processing for large jobs.""" + + def __init__(self, config: BatchConfig) -> None: + """Initialize batch processor. + + Args: + config: Batch processing configuration + """ + self.config = config + self.tuner: WorkerAutoTuner | None = None + + if config.auto_tune_workers: + self.tuner = WorkerAutoTuner( + target_cpu_percent=config.target_cpu_percent, + check_interval_seconds=30.0, + max_workers=min(config.max_workers, 16), + ) + + def should_use_batch_processing(self, total_files: int) -> bool: + """Determine if batch processing should be used for this job. + + Args: + total_files: Total number of files in job + + Returns: + True if batch processing should be used + """ + if not self.config.enabled: + return False + + # Use batch processing for jobs exceeding threshold + return total_files >= self.config.large_job_threshold + + def create_batches(self, items: list[Any], batch_size: int | None = None) -> list[list[Any]]: + """Create batches from a list of items. + + Args: + items: Items to batch + batch_size: Override default batch size (optional) + + Returns: + List of batches + """ + size = batch_size if batch_size is not None else self.config.batch_size + return split_into_batches(items, size) + + def process_batches( + self, + items: list[Any], + process_fn: Callable[[list[Any], int, int], dict[str, Any]], + progress_callback: Callable[[BatchState], None] | None = None, + check_cancelled: Callable[[], bool] | None = None, + ) -> dict[str, Any]: + """Process items in batches with progress tracking. + + Args: + items: Items to process + process_fn: Function to process each batch, receives: + - batch items + - batch_id (0-indexed) + - total_batches + Returns dict with 'success', 'processed', 'failed', 'bytes_uploaded' + progress_callback: Optional callback for batch progress updates + check_cancelled: Optional function to check if job was cancelled + + Returns: + Summary dict with total stats: { + 'success': bool, + 'total_processed': int, + 'total_uploaded': int, + 'total_failed': int, + 'total_bytes': int, + 'batches_completed': int, + 'batches_failed': int, + 'duration_seconds': float + } + """ + batches = self.create_batches(items) + total_batches = len(batches) + + logger.info( + f"Batch processor: Processing {len(items)} items in {total_batches} batches " + f"(batch_size={self.config.batch_size})" + ) + + # Tracking stats + total_processed = 0 + total_uploaded = 0 + total_failed = 0 + total_bytes = 0 + batches_completed = 0 + batches_failed = 0 + + start_time = time.time() + + for batch_id, batch_items in enumerate(batches): + # Check for cancellation + if check_cancelled and check_cancelled(): + logger.info("Batch processor: Job cancelled by user") + break + + # Create batch state + batch_state = BatchState( + batch_id=batch_id, + total_batches=total_batches, + files_in_batch=len(batch_items), + status=BatchStatus.PROCESSING, + started_at=datetime.now(UTC), + ) + + # Notify progress callback + if progress_callback: + progress_callback(batch_state) + + try: + # Process this batch + result = process_fn(batch_items, batch_id, total_batches) + + # Update batch state with results + batch_state.files_processed = result.get("processed", 0) + batch_state.files_uploaded = result.get("uploaded", 0) + batch_state.files_failed = result.get("failed", 0) + batch_state.bytes_uploaded = result.get("bytes_uploaded", 0) + batch_state.status = ( + BatchStatus.COMPLETED if result.get("success") else BatchStatus.FAILED + ) + batch_state.completed_at = datetime.now(UTC) + + # Update totals + total_processed += batch_state.files_processed + total_uploaded += batch_state.files_uploaded + total_failed += batch_state.files_failed + total_bytes += batch_state.bytes_uploaded + + if batch_state.status == BatchStatus.COMPLETED: + batches_completed += 1 + else: + batches_failed += 1 + + logger.info( + f"Batch {batch_id + 1}/{total_batches} completed: " + f"{batch_state.files_uploaded} uploaded, {batch_state.files_failed} failed" + ) + + except Exception as e: + logger.error(f"Batch {batch_id + 1}/{total_batches} failed: {e}", exc_info=True) + batch_state.status = BatchStatus.FAILED + batch_state.error_message = str(e) + batch_state.completed_at = datetime.now(UTC) + batches_failed += 1 + + # Notify progress callback with final state + if progress_callback: + progress_callback(batch_state) + + # Auto-tune workers between batches + if self.tuner: + # This would be used by the caller to adjust ThreadPoolExecutor + new_workers = self.tuner.adjust_if_needed(self.config.max_workers) + if new_workers != self.config.max_workers: + logger.info(f"Auto-tuner recommends {new_workers} workers") + + # Calculate final stats + duration = time.time() - start_time + + summary = { + "success": batches_failed == 0, + "total_processed": total_processed, + "total_uploaded": total_uploaded, + "total_failed": total_failed, + "total_bytes": total_bytes, + "batches_completed": batches_completed, + "batches_failed": batches_failed, + "total_batches": total_batches, + "duration_seconds": duration, + } + + logger.info( + f"Batch processing complete: {batches_completed}/{total_batches} batches succeeded, " + f"{total_uploaded} items uploaded, {total_failed} failed, " + f"{duration:.1f}s elapsed" + ) + + return summary + + def get_tuner_metrics(self) -> dict[str, Any]: + """Get current auto-tuner metrics.""" + if self.tuner: + return self.tuner.get_metrics() + return {"enabled": False} diff --git a/app/services/job_storage.py b/app/services/job_storage.py new file mode 100644 index 0000000..4d3b7b3 --- /dev/null +++ b/app/services/job_storage.py @@ -0,0 +1,455 @@ +"""Database-backed storage for large upload/delete jobs.""" + +import json +import logging +import sqlite3 +import threading +from datetime import UTC, datetime, timedelta +from typing import Any + +from app.config import BASE_DIR + +logger = logging.getLogger(__name__) + +# Database file location +DB_FILE = BASE_DIR / "upload_jobs.db" + +# SQL schema +SCHEMA_SQL = """ +CREATE TABLE IF NOT EXISTS upload_jobs ( + job_id TEXT PRIMARY KEY, + job_type TEXT NOT NULL, + status TEXT NOT NULL, + total_files INTEGER NOT NULL, + files_processed INTEGER DEFAULT 0, + files_uploaded INTEGER DEFAULT 0, + files_failed INTEGER DEFAULT 0, + total_bytes INTEGER DEFAULT 0, + created_at TEXT NOT NULL, + started_at TEXT, + completed_at TEXT, + metadata TEXT +); + +CREATE TABLE IF NOT EXISTS upload_files ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + job_id TEXT NOT NULL, + filename TEXT NOT NULL, + local_path TEXT NOT NULL, + file_size INTEGER NOT NULL, + status TEXT NOT NULL, + s3_path TEXT, + start_time TEXT, + bytes_uploaded INTEGER DEFAULT 0, + error_message TEXT, + is_duplicate INTEGER DEFAULT 0, + is_valid INTEGER DEFAULT 1, + upload_started_at TEXT, + upload_completed_at TEXT, + FOREIGN KEY (job_id) REFERENCES upload_jobs(job_id) +); + +CREATE INDEX IF NOT EXISTS idx_upload_files_job_id ON upload_files(job_id); +CREATE INDEX IF NOT EXISTS idx_upload_files_status ON upload_files(status); +CREATE INDEX IF NOT EXISTS idx_upload_jobs_created_at ON upload_jobs(created_at); +""" + + +class JobStorage: + """SQLite-backed storage for upload/delete jobs.""" + + _instance: "JobStorage | None" = None + _lock = threading.Lock() + + def __new__(cls) -> "JobStorage": + """Singleton pattern to ensure only one storage instance.""" + if cls._instance is None: + with cls._lock: + if cls._instance is None: + cls._instance = super().__new__(cls) + cls._instance._initialize_db() + return cls._instance + + def _initialize_db(self) -> None: + """Initialize the database and create tables if needed.""" + try: + conn = sqlite3.connect(str(DB_FILE), check_same_thread=False) + conn.executescript(SCHEMA_SQL) + conn.commit() + conn.close() + logger.info(f"Job storage database initialized at {DB_FILE}") + except Exception as e: + logger.error(f"Failed to initialize job storage database: {e}", exc_info=True) + raise + + def _get_connection(self) -> sqlite3.Connection: + """Get a database connection.""" + conn = sqlite3.connect(str(DB_FILE), check_same_thread=False) + conn.row_factory = sqlite3.Row + return conn + + def save_job( + self, + job_id: str, + job_type: str, + total_files: int, + file_states: list[dict[str, Any]], + metadata: dict[str, Any] | None = None, + ) -> None: + """Save a new job to the database. + + Args: + job_id: Unique job identifier + job_type: Type of job ('upload' or 'delete') + total_files: Total number of files in job + file_states: List of file state dictionaries + metadata: Optional job metadata + """ + try: + conn = self._get_connection() + cursor = conn.cursor() + + # Insert job record + cursor.execute( + """ + INSERT INTO upload_jobs ( + job_id, job_type, status, total_files, created_at, metadata + ) + VALUES (?, ?, ?, ?, ?, ?) + """, + ( + job_id, + job_type, + "pending", + total_files, + datetime.now(UTC).isoformat(), + json.dumps(metadata) if metadata else None, + ), + ) + + # Insert file records + for file_state in file_states: + cursor.execute( + """ + INSERT INTO upload_files ( + job_id, filename, local_path, file_size, status, + s3_path, start_time, is_duplicate, is_valid + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + job_id, + file_state.get("filename", ""), + file_state.get("local_path", ""), + file_state.get("file_size", 0), + file_state.get("status", "pending"), + file_state.get("s3_path", ""), + file_state.get("start_time"), + 1 if file_state.get("is_duplicate", False) else 0, + 1 if file_state.get("is_valid", True) else 0, + ), + ) + + conn.commit() + conn.close() + logger.info(f"Saved job {job_id} with {len(file_states)} files to database") + + except Exception as e: + logger.error(f"Failed to save job {job_id}: {e}", exc_info=True) + raise + + def update_job_status( + self, + job_id: str, + status: str, + files_processed: int | None = None, + files_uploaded: int | None = None, + files_failed: int | None = None, + total_bytes: int | None = None, + started_at: datetime | None = None, + completed_at: datetime | None = None, + ) -> None: + """Update job status and statistics. + + Args: + job_id: Job identifier + status: New job status + files_processed: Number of files processed (optional) + files_uploaded: Number of files uploaded (optional) + files_failed: Number of files failed (optional) + total_bytes: Total bytes uploaded (optional) + started_at: Job start time (optional) + completed_at: Job completion time (optional) + """ + try: + conn = self._get_connection() + cursor = conn.cursor() + + # Build dynamic UPDATE query + updates = ["status = ?"] + params: list[Any] = [status] + + if files_processed is not None: + updates.append("files_processed = ?") + params.append(files_processed) + + if files_uploaded is not None: + updates.append("files_uploaded = ?") + params.append(files_uploaded) + + if files_failed is not None: + updates.append("files_failed = ?") + params.append(files_failed) + + if total_bytes is not None: + updates.append("total_bytes = ?") + params.append(total_bytes) + + if started_at is not None: + updates.append("started_at = ?") + params.append(started_at.isoformat()) + + if completed_at is not None: + updates.append("completed_at = ?") + params.append(completed_at.isoformat()) + + params.append(job_id) + + query = f"UPDATE upload_jobs SET {', '.join(updates)} WHERE job_id = ?" + cursor.execute(query, params) + + conn.commit() + conn.close() + + except Exception as e: + logger.error(f"Failed to update job {job_id}: {e}", exc_info=True) + raise + + def update_file_status( + self, + job_id: str, + filename: str, + status: str, + bytes_uploaded: int | None = None, + error_message: str | None = None, + upload_started_at: datetime | None = None, + upload_completed_at: datetime | None = None, + ) -> None: + """Update status of a specific file in a job. + + Args: + job_id: Job identifier + filename: Filename to update + status: New file status + bytes_uploaded: Bytes uploaded (optional) + error_message: Error message (optional) + upload_started_at: Upload start time (optional) + upload_completed_at: Upload completion time (optional) + """ + try: + conn = self._get_connection() + cursor = conn.cursor() + + # Build dynamic UPDATE query + updates = ["status = ?"] + params: list[Any] = [status] + + if bytes_uploaded is not None: + updates.append("bytes_uploaded = ?") + params.append(bytes_uploaded) + + if error_message is not None: + updates.append("error_message = ?") + params.append(error_message) + + if upload_started_at is not None: + updates.append("upload_started_at = ?") + params.append(upload_started_at.isoformat()) + + if upload_completed_at is not None: + updates.append("upload_completed_at = ?") + params.append(upload_completed_at.isoformat()) + + params.extend([job_id, filename]) + + query = ( + f"UPDATE upload_files SET {', '.join(updates)} WHERE job_id = ? AND filename = ?" + ) + cursor.execute(query, params) + + conn.commit() + conn.close() + + except Exception as e: + logger.error(f"Failed to update file {filename} in job {job_id}: {e}", exc_info=True) + + def get_job(self, job_id: str) -> dict[str, Any] | None: + """Get job metadata by ID. + + Args: + job_id: Job identifier + + Returns: + Job metadata dict or None if not found + """ + try: + conn = self._get_connection() + cursor = conn.cursor() + + cursor.execute( + """ + SELECT job_id, job_type, status, total_files, + files_processed, files_uploaded, files_failed, total_bytes, + created_at, started_at, completed_at, metadata + FROM upload_jobs + WHERE job_id = ? + """, + (job_id,), + ) + + row = cursor.fetchone() + conn.close() + + if row is None: + return None + + return { + "job_id": row["job_id"], + "job_type": row["job_type"], + "status": row["status"], + "total_files": row["total_files"], + "files_processed": row["files_processed"], + "files_uploaded": row["files_uploaded"], + "files_failed": row["files_failed"], + "total_bytes": row["total_bytes"], + "created_at": row["created_at"], + "started_at": row["started_at"], + "completed_at": row["completed_at"], + "metadata": json.loads(row["metadata"]) if row["metadata"] else {}, + } + + except Exception as e: + logger.error(f"Failed to get job {job_id}: {e}", exc_info=True) + return None + + def get_job_results(self, job_id: str, page: int = 1, per_page: int = 100) -> dict[str, Any]: + """Get paginated file results for a job. + + Args: + job_id: Job identifier + page: Page number (1-indexed) + per_page: Results per page (default 100) + + Returns: + Dict with 'files' list and pagination info + """ + try: + conn = self._get_connection() + cursor = conn.cursor() + + # Get total count + cursor.execute("SELECT COUNT(*) as count FROM upload_files WHERE job_id = ?", (job_id,)) + total_files = cursor.fetchone()["count"] + + # Get paginated files + offset = (page - 1) * per_page + cursor.execute( + """ + SELECT filename, local_path, file_size, status, s3_path, + start_time, bytes_uploaded, error_message, + is_duplicate, is_valid, upload_started_at, upload_completed_at + FROM upload_files + WHERE job_id = ? + ORDER BY id + LIMIT ? OFFSET ? + """, + (job_id, per_page, offset), + ) + + files = [] + for row in cursor.fetchall(): + files.append( + { + "filename": row["filename"], + "local_path": row["local_path"], + "file_size": row["file_size"], + "status": row["status"], + "s3_path": row["s3_path"], + "start_time": row["start_time"], + "bytes_uploaded": row["bytes_uploaded"], + "error_message": row["error_message"], + "is_duplicate": bool(row["is_duplicate"]), + "is_valid": bool(row["is_valid"]), + "upload_started_at": row["upload_started_at"], + "upload_completed_at": row["upload_completed_at"], + } + ) + + conn.close() + + total_pages = (total_files + per_page - 1) // per_page + + return { + "job_id": job_id, + "files": files, + "pagination": { + "page": page, + "per_page": per_page, + "total_files": total_files, + "total_pages": total_pages, + "has_next": page < total_pages, + "has_prev": page > 1, + }, + } + + except Exception as e: + logger.error(f"Failed to get results for job {job_id}: {e}", exc_info=True) + return {"job_id": job_id, "files": [], "pagination": {}} + + def cleanup_old_jobs(self, days: int = 7) -> int: + """Delete jobs older than specified days. + + Args: + days: Age threshold in days + + Returns: + Number of jobs deleted + """ + try: + cutoff_date = datetime.now(UTC) - timedelta(days=days) + conn = self._get_connection() + cursor = conn.cursor() + + # Delete old files first (foreign key constraint) + cursor.execute( + """ + DELETE FROM upload_files + WHERE job_id IN ( + SELECT job_id FROM upload_jobs + WHERE created_at < ? + ) + """, + (cutoff_date.isoformat(),), + ) + + # Delete old jobs + cursor.execute( + "DELETE FROM upload_jobs WHERE created_at < ?", + (cutoff_date.isoformat(),), + ) + + deleted_count = cursor.rowcount + conn.commit() + conn.close() + + logger.info(f"Cleaned up {deleted_count} jobs older than {days} days") + return deleted_count + + except Exception as e: + logger.error(f"Failed to cleanup old jobs: {e}", exc_info=True) + return 0 + + +def get_job_storage() -> JobStorage: + """Get the singleton JobStorage instance.""" + return JobStorage() From fc545ee43ce56cc4472290a7b9f96c69cf101639 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Mon, 2 Mar 2026 07:03:13 -0700 Subject: [PATCH 139/173] Dev: Add psutil requirement --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 8f31002..3db7f7f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,4 @@ python-dotenv>=1.0.0 gunicorn>=21.0.0 modaq_toolkit[mcap] @ git+https://github.com/MODAQ2/MODAQ_toolkit.git mcap-ros2-support +psutil>=5.9.0 From fe3b802041b3e416c482a9f81597bfb8d2c290c9 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Mon, 2 Mar 2026 07:04:06 -0700 Subject: [PATCH 140/173] Config: Add batch processing spec --- app/config.py | 58 ++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 48 insertions(+), 10 deletions(-) diff --git a/app/config.py b/app/config.py index 4f78506..e79d22e 100644 --- a/app/config.py +++ b/app/config.py @@ -58,6 +58,7 @@ class Settings: _instance: "Settings | None" = None _settings: dict[str, Any] + _provenance: dict[str, dict[str, str]] def __new__(cls) -> "Settings": """Singleton pattern to ensure only one settings instance exists.""" @@ -83,34 +84,58 @@ def _load_settings(self) -> None: "default_upload_folder": "", "display_name": "MODAQ Uploader", "log_directory": "logs", + "batch_processing": { + "enabled": True, + "batch_size": 100, + "auto_tune_workers": True, + "max_workers": 4, + "target_cpu_percent": 70.0, + "skip_mcap_validation": False, + "use_database_for_large_jobs": True, + "large_job_threshold": 1000, + }, } + # Track the source of each setting value as it is applied layer by layer. + provenance: dict[str, dict[str, str]] = {k: {"source": "builtin"} for k in defaults} + # Load from settings.default.json if it exists if SETTINGS_DEFAULT_FILE.exists(): with open(SETTINGS_DEFAULT_FILE, encoding="utf-8") as f: - defaults.update(json.load(f)) + default_data = json.load(f) + defaults.update(default_data) + for k in default_data: + provenance[k] = {"source": "default_file", "path": str(SETTINGS_DEFAULT_FILE)} # Load from settings.json if it exists if SETTINGS_FILE.exists(): with open(SETTINGS_FILE, encoding="utf-8") as f: - defaults.update(json.load(f)) + user_data = json.load(f) + defaults.update(user_data) + for k in user_data: + provenance[k] = {"source": "settings_file", "path": str(SETTINGS_FILE)} # Override with environment variables (highest priority) - env_overrides = { - "aws_profile": os.environ.get(ENV_AWS_PROFILE), - "aws_region": os.environ.get(ENV_AWS_REGION), - "s3_bucket": os.environ.get(ENV_S3_BUCKET), - "default_upload_folder": os.environ.get(ENV_DEFAULT_UPLOAD_FOLDER), - "display_name": os.environ.get(ENV_DISPLAY_NAME), - "log_directory": os.environ.get(ENV_LOG_DIRECTORY), + env_overrides: dict[str, tuple[str | None, str]] = { + "aws_profile": (os.environ.get(ENV_AWS_PROFILE), ENV_AWS_PROFILE), + "aws_region": (os.environ.get(ENV_AWS_REGION), ENV_AWS_REGION), + "s3_bucket": (os.environ.get(ENV_S3_BUCKET), ENV_S3_BUCKET), + "default_upload_folder": ( + os.environ.get(ENV_DEFAULT_UPLOAD_FOLDER), + ENV_DEFAULT_UPLOAD_FOLDER, + ), + "display_name": (os.environ.get(ENV_DISPLAY_NAME), ENV_DISPLAY_NAME), + "log_directory": (os.environ.get(ENV_LOG_DIRECTORY), ENV_LOG_DIRECTORY), } # Only apply non-None environment values - for key, value in env_overrides.items(): + for key, (value, env_var) in env_overrides.items(): if value is not None: defaults[key] = value + provenance[key] = {"source": "env", "env_var": env_var} self._settings = defaults + self._provenance = provenance # Save settings.json if it doesn't exist if not SETTINGS_FILE.exists(): @@ -139,6 +164,10 @@ def all(self) -> dict[str, Any]: """Get all settings as a dictionary.""" return self._settings.copy() + def to_response(self) -> dict[str, Any]: + """Return settings with value_sources provenance for API responses.""" + return {**self._settings, "value_sources": self._provenance} + def reload(self) -> None: """Reload settings from file.""" self._load_settings() @@ -177,6 +206,15 @@ def log_directory(self) -> Path: path = BASE_DIR / path return path + @property + def batch_processing(self) -> dict[str, Any]: + """Get batch processing configuration.""" + return dict(self._settings.get("batch_processing", {})) + + def get_batch_config(self) -> dict[str, Any]: + """Get batch processing configuration (alias for batch_processing property).""" + return self.batch_processing + def get_settings() -> Settings: """Get the singleton Settings instance.""" From e4c45b6d5c3c71190904b473555b771f8d51bcd8 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Mon, 2 Mar 2026 07:04:38 -0700 Subject: [PATCH 141/173] Backend: Handle delete in 1k file batches --- app/routes/delete.py | 103 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 83 insertions(+), 20 deletions(-) diff --git a/app/routes/delete.py b/app/routes/delete.py index 8c1eb2f..7f4b518 100644 --- a/app/routes/delete.py +++ b/app/routes/delete.py @@ -77,15 +77,17 @@ def scan_folder() -> tuple[Response, int]: total_size = sum(f.file_size for f in job.files) has_permission_issues = any(not f.writable for f in job.files) - return jsonify({ - "success": True, - "job_id": job.job_id, - "folder_path": str(folder_path.absolute()), - "files": [f.to_dict() for f in job.files], - "total_files": len(job.files), - "total_size": total_size, - "permission_warning": has_permission_issues, - }), 200 + return jsonify( + { + "success": True, + "job_id": job.job_id, + "folder_path": str(folder_path.absolute()), + "files": [f.to_dict() for f in job.files], + "total_files": len(job.files), + "total_size": total_size, + "permission_warning": has_permission_issues, + } + ), 200 @delete_bp.route("/start/", methods=["POST"]) @@ -110,9 +112,11 @@ def progress_callback(job: DeleteJob) -> None: if job.status in ("completed", "failed", "cancelled"): _send_sse_event(job.job_id, {"type": "delete_complete", **job.to_dict()}) else: - _send_sse_event( - job.job_id, {"type": "delete_progress", **job.to_progress_dict()} - ) + _send_sse_event(job.job_id, {"type": "delete_progress", **job.to_progress_dict()}) + + def batch_callback(batch_event: dict[str, Any]) -> None: + """Send batch-level events via SSE.""" + _send_sse_event(job_id, batch_event) def run_delete() -> None: manager.start_delete_job( @@ -120,6 +124,7 @@ def run_delete() -> None: settings.aws_profile, settings.aws_region, progress_callback=progress_callback, + batch_callback=batch_callback, ) thread = threading.Thread(target=run_delete, daemon=True) @@ -212,6 +217,64 @@ def get_status(job_id: str) -> tuple[Response, int]: return jsonify(job.to_dict()), 200 +@delete_bp.route("/results/", methods=["GET"]) +def get_results(job_id: str) -> tuple[Response, int]: + """Get paginated results for a completed delete job. + + For large jobs (>1000 files), results can be retrieved in pages + to prevent memory overload and large response payloads. + + Args: + job_id: The delete job to get results for + + Query parameters: + page: Page number (default: 1) + per_page: Results per page (default: 100, max: 500) + + Returns: + JSON response with paginated file results and job metadata + """ + manager = get_delete_manager() + + # Get the job + job = manager.get_job(job_id) + if not job: + return jsonify({"error": "Job not found"}), 404 + + # Parse pagination parameters + page = request.args.get("page", 1, type=int) + per_page = min(request.args.get("per_page", 100, type=int), 500) + + # Calculate pagination + total_files = len(job.files) + total_pages = (total_files + per_page - 1) // per_page + offset = (page - 1) * per_page + paginated_files = job.files[offset : offset + per_page] + + # Build response + return jsonify( + { + "job_id": job_id, + "files": [f.to_dict() for f in paginated_files], + "pagination": { + "page": page, + "per_page": per_page, + "total_files": total_files, + "total_pages": total_pages, + "has_next": page < total_pages, + "has_prev": page > 1, + }, + "job_metadata": { + "job_id": job.job_id, + "status": job.status, + "total_files": total_files, + "status_counts": job.to_progress_dict().get("status_counts", {}), + "total_deleted_size": job.to_progress_dict().get("total_deleted_size", 0), + }, + } + ), 200 + + @delete_bp.route("/cancel/", methods=["POST"]) def cancel_delete(job_id: str) -> tuple[Response, int]: """Cancel a delete job. @@ -226,11 +289,13 @@ def cancel_delete(job_id: str) -> tuple[Response, int]: if manager.cancel_job(job_id): job = manager.get_job(job_id) - return jsonify({ - "success": True, - "job_id": job_id, - "job": job.to_dict() if job else None, - }), 200 + return jsonify( + { + "success": True, + "job_id": job_id, + "job": job.to_dict() if job else None, + } + ), 200 return jsonify({"error": "Job not found"}), 404 @@ -269,9 +334,7 @@ def fix_permissions() -> tuple[Response, int]: try: resolved = folder_path.resolve() if not str(resolved).startswith("/media/"): - return jsonify( - {"error": "Permission fix is only allowed for paths under /media/"} - ), 403 + return jsonify({"error": "Permission fix is only allowed for paths under /media/"}), 403 except (OSError, ValueError): return jsonify({"error": "Invalid path"}), 400 From cc23c6ade6b56c2db921043f97dda54550b3cdbe Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Mon, 2 Mar 2026 07:05:17 -0700 Subject: [PATCH 142/173] Dev: Lint --- app/routes/files.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/app/routes/files.py b/app/routes/files.py index fddc02d..3237814 100644 --- a/app/routes/files.py +++ b/app/routes/files.py @@ -175,9 +175,9 @@ def _walk_error(err: OSError) -> None: except OSError: continue - uploaded = cache.check_exists_by_filename( - bucket, mcap_path.name, file_stat.st_size - ) is True + uploaded = ( + cache.check_exists_by_filename(bucket, mcap_path.name, file_stat.st_size) is True + ) if len(parts) == 1: # Direct child MCAP file @@ -276,9 +276,7 @@ def _walk_error(err: OSError) -> None: # If it's a user directory (e.g. /media/username), list its children try: children = [ - c - for c in entry.iterdir() - if c.is_dir() and not c.name.startswith(".") + c for c in entry.iterdir() if c.is_dir() and not c.name.startswith(".") ] except PermissionError: children = [] From 7b4d930164ef9136e7d79244cdcc6114b20924e4 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Mon, 2 Mar 2026 07:06:08 -0700 Subject: [PATCH 143/173] Backend: Do a smart merge of settings from different locations --- app/routes/settings.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/app/routes/settings.py b/app/routes/settings.py index 7f2d2d0..f58fb9a 100644 --- a/app/routes/settings.py +++ b/app/routes/settings.py @@ -4,6 +4,7 @@ import signal import sys import threading +from typing import Any from flask import Blueprint, Response, jsonify, request @@ -23,7 +24,7 @@ def get_all_settings() -> tuple[Response, int]: JSON response with all settings """ settings = get_settings() - return jsonify(settings.all()), 200 + return jsonify(settings.to_response()), 200 @settings_bp.route("", methods=["PUT"]) @@ -45,12 +46,17 @@ def update_settings() -> tuple[Response, int]: settings = get_settings() - # Validate settings - allowed_keys = { - "aws_profile", "aws_region", "s3_bucket", "default_upload_folder", "display_name", - "log_directory", - } - filtered_data = {k: v for k, v in data.items() if k in allowed_keys} + # Accept any key present in the current settings schema; deep-merge nested dicts + # (e.g. batch_processing) so callers can send partial sub-objects. + current = settings.all() + filtered_data: dict[str, Any] = {} + for k, v in data.items(): + if k not in current: + continue + if isinstance(current[k], dict) and isinstance(v, dict): + filtered_data[k] = {**current[k], **v} + else: + filtered_data[k] = v if not filtered_data: return jsonify({"error": "No valid settings provided"}), 400 @@ -65,7 +71,7 @@ def update_settings() -> tuple[Response, int]: {"changed_keys": list(filtered_data.keys())}, ) - return jsonify(settings.all()), 200 + return jsonify(settings.to_response()), 200 @settings_bp.route("/profiles", methods=["GET"]) From 094945ebe20662f86c5a936abb3011c03b17413e Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Mon, 2 Mar 2026 07:07:04 -0700 Subject: [PATCH 144/173] Backend: Refactor upload to be more efficient --- app/routes/upload.py | 183 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 173 insertions(+), 10 deletions(-) diff --git a/app/routes/upload.py b/app/routes/upload.py index e93daa4..cd4cb74 100644 --- a/app/routes/upload.py +++ b/app/routes/upload.py @@ -23,16 +23,55 @@ # Store for SSE clients per job _sse_queues: dict[str, list[deque[dict[str, Any]]]] = {} +_sse_events: dict[str, threading.Event] = {} # Events to signal new data (replaces polling) +_sse_timestamps: dict[str, float] = {} # Track last activity for TTL cleanup _sse_lock = threading.Lock() +# Configuration +SSE_QUEUE_TTL_SECONDS = 3600 # Clean up queues after 1 hour of inactivity +SSE_HEARTBEAT_INTERVAL_SECONDS = 15 # Send heartbeat every 15 seconds + + +def _cleanup_old_sse_queues() -> int: + """Clean up SSE queues that haven't been accessed recently. + + Returns: + Number of queues removed + """ + now = time.time() + removed = 0 + with _sse_lock: + expired = [ + job_id + for job_id, timestamp in _sse_timestamps.items() + if now - timestamp > SSE_QUEUE_TTL_SECONDS + ] + for job_id in expired: + _sse_queues.pop(job_id, None) + _sse_events.pop(job_id, None) + _sse_timestamps.pop(job_id, None) + removed += 1 + return removed + def send_sse_event(job_id: str, data: dict[str, Any]) -> None: - """Send an SSE event to all clients listening for a job.""" + """Send an SSE event to all clients listening for a job. + + This wakes up all waiting SSE generators via the event signal, + avoiding busy-wait polling. + """ with _sse_lock: queues = _sse_queues.get(job_id, []) for q in queues: q.append(data) + # Update last activity timestamp + _sse_timestamps[job_id] = time.time() + + # Signal waiting threads that new data is available + if job_id in _sse_events: + _sse_events[job_id].set() + def _make_analysis_callback( job_id: str, @@ -187,6 +226,9 @@ def run_upload() -> None: def get_progress(job_id: str) -> Response: """Stream progress updates for a job via Server-Sent Events. + Uses event-driven signaling (instead of polling) and periodic heartbeats + to efficiently detect client disconnects. + Args: job_id: The job ID to monitor @@ -196,12 +238,19 @@ def get_progress(job_id: str) -> Response: manager = get_upload_manager() def generate() -> Generator[str, None, None]: - # Create a queue for this client + # Create a queue for this client and an event for signaling queue: deque[dict[str, Any]] = deque() + event = threading.Event() + with _sse_lock: if job_id not in _sse_queues: _sse_queues[job_id] = [] _sse_queues[job_id].append(queue) + _sse_events[job_id] = event + _sse_timestamps[job_id] = time.time() + + # Periodic cleanup of old queues + _cleanup_old_sse_queues() try: # Send initial state @@ -216,15 +265,33 @@ def generate() -> Generator[str, None, None]: yield f"data: {json.dumps(job.to_dict())}\n\n" else: yield f"data: {json.dumps(job.to_progress_dict())}\n\n" + # Replay per-file states for files already past PENDING. + # Covers the race window where ANALYZING events fired + # before the EventSource connected. + analysis_complete = job.status.value in ("ready", "failed") + for fs in job.files: + if fs.status != UploadStatus.PENDING: + replay = { + "type": "analysis_progress", + "job_id": job.job_id, + "job_status": job.status.value, + "file": fs.to_dict(), + "total_files": len(job.files), + "analysis_complete": analysis_complete, + } + yield f"data: {json.dumps(replay)}\n\n" elif scan_job: yield f"data: {json.dumps({'type': 'scan_initial', 'status': scan_job.status})}\n\n" - # Stream updates + last_heartbeat_time = time.time() + + # Stream updates with event-driven waiting (no polling) while True: - # Check for updates + # Process all queued events while queue: data = queue.popleft() yield f"data: {json.dumps(data)}\n\n" + last_heartbeat_time = time.time() # Check if job is complete (upload jobs) if data.get("status") in ("completed", "failed", "cancelled"): @@ -235,8 +302,15 @@ def generate() -> Generator[str, None, None]: if not data.get("type"): return - # Small delay to prevent busy waiting - time.sleep(0.1) + # Send heartbeat if no activity for a while + now = time.time() + if now - last_heartbeat_time > SSE_HEARTBEAT_INTERVAL_SECONDS: + yield ": heartbeat\n\n" # Comment line, ignored by EventSource + last_heartbeat_time = now + + # Wait for signal (blocking, no CPU waste) with timeout for heartbeat + event.wait(timeout=SSE_HEARTBEAT_INTERVAL_SECONDS) + event.clear() # Check if job still exists (upload or scan) job = manager.get_job(job_id) @@ -281,12 +355,15 @@ def generate() -> Generator[str, None, None]: return finally: - # Clean up queue + # Clean up queue and event with _sse_lock: if job_id in _sse_queues and queue in _sse_queues[job_id]: _sse_queues[job_id].remove(queue) if not _sse_queues[job_id]: - del _sse_queues[job_id] + # Last client disconnected, remove event too + _sse_queues.pop(job_id, None) + _sse_events.pop(job_id, None) + # Keep timestamp for TTL cleanup return Response( generate(), @@ -318,6 +395,68 @@ def get_status(job_id: str) -> tuple[Response, int]: return jsonify(job.to_dict()), 200 +@upload_bp.route("/results/", methods=["GET"]) +def get_results(job_id: str) -> tuple[Response, int]: + """Get paginated results for a completed job. + + For large jobs (>1000 files), results are stored in database and retrieved + in pages to prevent memory overload and large response payloads. + + Args: + job_id: The job ID to get results for + + Query parameters: + page: Page number (default: 1) + per_page: Results per page (default: 100, max: 500) + + Returns: + JSON response with paginated file results and job metadata + """ + from app.services.job_storage import get_job_storage + + manager = get_upload_manager() + storage = get_job_storage() + + # Try to get from database first (for large jobs) + db_job = storage.get_job(job_id) + if db_job: + page = request.args.get("page", 1, type=int) + per_page = min(request.args.get("per_page", 100, type=int), 500) + + results = storage.get_job_results(job_id, page=page, per_page=per_page) + results["job_metadata"] = db_job + return jsonify(results), 200 + + # Fall back to in-memory job (for small jobs) + job = manager.get_job(job_id) + if not job: + return jsonify({"error": "Job not found"}), 404 + + # Return in-memory job with all files (small jobs only) + return jsonify( + { + "job_id": job_id, + "files": [f.to_dict() for f in job.files], + "pagination": { + "page": 1, + "per_page": len(job.files), + "total_files": len(job.files), + "total_pages": 1, + "has_next": False, + "has_prev": False, + }, + "job_metadata": { + "job_id": job.job_id, + "status": job.status.value, + "total_files": len(job.files), + "files_uploaded": sum(1 for f in job.files if f.status == UploadStatus.COMPLETED), + "files_failed": job.files_failed, + "total_bytes": job.total_bytes, + }, + } + ), 200 + + @upload_bp.route("/active", methods=["GET"]) def get_active_job() -> tuple[Response, int]: """Get the most recent active job (for state restoration on page refresh). @@ -345,6 +484,27 @@ def get_active_job() -> tuple[Response, int]: return jsonify({"job_id": None, "job": None}), 200 +@upload_bp.route("/cleanup-sse", methods=["POST"]) +def cleanup_sse_queues() -> tuple[Response, int]: + """Clean up stale SSE queues and events. + + This can be called periodically or manually to reclaim memory from + abandoned connections. Returns the number of queues removed. + + Returns: + JSON response with cleanup statistics + """ + removed = _cleanup_old_sse_queues() + return jsonify( + { + "success": True, + "queues_removed": removed, + "active_queues": len(_sse_queues), + "ttl_seconds": SSE_QUEUE_TTL_SECONDS, + } + ), 200 + + @upload_bp.route("/cancel/", methods=["POST"]) def cancel_upload(job_id: str) -> tuple[Response, int]: """Cancel an upload job. @@ -538,8 +698,11 @@ def bulk_analyze() -> tuple[Response, int]: } ), 200 - # When force-reuploading, analyze ALL files (not just non-duplicates) - job_files = file_paths if not skip_duplicates else files_to_analyze + # Always include all user-selected files in the job. The pipeline marks + # already-uploaded files as "skipped" when skip_duplicates=True rather than + # silently dropping them — an empty job_files here causes a 0-file job that + # completes instantly and bypasses the upload screen entirely. + job_files = file_paths # Create job with files that need analysis (no temp_dir - direct file access) job = manager.create_job(job_files, auto_upload=auto_upload) From 6b38249787b736a18c9c2decccebae138d16a05e Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Mon, 2 Mar 2026 07:07:33 -0700 Subject: [PATCH 145/173] Backend: Handle delete in batches --- app/services/delete_manager.py | 195 ++++++++++++++++++++++++++++++++- 1 file changed, 189 insertions(+), 6 deletions(-) diff --git a/app/services/delete_manager.py b/app/services/delete_manager.py index f5d3b21..cbb8680 100644 --- a/app/services/delete_manager.py +++ b/app/services/delete_manager.py @@ -167,9 +167,24 @@ def is_multipart_etag(etag: str) -> bool: class DeleteManager: """Manages local file deletion jobs with MD5 verification against S3.""" - def __init__(self) -> None: + def __init__(self, batch_config: dict[str, Any] | None = None) -> None: self.jobs: dict[str, DeleteJob] = {} + # Load batch processing configuration + if batch_config is None: + from app.config import get_settings + + settings = get_settings() + batch_config = settings.get_batch_config() + + # Import BatchConfig and BatchProcessor + from app.services.batch_processor import BatchConfig, BatchProcessor + + self.batch_config = BatchConfig.from_dict(batch_config) + self.batch_processor = ( + BatchProcessor(self.batch_config) if self.batch_config.enabled else None + ) + def scan_folder( self, folder_path: str, @@ -237,6 +252,7 @@ def start_delete_job( aws_profile: str, aws_region: str, progress_callback: Callable[[DeleteJob], None] | None = None, + batch_callback: Callable[[dict[str, Any]], None] | None = None, ) -> bool: """Start verification and deletion for a job. @@ -249,6 +265,7 @@ def start_delete_job( aws_profile: AWS profile name aws_region: AWS region progress_callback: Called after each file status change + batch_callback: Called for batch-level events (batch_started, batch_completed) Returns: True if job was started @@ -328,9 +345,7 @@ def verify_against_s3(file_state: FileDeleteState) -> None: return try: - metadata = get_object_metadata( - s3_client, file_state.s3_bucket, file_state.s3_path - ) + metadata = get_object_metadata(s3_client, file_state.s3_bucket, file_state.s3_path) if not metadata["success"]: with job.lock: file_state.status = DeleteStatus.FAILED @@ -379,8 +394,50 @@ def verify_against_s3(file_state: FileDeleteState) -> None: if progress_callback: progress_callback(job) - with ThreadPoolExecutor(max_workers=4) as executor: - executor.map(verify_against_s3, job.files) + # Phase 2: Verify files against S3 + # Use batch processing for large jobs to improve performance and UI responsiveness + use_batch_processing = ( + self.batch_processor is not None + and self.batch_processor.should_use_batch_processing(len(job.files)) + ) + + if use_batch_processing and self.batch_processor: + log.info( + "delete", + "using_batch_processing", + f"Using batch processing for {len(job.files)} files", + {"job_id": job_id, "total_files": len(job.files)}, + ) + + # Process verification in batches + def check_cancelled() -> bool: + return job.cancelled + + # Create a wrapper that uses _verify_batch + def process_batch_fn( + batch_files: list[FileDeleteState], batch_id: int, total_batches: int + ) -> dict[str, Any]: + return self._verify_batch( + batch_files, + batch_id, + total_batches, + job, + s3_client, + progress_callback, + batch_callback, + ) + + # Use batch processor + self.batch_processor.process_batches( + job.files, + process_batch_fn, + progress_callback=None, # We handle progress in _verify_batch + check_cancelled=check_cancelled, + ) + else: + # Traditional non-batch processing + with ThreadPoolExecutor(max_workers=4) as executor: + executor.map(verify_against_s3, job.files) if job.cancelled: self._finalize_cancelled(job, progress_callback) @@ -451,6 +508,132 @@ def verify_against_s3(file_state: FileDeleteState) -> None: return True + def _verify_batch( + self, + batch_files: list[FileDeleteState], + batch_id: int, + total_batches: int, + job: DeleteJob, + s3_client: Any, + progress_callback: Callable[[DeleteJob], None] | None, + batch_callback: Callable[[dict[str, Any]], None] | None, + ) -> dict[str, Any]: + """Verify a batch of files against S3. + + Args: + batch_files: Files to verify in this batch + batch_id: 0-indexed batch number + total_batches: Total number of batches + job: The delete job + s3_client: S3 client for HEAD requests + progress_callback: Called after each file verification + batch_callback: Called for batch-level events + + Returns: + Dict with batch statistics + """ + # Send batch_started event + if batch_callback: + batch_callback( + { + "type": "batch_started", + "batch_id": batch_id, + "total_batches": total_batches, + "files_in_batch": len(batch_files), + } + ) + + verified = 0 + failed = 0 + + def verify_against_s3(file_state: FileDeleteState) -> None: + """Verify a file against S3 using HEAD + size (primary) and MD5 (secondary).""" + nonlocal verified, failed + + if job.cancelled: + return + if file_state.status == DeleteStatus.FAILED: + failed += 1 + return + + try: + metadata = get_object_metadata(s3_client, file_state.s3_bucket, file_state.s3_path) + if not metadata["success"]: + with job.lock: + file_state.status = DeleteStatus.FAILED + file_state.error_message = ( + f"S3 object not found: {metadata.get('error', 'unknown')}" + ) + failed += 1 + return + + s3_size = int(metadata["size"]) + etag = str(metadata["etag"]) + file_state.s3_etag = etag + file_state.s3_size = s3_size + + # Primary check: file size must match + if s3_size != file_state.file_size: + with job.lock: + file_state.status = DeleteStatus.MISMATCH + file_state.error_message = ( + f"Size mismatch: local={file_state.file_size}, s3={s3_size}" + ) + failed += 1 + return + + # Secondary check: MD5 vs ETag (only possible for single-part uploads) + if is_multipart_etag(etag): + with job.lock: + file_state.status = DeleteStatus.VERIFIED + file_state.verification = "size" + verified += 1 + else: + if etag == file_state.local_md5: + with job.lock: + file_state.status = DeleteStatus.VERIFIED + file_state.verification = "md5+size" + verified += 1 + else: + with job.lock: + file_state.status = DeleteStatus.MISMATCH + file_state.error_message = ( + f"MD5 mismatch: local={file_state.local_md5}, s3={etag}" + ) + failed += 1 + except Exception as e: + with job.lock: + file_state.status = DeleteStatus.FAILED + file_state.error_message = f"S3 verification failed: {e}" + failed += 1 + + if progress_callback: + progress_callback(job) + + # Process batch with ThreadPoolExecutor + max_workers = self.batch_config.max_workers if self.batch_processor else 4 + with ThreadPoolExecutor(max_workers=max_workers) as executor: + executor.map(verify_against_s3, batch_files) + + # Send batch_completed event + if batch_callback: + batch_callback( + { + "type": "batch_completed", + "batch_id": batch_id, + "files_verified": verified, + "files_failed": failed, + } + ) + + return { + "success": not job.cancelled, + "processed": len(batch_files), + "uploaded": verified, # "uploaded" maps to "verified" for delete operations + "failed": failed, + "bytes_uploaded": 0, # Not applicable for delete + } + def _finalize_cancelled( self, job: DeleteJob, From 5691b0b8efed43bf5b17f0cad2d2ebb79c9b0306 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Mon, 2 Mar 2026 07:08:01 -0700 Subject: [PATCH 146/173] Backend: Add additional information to logs --- app/services/log_service.py | 138 +++++++++++++++++++----------------- 1 file changed, 73 insertions(+), 65 deletions(-) diff --git a/app/services/log_service.py b/app/services/log_service.py index ba2a4d6..854be2c 100644 --- a/app/services/log_service.py +++ b/app/services/log_service.py @@ -42,11 +42,7 @@ def _get_hive_dir(self, subdir: str, dt: datetime) -> Path: """ log_dir = self._get_log_dir() hive_dir = ( - log_dir - / subdir - / f"year={dt.year:04d}" - / f"month={dt.month:02d}" - / f"day={dt.day:02d}" + log_dir / subdir / f"year={dt.year:04d}" / f"month={dt.month:02d}" / f"day={dt.day:02d}" ) hive_dir.mkdir(parents=True, exist_ok=True) return hive_dir @@ -211,22 +207,24 @@ def save_job_csv( if duration and duration > 0 else None ) - writer.writerow([ - job_id, - f.filename, - f.file_size, - format_file_size(f.file_size), - f.s3_path, - f.status.value, - f.start_time.isoformat() if f.start_time else "", - f.upload_started_at.isoformat() if f.upload_started_at else "", - f.upload_completed_at.isoformat() if f.upload_completed_at else "", - f.upload_duration_seconds, - speed, - f.is_duplicate, - f.is_valid, - f.error_message, - ]) + writer.writerow( + [ + job_id, + f.filename, + f.file_size, + format_file_size(f.file_size), + f.s3_path, + f.status.value, + f.start_time.isoformat() if f.start_time else "", + f.upload_started_at.isoformat() if f.upload_started_at else "", + f.upload_completed_at.isoformat() if f.upload_completed_at else "", + f.upload_duration_seconds, + speed, + f.is_duplicate, + f.is_valid, + f.error_message, + ] + ) with self._write_lock: with open(out_path, "w", encoding="utf-8", newline="") as fh: @@ -251,28 +249,32 @@ def list_log_files(self) -> list[dict[str, Any]]: for f in sorted(json_dir.rglob("*.jsonl"), reverse=True): date_str = self._extract_date_from_hive_path(f) rel_path = f.relative_to(log_dir) - result.append({ - "date": date_str, - "filename": f.name, - "path": str(f), - "relative_path": str(rel_path), - "size_bytes": f.stat().st_size, - "type": "jsonl", - }) + result.append( + { + "date": date_str, + "filename": f.name, + "path": str(f), + "relative_path": str(rel_path), + "size_bytes": f.stat().st_size, + "type": "jsonl", + } + ) # Collect CSV files under csv/ if csv_dir.exists(): for f in sorted(csv_dir.rglob("*.csv"), reverse=True): date_str = self._extract_date_from_hive_path(f) rel_path = f.relative_to(log_dir) - result.append({ - "date": date_str, - "filename": f.name, - "path": str(f), - "relative_path": str(rel_path), - "size_bytes": f.stat().st_size, - "type": "csv", - }) + result.append( + { + "date": date_str, + "filename": f.name, + "path": str(f), + "relative_path": str(rel_path), + "size_bytes": f.stat().st_size, + "type": "csv", + } + ) return result @@ -423,12 +425,14 @@ def get_log_stats(self) -> dict[str, Any]: for csv_file in sorted(csv_dir.rglob("*.csv"), reverse=True): date_str = self._extract_date_from_hive_path(csv_file) or "" rel_path = str(csv_file.relative_to(log_dir)) - csv_files.append({ - "path": rel_path, - "filename": csv_file.name, - "date": date_str, - "size": csv_file.stat().st_size, - }) + csv_files.append( + { + "path": rel_path, + "filename": csv_file.name, + "date": date_str, + "size": csv_file.stat().st_size, + } + ) return { "total_entries": total_entries, @@ -511,14 +515,16 @@ def get_upload_stats(self) -> dict[str, Any]: session_duration += duration - session_files.append({ - "filename": row.get("filename", ""), - "file_size_formatted": row.get("file_size_formatted", ""), - "status": status, - "upload_speed_mbps": speed, - "s3_path": row.get("s3_path", ""), - "error_message": row.get("error_message", ""), - }) + session_files.append( + { + "filename": row.get("filename", ""), + "file_size_formatted": row.get("file_size_formatted", ""), + "status": status, + "upload_speed_mbps": speed, + "s3_path": row.get("s3_path", ""), + "error_message": row.get("error_message", ""), + } + ) except (OSError, csv.Error): continue @@ -531,20 +537,22 @@ def get_upload_stats(self) -> dict[str, Any]: if session_duration > 0 and session_bytes > 0: avg_speed = round(session_bytes / session_duration / 1024 / 1024 * 8, 1) - sessions.append({ - "csv_path": rel_path, - "date": date_str, - "time": time_str, - "total_files": len(session_files), - "completed": session_completed, - "failed": session_failed, - "skipped": session_skipped, - "total_bytes": session_bytes, - "total_bytes_formatted": format_file_size(session_bytes), - "total_duration_seconds": round(session_duration, 1), - "avg_speed_mbps": avg_speed, - "files": session_files, - }) + sessions.append( + { + "csv_path": rel_path, + "date": date_str, + "time": time_str, + "total_files": len(session_files), + "completed": session_completed, + "failed": session_failed, + "skipped": session_skipped, + "total_bytes": session_bytes, + "total_bytes_formatted": format_file_size(session_bytes), + "total_duration_seconds": round(session_duration, 1), + "avg_speed_mbps": avg_speed, + "files": session_files, + } + ) return { "total_files_uploaded": total_uploaded, From 7c43ebc93abcb2678e74d8689c7e5d1c9c69d491 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Mon, 2 Mar 2026 07:08:53 -0700 Subject: [PATCH 147/173] Backend: Extract mcap dates from filename in "fast mode" --- app/services/mcap_service.py | 55 ++++++++++++++++++++++++++++++------ 1 file changed, 47 insertions(+), 8 deletions(-) diff --git a/app/services/mcap_service.py b/app/services/mcap_service.py index 705d7c6..40e622a 100644 --- a/app/services/mcap_service.py +++ b/app/services/mcap_service.py @@ -163,15 +163,50 @@ def _find_datetime_in_dataframes(dataframes: dict[str, pd.DataFrame]) -> datetim return earliest_time -def extract_start_time(file_path: Path | str) -> datetime: - """Extract the earliest timestamp from an MCAP file using modaq_toolkit. +def extract_start_time_fast(file_path: Path | str) -> datetime: + """Extract timestamp from filename only (skip MCAP parsing for speed). + + This is a fast path that extracts timestamps solely from the filename, + skipping the expensive MCAP parsing step. Use when you trust your + filenames are correctly formatted and want maximum performance. + + Performance: ~0.1ms vs ~200ms for full MCAP parsing (2000x speedup) + + Args: + file_path: Path to the MCAP file + + Returns: + datetime: The timestamp extracted from the filename + + Raises: + ValueError: If timestamp cannot be extracted from filename + FileNotFoundError: If the file does not exist + """ + path = Path(file_path) + if not path.exists(): + raise FileNotFoundError(f"MCAP file not found: {path}") + + timestamp = _extract_timestamp_from_filename(path.name) + if timestamp is None: + raise ValueError( + f"Cannot extract timestamp from filename: {path.name}. " + "Consider using extract_start_time() for full MCAP parsing." + ) + + return timestamp + + +def extract_start_time(file_path: Path | str, skip_validation: bool = False) -> datetime: + """Extract the earliest timestamp from an MCAP file. Tries multiple strategies: - 1. Parse MCAP file and look for datetime indices/columns - 2. Extract timestamp from filename if MCAP parsing fails or returns invalid dates + 1. If skip_validation=True: Extract from filename only (fast path) + 2. If skip_validation=False: Parse MCAP file and look for datetime indices/columns + 3. Fallback: Extract timestamp from filename if MCAP parsing fails Args: file_path: Path to the MCAP file + skip_validation: If True, skip MCAP parsing and extract from filename only Returns: datetime: The earliest timestamp found in the MCAP file @@ -180,6 +215,9 @@ def extract_start_time(file_path: Path | str) -> datetime: ValueError: If the file cannot be parsed or has no timestamps FileNotFoundError: If the file does not exist """ + # Fast path: skip MCAP validation + if skip_validation: + return extract_start_time_fast(file_path) from modaq_toolkit import MCAPParser path = Path(file_path) @@ -262,11 +300,14 @@ def generate_s3_path(start_time: datetime, filename: str) -> str: return path -def get_file_info(file_path: Path | str) -> dict[str, str | int | None]: +def get_file_info( + file_path: Path | str, skip_validation: bool = False +) -> dict[str, str | int | None]: """Get information about an MCAP file. Args: file_path: Path to the MCAP file + skip_validation: If True, skip MCAP parsing and extract from filename only Returns: Dictionary containing file information @@ -283,12 +324,10 @@ def get_file_info(file_path: Path | str) -> dict[str, str | int | None]: } try: - start_time = extract_start_time(path) + start_time = extract_start_time(path, skip_validation=skip_validation) info["start_time"] = start_time.isoformat() info["s3_path"] = generate_s3_path(start_time, path.name) except Exception as e: info["error"] = str(e) return info - - From acadc65f90bf43d552b17aa84c23374a147f0a79 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Mon, 2 Mar 2026 07:10:45 -0700 Subject: [PATCH 148/173] Backend: Refactor upload with skip validation and more correct pending logic --- app/services/s3_service.py | 1 + app/services/upload_manager.py | 519 +++++++++++++++++++-------------- 2 files changed, 306 insertions(+), 214 deletions(-) diff --git a/app/services/s3_service.py b/app/services/s3_service.py index 362e3a1..f1afa60 100644 --- a/app/services/s3_service.py +++ b/app/services/s3_service.py @@ -14,6 +14,7 @@ class UploadCancelledError(Exception): """Raised when an upload is cancelled mid-transfer.""" + # Multipart threshold: files below this size are uploaded as a single PUT request, # which produces a simple MD5 ETag. Files above use multipart upload, which produces # a composite ETag (md5_of_part_md5s-part_count) that can't be compared to a local MD5. diff --git a/app/services/upload_manager.py b/app/services/upload_manager.py index b8e7fd9..849b9bb 100644 --- a/app/services/upload_manager.py +++ b/app/services/upload_manager.py @@ -6,7 +6,13 @@ import threading import uuid from collections.abc import Callable -from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed +from concurrent.futures import ( + FIRST_COMPLETED, + ProcessPoolExecutor, + ThreadPoolExecutor, + as_completed, + wait, +) from dataclasses import dataclass, field from datetime import UTC, datetime from enum import Enum @@ -25,14 +31,18 @@ EPOCH_CUTOFF = datetime(1980, 1, 1, tzinfo=UTC) -def _extract_start_time_worker(local_path: str) -> datetime | str: +def _extract_start_time_worker(local_path: str, skip_validation: bool = False) -> datetime | str: """Worker function for ProcessPoolExecutor — must be top-level for pickling. + Args: + local_path: Path to the MCAP file + skip_validation: If True, skip MCAP parsing and extract from filename only + Returns: datetime on success, or error message string on failure. """ try: - return mcap_service.extract_start_time(local_path) + return mcap_service.extract_start_time(local_path, skip_validation=skip_validation) except Exception as e: return str(e) @@ -304,12 +314,27 @@ class ScanJob: class UploadManager: """Manages upload jobs and their execution.""" - def __init__(self, max_workers: int = 4) -> None: + def __init__(self, max_workers: int = 4, batch_config: dict[str, Any] | None = None) -> None: self.jobs: dict[str, UploadJob] = {} self.scan_jobs: dict[str, ScanJob] = {} self.max_workers = max_workers self._lock = threading.Lock() + # Load batch processing configuration + if batch_config is None: + from app.config import get_settings + + settings = get_settings() + batch_config = settings.get_batch_config() + + # Import BatchConfig and BatchProcessor + from app.services.batch_processor import BatchConfig, BatchProcessor + + self.batch_config = BatchConfig.from_dict(batch_config) + self.batch_processor = ( + BatchProcessor(self.batch_config) if self.batch_config.enabled else None + ) + def create_job( self, file_paths: list[str], @@ -367,6 +392,7 @@ def analyze_job( aws_profile: str, aws_region: str, s3_bucket: str, + skip_validation: bool | None = None, ) -> UploadJob | None: """Analyze files in a job - extract timestamps and check for duplicates. @@ -375,10 +401,17 @@ def analyze_job( aws_profile: AWS profile to use aws_region: AWS region s3_bucket: S3 bucket to check for duplicates + skip_validation: If True, skip MCAP parsing. If None, use live settings. Returns: The updated UploadJob or None if not found """ + if skip_validation is None: + from app.config import get_settings + + skip_validation = bool( + get_settings().batch_processing.get("skip_mcap_validation", False) + ) job = self.get_job(job_id) if not job: return None @@ -400,7 +433,9 @@ def analyze_job( file_state.status = UploadStatus.ANALYZING try: # Extract timestamp from MCAP - start_time = mcap_service.extract_start_time(file_state.local_path) + start_time = mcap_service.extract_start_time( + file_state.local_path, skip_validation=skip_validation + ) file_state.start_time = start_time # Generate S3 path @@ -463,6 +498,7 @@ def _analyze_single_file( job_id: str = "", progress_callback: Callable[["UploadJob", FileUploadState], None] | None = None, job: "UploadJob | None" = None, + skip_validation: bool = False, ) -> FileUploadState: """Analyze a single file - extract timestamp and check for duplicates. @@ -474,6 +510,7 @@ def _analyze_single_file( job_id: The parent job ID (for logging) progress_callback: Optional callback fired when file starts analyzing job: The parent UploadJob (needed for callback) + skip_validation: If True, skip MCAP parsing and extract from filename only Returns: The updated FileUploadState @@ -484,7 +521,9 @@ def _analyze_single_file( progress_callback(job, file_state) try: # Extract timestamp from MCAP - start_time = mcap_service.extract_start_time(file_state.local_path) + start_time = mcap_service.extract_start_time( + file_state.local_path, skip_validation=skip_validation + ) file_state.start_time = start_time # Check if timestamp is valid (after 1980) @@ -557,6 +596,7 @@ def analyze_job_async( s3_bucket: str, progress_callback: Callable[["UploadJob", FileUploadState], None] | None = None, use_cache: bool = True, + skip_validation: bool | None = None, ) -> UploadJob | None: """Analyze files in a job asynchronously with parallel processing. @@ -567,10 +607,17 @@ def analyze_job_async( s3_bucket: S3 bucket to check for duplicates progress_callback: Optional callback called after each file completes use_cache: Whether to use cache for duplicate checking + skip_validation: If True, skip MCAP parsing. If None, use live settings. Returns: The updated UploadJob or None if not found """ + if skip_validation is None: + from app.config import get_settings + + skip_validation = bool( + get_settings().batch_processing.get("skip_mcap_validation", False) + ) log = get_log_service() job = self.get_job(job_id) if not job: @@ -601,38 +648,55 @@ def analyze_job_async( # parallelism across cores, bypassing the GIL. cpu_workers = max(1, (os.cpu_count() or 4) - 1) for file_state in job.files: - file_state.status = UploadStatus.ANALYZING + file_state.status = UploadStatus.PENDING + + files_iter_async = iter(job.files) + active_async: dict[Any, FileUploadState] = {} + + def _submit_next_async(proc_executor: ProcessPoolExecutor) -> None: + fs = next(files_iter_async, None) + if fs is None or job.cancelled: + return + fs.status = UploadStatus.ANALYZING + if progress_callback: + progress_callback(job, fs) # "queued → analyzing" event + fut = proc_executor.submit(_extract_start_time_worker, fs.local_path, skip_validation) + active_async[fut] = fs with ProcessPoolExecutor(max_workers=cpu_workers) as proc_executor: - parse_futures = { - proc_executor.submit(_extract_start_time_worker, file_state.local_path): file_state - for file_state in job.files - } - for future in as_completed(parse_futures): + for _ in range(cpu_workers): + _submit_next_async(proc_executor) + + while active_async: if job.cancelled: - for pending_future in parse_futures: - pending_future.cancel() + for f in list(active_async.keys()): + f.cancel() break - file_state = parse_futures[future] - result = future.result() - if isinstance(result, str): - # Error message returned from worker - file_state.status = UploadStatus.FAILED - file_state.error_message = result - log.error( - "analysis", - "file_analysis_failed", - f"Failed to analyze {file_state.filename}: {result}", - {"job_id": job_id, "filename": file_state.filename, "error": result}, - ) - else: - file_state.start_time = result - naive_start = mcap_service.to_naive_utc(result) - file_state.is_valid = naive_start >= EPOCH_CUTOFF.replace(tzinfo=None) - file_state.s3_path = mcap_service.generate_s3_path(result, file_state.filename) - if progress_callback: - progress_callback(job, file_state) + done, _ = wait(list(active_async.keys()), return_when=FIRST_COMPLETED) + for future in done: + file_state = active_async.pop(future) + result = future.result() + if isinstance(result, str): + # Error message returned from worker + file_state.status = UploadStatus.FAILED + file_state.error_message = result + log.error( + "analysis", + "file_analysis_failed", + f"Failed to analyze {file_state.filename}: {result}", + {"job_id": job_id, "filename": file_state.filename, "error": result}, + ) + else: + file_state.start_time = result + naive_start = mcap_service.to_naive_utc(result) + file_state.is_valid = naive_start >= EPOCH_CUTOFF.replace(tzinfo=None) + file_state.s3_path = mcap_service.generate_s3_path( + result, file_state.filename + ) + if progress_callback: + progress_callback(job, file_state) + _submit_next_async(proc_executor) # Phase 2: S3 duplicate checks (I/O-bound) — threads are fine here. parsed_files = [f for f in job.files if f.status != UploadStatus.FAILED] @@ -990,6 +1054,7 @@ def analyze_and_upload_pipeline( analysis_callback: Callable[["UploadJob", FileUploadState], None] | None = None, upload_callback: Callable[["UploadJob"], None] | None = None, use_cache: bool = True, + skip_validation: bool | None = None, ) -> None: """Analyze each file and upload it immediately — pipeline approach. @@ -1007,7 +1072,17 @@ def analyze_and_upload_pipeline( analysis_callback: Called after each file is analyzed upload_callback: Called for upload progress updates use_cache: Whether to use cache for duplicate checking + skip_validation: If True, skip MCAP parsing. If None, use batch_config setting """ + # Determine skip_validation setting — read live from settings so that + # changes made in the Settings UI take effect without a server restart. + # (UploadManager is a singleton whose batch_config is frozen at init time.) + if skip_validation is None: + from app.config import get_settings + + skip_validation = bool( + get_settings().batch_processing.get("skip_mcap_validation", False) + ) log = get_log_service() job = self.get_job(job_id) if not job: @@ -1040,233 +1115,249 @@ def analyze_and_upload_pipeline( cpu_workers = max(1, (os.cpu_count() or 4) - 1) upload_executor = ThreadPoolExecutor(max_workers=self.max_workers) - # Set all files to ANALYZING + # Mark all files as PENDING (waiting their turn in the analysis pool) for fs in job.files: + fs.status = UploadStatus.PENDING + + files_iter = iter(job.files) + active: dict[Any, FileUploadState] = {} + + def _submit_next(proc_executor: ProcessPoolExecutor) -> None: + fs = next(files_iter, None) + if fs is None or job.cancelled: + return fs.status = UploadStatus.ANALYZING + if analysis_callback: + analysis_callback(job, fs) # "queued → analyzing" event + fut = proc_executor.submit(_extract_start_time_worker, fs.local_path, skip_validation) + active[fut] = fs try: - # Submit all files for MCAP parsing (CPU-bound, true parallelism) with ProcessPoolExecutor(max_workers=cpu_workers) as proc_executor: - parse_futures = { - proc_executor.submit(_extract_start_time_worker, fs.local_path): fs - for fs in job.files - } + # Fill initial slots + for _ in range(cpu_workers): + _submit_next(proc_executor) - for future in as_completed(parse_futures): + while active: if job.cancelled: - # Cancel remaining parse futures that haven't started yet - for pending_future in parse_futures: - pending_future.cancel() + for f in list(active.keys()): + f.cancel() break - fs = parse_futures[future] - result = future.result() - - if isinstance(result, str): - # Parse failed - fs.status = UploadStatus.FAILED - fs.error_message = result - log.error( - "analysis", - "file_analysis_failed", - f"Failed to analyze {fs.filename}: {result}", - {"job_id": job_id, "filename": fs.filename, "error": result}, - ) - if analysis_callback: - analysis_callback(job, fs) - continue - - # Parse succeeded — set timestamp and generate S3 path - fs.start_time = result - naive_start = mcap_service.to_naive_utc(result) - fs.is_valid = naive_start >= EPOCH_CUTOFF.replace(tzinfo=None) - fs.s3_path = mcap_service.generate_s3_path(result, fs.filename) - - # Check duplicate (I/O but fast — cache lookup or S3 HEAD) - self._check_duplicate(fs, s3_client, s3_bucket, use_cache) - fs.status = UploadStatus.READY - - log.info( - "analysis", - "file_analysis_completed", - f"Analyzed {fs.filename}", - { - "job_id": job_id, - "filename": fs.filename, - "file_size": fs.file_size, - "s3_path": fs.s3_path, - "is_duplicate": fs.is_duplicate, - "is_valid": fs.is_valid, - }, - ) + done, _ = wait(list(active.keys()), return_when=FIRST_COMPLETED) + for future in done: + fs = active.pop(future) + result = future.result() + + if isinstance(result, str): + # Parse failed + fs.status = UploadStatus.FAILED + fs.error_message = result + log.error( + "analysis", + "file_analysis_failed", + f"Failed to analyze {fs.filename}: {result}", + {"job_id": job_id, "filename": fs.filename, "error": result}, + ) + if analysis_callback: + analysis_callback(job, fs) + _submit_next(proc_executor) + continue - # Notify frontend of analysis result - if analysis_callback: - analysis_callback(job, fs) + # Parse succeeded — set timestamp and generate S3 path + fs.start_time = result + naive_start = mcap_service.to_naive_utc(result) + fs.is_valid = naive_start >= EPOCH_CUTOFF.replace(tzinfo=None) + fs.s3_path = mcap_service.generate_s3_path(result, fs.filename) - # Decide: skip or upload? - if not fs.is_valid: - fs.status = UploadStatus.SKIPPED - fs.error_message = "Invalid timestamp (pre-1980)" - log.warning( - "upload", - "file_upload_skipped", - f"Skipped invalid timestamp: {fs.filename}", - { - "job_id": job_id, - "filename": fs.filename, - "reason": "invalid_timestamp", - }, - ) - if upload_callback: - upload_callback(job) - continue + # Check duplicate (I/O but fast — cache lookup or S3 HEAD) + self._check_duplicate(fs, s3_client, s3_bucket, use_cache) + fs.status = UploadStatus.READY - if skip_duplicates and fs.is_duplicate: - fs.status = UploadStatus.SKIPPED - fs.bytes_uploaded = fs.file_size log.info( - "upload", - "file_upload_skipped", - f"Skipped duplicate: {fs.filename}", + "analysis", + "file_analysis_completed", + f"Analyzed {fs.filename}", { "job_id": job_id, "filename": fs.filename, - "reason": "duplicate", + "file_size": fs.file_size, + "s3_path": fs.s3_path, + "is_duplicate": fs.is_duplicate, + "is_valid": fs.is_valid, }, ) - if upload_callback: - upload_callback(job) - continue - - # Submit for upload immediately - def make_upload_task( - file_state: FileUploadState, - ) -> Callable[[], Any]: - def upload_task() -> Any: - if job.cancelled: - with job.lock: - file_state.status = UploadStatus.CANCELLED - if analysis_callback: - analysis_callback(job, file_state) - if upload_callback: - upload_callback(job) - return None - try: - with job.lock: - file_state.status = UploadStatus.UPLOADING - file_state.upload_started_at = datetime.now(UTC) - log.info( - "upload", - "file_upload_started", - f"Uploading {file_state.filename}", - { - "job_id": job_id, - "filename": file_state.filename, - "file_size": file_state.file_size, - "s3_path": file_state.s3_path, - }, - ) - if upload_callback: - upload_callback(job) + # Notify frontend of analysis result + if analysis_callback: + analysis_callback(job, fs) - def byte_callback(uploaded: int, total: int) -> None: + # Fill freed slot immediately + _submit_next(proc_executor) + + # Decide: skip or upload? + if not fs.is_valid: + fs.status = UploadStatus.SKIPPED + fs.error_message = "Invalid timestamp (pre-1980)" + log.warning( + "upload", + "file_upload_skipped", + f"Skipped invalid timestamp: {fs.filename}", + { + "job_id": job_id, + "filename": fs.filename, + "reason": "invalid_timestamp", + }, + ) + if upload_callback: + upload_callback(job) + continue + + if skip_duplicates and fs.is_duplicate: + fs.status = UploadStatus.SKIPPED + fs.bytes_uploaded = fs.file_size + log.info( + "upload", + "file_upload_skipped", + f"Skipped duplicate: {fs.filename}", + { + "job_id": job_id, + "filename": fs.filename, + "reason": "duplicate", + }, + ) + if upload_callback: + upload_callback(job) + continue + + # Submit for upload immediately + def make_upload_task( + file_state: FileUploadState, + ) -> Callable[[], Any]: + def upload_task() -> Any: + if job.cancelled: with job.lock: - file_state.bytes_uploaded = uploaded + file_state.status = UploadStatus.CANCELLED + if analysis_callback: + analysis_callback(job, file_state) if upload_callback: upload_callback(job) + return None - upload_result = s3_service.upload_file_with_progress( - s3_client, - file_state.local_path, - s3_bucket, - file_state.s3_path, - byte_callback, - cancel_check=lambda: job.cancelled, - ) - - # Handle completion inline - file_state.upload_completed_at = datetime.now(UTC) - if upload_result["success"]: - file_state.status = UploadStatus.COMPLETED - file_state.bytes_uploaded = file_state.file_size + try: + with job.lock: + file_state.status = UploadStatus.UPLOADING + file_state.upload_started_at = datetime.now(UTC) log.info( "upload", - "file_upload_completed", - f"Uploaded {file_state.filename}", + "file_upload_started", + f"Uploading {file_state.filename}", { "job_id": job_id, "filename": file_state.filename, "file_size": file_state.file_size, - "upload_duration_seconds": ( - file_state.upload_duration_seconds - ), "s3_path": file_state.s3_path, }, ) - try: - cache = get_cache_service() - cache.update_cache( - s3_bucket, - file_state.s3_path, - exists=True, - filename=file_state.filename, - file_size=file_state.file_size, + if upload_callback: + upload_callback(job) + + def byte_callback(uploaded: int, total: int) -> None: + with job.lock: + file_state.bytes_uploaded = uploaded + if upload_callback: + upload_callback(job) + + upload_result = s3_service.upload_file_with_progress( + s3_client, + file_state.local_path, + s3_bucket, + file_state.s3_path, + byte_callback, + cancel_check=lambda: job.cancelled, + ) + + # Handle completion inline + file_state.upload_completed_at = datetime.now(UTC) + if upload_result["success"]: + file_state.status = UploadStatus.COMPLETED + file_state.bytes_uploaded = file_state.file_size + log.info( + "upload", + "file_upload_completed", + f"Uploaded {file_state.filename}", + { + "job_id": job_id, + "filename": file_state.filename, + "file_size": file_state.file_size, + "upload_duration_seconds": ( + file_state.upload_duration_seconds + ), + "s3_path": file_state.s3_path, + }, + ) + try: + cache = get_cache_service() + cache.update_cache( + s3_bucket, + file_state.s3_path, + exists=True, + filename=file_state.filename, + file_size=file_state.file_size, + ) + except Exception: + logger.debug( + "Cache update failed after upload", + exc_info=True, + ) + else: + file_state.status = UploadStatus.FAILED + file_state.error_message = upload_result.get( + "error", "Unknown error" ) - except Exception: - logger.debug( - "Cache update failed after upload", - exc_info=True, + log.error( + "upload", + "file_upload_failed", + f"Failed to upload {file_state.filename}: " + f"{file_state.error_message}", + { + "job_id": job_id, + "filename": file_state.filename, + "error": file_state.error_message, + }, ) - else: + except UploadCancelledError: + with job.lock: + file_state.status = UploadStatus.CANCELLED + file_state.upload_completed_at = datetime.now(UTC) + except Exception as e: + file_state.upload_completed_at = datetime.now(UTC) file_state.status = UploadStatus.FAILED - file_state.error_message = upload_result.get( - "error", "Unknown error" - ) + file_state.error_message = str(e) log.error( "upload", "file_upload_failed", - f"Failed to upload {file_state.filename}: " - f"{file_state.error_message}", + f"Failed to upload {file_state.filename}: {e}", { "job_id": job_id, "filename": file_state.filename, - "error": file_state.error_message, + "error": str(e), }, ) - except UploadCancelledError: - with job.lock: - file_state.status = UploadStatus.CANCELLED - file_state.upload_completed_at = datetime.now(UTC) - except Exception as e: - file_state.upload_completed_at = datetime.now(UTC) - file_state.status = UploadStatus.FAILED - file_state.error_message = str(e) - log.error( - "upload", - "file_upload_failed", - f"Failed to upload {file_state.filename}: {e}", - { - "job_id": job_id, - "filename": file_state.filename, - "error": str(e), - }, - ) - - # Notify per-file status so the frontend - # updates this row immediately (the progress - # dict only includes active files, so without - # this the row would keep spinning). - if analysis_callback: - analysis_callback(job, file_state) - if upload_callback: - upload_callback(job) - return None - return upload_task + # Notify per-file status so the frontend + # updates this row immediately (the progress + # dict only includes active files, so without + # this the row would keep spinning). + if analysis_callback: + analysis_callback(job, file_state) + if upload_callback: + upload_callback(job) + return None + + return upload_task - upload_executor.submit(make_upload_task(fs)) + upload_executor.submit(make_upload_task(fs)) except Exception as e: log.error( From 8e001167ea71f3d22b5a740459904c694d571001 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Mon, 2 Mar 2026 07:12:03 -0700 Subject: [PATCH 149/173] Frontend: Add hook support for folder scanStart --- frontend/src/hooks/useFolderScan.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/frontend/src/hooks/useFolderScan.ts b/frontend/src/hooks/useFolderScan.ts index 2308003..c54e413 100644 --- a/frontend/src/hooks/useFolderScan.ts +++ b/frontend/src/hooks/useFolderScan.ts @@ -12,6 +12,7 @@ import { useUploadStore } from "../stores/uploadStore.ts"; import type { ScannedFolder, ScanEvent, + ScanStartedEvent, ScanFolderCompleteEvent, ScanCompleteEvent, } from "../types/api.ts"; @@ -32,6 +33,7 @@ interface UseFolderScanResult { startScan: (folderPath: string, cacheOnly?: boolean, exclusions?: ScanExclusions) => Promise; cancelScan: () => Promise; folders: ScannedFolder[]; + foldersTotal: number; isScanning: boolean; scanComplete: boolean; totals: ScanTotals; @@ -44,11 +46,13 @@ export function useFolderScan(): UseFolderScanResult { scanFolders: folders, isScanning, scanComplete, + scanFoldersTotal: foldersTotal, scanTotals: totals, setScanJobId, addScanFolder, setScanComplete, setIsScanning, + setScanFoldersTotal, updateScanTotals, } = useUploadStore(); @@ -59,9 +63,11 @@ export function useFolderScan(): UseFolderScanResult { if (!data || typeof data !== "object") return; switch (data.type) { - case "scan_started": - // Just update scanning state — we already set it in startScan. + case "scan_started": { + const evt = data as ScanStartedEvent; + setScanFoldersTotal(evt.folders_total); break; + } case "scan_folder_complete": { const evt = data as ScanFolderCompleteEvent; @@ -88,7 +94,7 @@ export function useFolderScan(): UseFolderScanResult { } } }, - [addScanFolder, updateScanTotals, setScanComplete, setIsScanning], + [addScanFolder, updateScanTotals, setScanComplete, setIsScanning, setScanFoldersTotal], ); // Only connect when we have a jobId and are still scanning. @@ -151,5 +157,5 @@ export function useFolderScan(): UseFolderScanResult { setJobId(null); }, [setIsScanning, setScanComplete]); - return { startScan, cancelScan, folders, isScanning, scanComplete, totals }; + return { startScan, cancelScan, folders, foldersTotal, isScanning, scanComplete, totals }; } From 41381b6719b93b2596e082a8c284750ad1ae612f Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Mon, 2 Mar 2026 07:15:03 -0700 Subject: [PATCH 150/173] Front: Update useSSE with heartbeat detection --- frontend/src/hooks/useSSE.ts | 37 ++++++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/frontend/src/hooks/useSSE.ts b/frontend/src/hooks/useSSE.ts index cce1988..b28039a 100644 --- a/frontend/src/hooks/useSSE.ts +++ b/frontend/src/hooks/useSSE.ts @@ -3,6 +3,11 @@ * * Creates an EventSource when `url` is non-null; closes on cleanup or when * the stream ends. No auto-reconnect — our SSE streams are finite. + * + * Features: + * - Automatic timeout detection (60s without messages triggers error) + * - Heartbeat support (server sends ": heartbeat" comments to keep alive) + * - Clean error handling and connection cleanup */ import { useEffect, useRef } from "react"; @@ -12,11 +17,13 @@ export interface UseSSEOptions { url: string | null; /** Called for every `data:` line (already JSON-parsed). */ onMessage: (data: unknown) => void; - /** Called on EventSource errors. */ + /** Called on EventSource errors or timeout. */ onError?: (error: Event) => void; + /** Timeout in milliseconds (default: 60000 = 60s). Set to 0 to disable. */ + timeout?: number; } -export function useSSE({ url, onMessage, onError }: UseSSEOptions): void { +export function useSSE({ url, onMessage, onError, timeout = 60000 }: UseSSEOptions): void { // Store latest callbacks in refs so we never re-open a connection just // because the caller created a new closure. const onMessageRef = useRef(onMessage); @@ -31,17 +38,38 @@ export function useSSE({ url, onMessage, onError }: UseSSEOptions): void { if (!url) return; const es = new EventSource(url); + let timeoutId: ReturnType | null = null; + + // Start timeout timer if enabled + const resetTimeout = () => { + if (timeout > 0) { + if (timeoutId) clearTimeout(timeoutId); + timeoutId = setTimeout(() => { + // No messages received for timeout duration — connection likely dead + const timeoutError = new Event('timeout'); + onErrorRef.current?.(timeoutError); + es.close(); + }, timeout); + } + }; + + resetTimeout(); // Initial timeout es.onmessage = (event: MessageEvent) => { + // Reset timeout on any message (including heartbeats) + resetTimeout(); + try { const data: unknown = JSON.parse(event.data as string); onMessageRef.current(data); } catch { - // Ignore non-JSON messages (e.g., keep-alive pings) + // Ignore non-JSON messages (e.g., ": heartbeat" comment lines) + // These still reset the timeout above, which is their purpose } }; es.onerror = (event: Event) => { + if (timeoutId) clearTimeout(timeoutId); onErrorRef.current?.(event); // The server closes the stream on terminal events, which fires an error // event with readyState CLOSED. We just close our side too. @@ -49,7 +77,8 @@ export function useSSE({ url, onMessage, onError }: UseSSEOptions): void { }; return () => { + if (timeoutId) clearTimeout(timeoutId); es.close(); }; - }, [url]); + }, [url, timeout]); } From da5eabbca46127909b1ff9c12efebc07e6627283 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Mon, 2 Mar 2026 07:16:22 -0700 Subject: [PATCH 151/173] Frontend: Add Settings for performance tuning --- .../settings/PerformanceSection.tsx | 262 ++++++++++++++++++ 1 file changed, 262 insertions(+) create mode 100644 frontend/src/components/settings/PerformanceSection.tsx diff --git a/frontend/src/components/settings/PerformanceSection.tsx b/frontend/src/components/settings/PerformanceSection.tsx new file mode 100644 index 0000000..9bb2543 --- /dev/null +++ b/frontend/src/components/settings/PerformanceSection.tsx @@ -0,0 +1,262 @@ +/** + * Performance settings section for batch processing configuration. + * + * Allows users to configure: + * - Skip MCAP validation (fast filename-only parsing) + * - Batch size for large uploads + * - Auto-tune workers based on CPU + * - Max worker count + * + * Settings auto-save after changes (debounced for continuous inputs like sliders). + */ + +import { useCallback, useEffect, useRef, useState } from "react"; +import { useAppStore } from "../../stores/appStore.ts"; +import type { BatchProcessingSettings, ValueSource } from "../../types/api.ts"; +import { CheckIcon, InfoIcon, SpinnerIcon } from "../../utils/icons.tsx"; + +function SectionSourceNote({ source }: { source?: ValueSource }) { + if (!source || source.source === "builtin") return null; + + if (source.source === "settings_file" || source.source === "default_file") { + const filename = source.path?.split("/").pop() ?? source.path ?? ""; + const label = + source.source === "default_file" + ? `Default values — ${filename}` + : `Saved in ${filename}`; + return ( +

+ {label} +

+ ); + } + + return null; // batch_processing has no env override support +} + +function getDefaultSettings(): BatchProcessingSettings { + return { + enabled: true, + batch_size: 100, + auto_tune_workers: true, + max_workers: 4, + target_cpu_percent: 70.0, + skip_mcap_validation: false, + use_database_for_large_jobs: true, + large_job_threshold: 1000, + }; +} + +type SaveStatus = "idle" | "saving" | "saved" | "error"; + +export default function PerformanceSection() { + const { settings: appSettings, updateSettings } = useAppStore(); + const batchSource = appSettings?.value_sources?.["batch_processing"]; + + const [settings, setSettings] = useState( + appSettings?.batch_processing ?? getDefaultSettings(), + ); + const [saveStatus, setSaveStatus] = useState("idle"); + + // Track whether a change originated from the user (vs. store sync). + const userChangedRef = useRef(false); + const debounceRef = useRef | null>(null); + const savedTimerRef = useRef | null>(null); + + // Re-sync from the store when settings are loaded externally (e.g. page + // navigation reload), but only when there is no pending user change. + useEffect(() => { + if (appSettings?.batch_processing && !userChangedRef.current) { + setSettings(appSettings.batch_processing); + } + }, [appSettings]); + + // Cleanup timers on unmount. + useEffect(() => { + return () => { + if (debounceRef.current !== null) clearTimeout(debounceRef.current); + if (savedTimerRef.current !== null) clearTimeout(savedTimerRef.current); + }; + }, []); + + const save = useCallback( + async (toSave: BatchProcessingSettings) => { + setSaveStatus("saving"); + try { + await updateSettings({ batch_processing: toSave }); + userChangedRef.current = false; + setSaveStatus("saved"); + // Clear "saved" indicator after 2 seconds. + savedTimerRef.current = setTimeout(() => setSaveStatus("idle"), 2000); + } catch { + setSaveStatus("error"); + } + }, + [updateSettings], + ); + + function handleChange(field: keyof BatchProcessingSettings, value: unknown) { + const next = { ...settings, [field]: value }; + setSettings(next); + userChangedRef.current = true; + + // Debounce the save so continuous inputs (sliders) don't fire on every tick. + if (debounceRef.current !== null) clearTimeout(debounceRef.current); + if (savedTimerRef.current !== null) clearTimeout(savedTimerRef.current); + debounceRef.current = setTimeout(() => save(next), 600); + } + + async function handleReset() { + const defaults = getDefaultSettings(); + setSettings(defaults); + userChangedRef.current = true; + if (debounceRef.current !== null) clearTimeout(debounceRef.current); + if (savedTimerRef.current !== null) clearTimeout(savedTimerRef.current); + await save(defaults); + } + + return ( +
+
+
+

Performance

+ {saveStatus === "saving" && ( + + + Saving... + + )} + {saveStatus === "saved" && ( + + + Saved + + )} + {saveStatus === "error" && ( + Save failed + )} +
+ +
+ +
+ {/* Skip MCAP Validation */} +
+ handleChange("skip_mcap_validation", e.target.checked)} + className="mt-1 h-4 w-4 text-nlr-blue border-gray-300 rounded focus:ring-nlr-blue" + /> +
+ +

+ Extract timestamps from filenames only (3000x faster). Use when filenames are correctly formatted. +

+
+
+ + {/* Batch Size */} +
+ + handleChange("batch_size", Number.parseInt(e.target.value))} + className="w-full h-2 bg-gray-200 rounded-lg appearance-none cursor-pointer accent-nlr-blue" + /> +

+ Number of files processed per batch (50-500). Lower values reduce memory usage. +

+
+ + {/* Auto-tune Workers */} +
+ handleChange("auto_tune_workers", e.target.checked)} + className="mt-1 h-4 w-4 text-nlr-blue border-gray-300 rounded focus:ring-nlr-blue" + /> +
+ +

+ Automatically adjust worker count based on CPU/memory utilization. Recommended for large jobs. +

+
+
+ + {/* Max Workers */} +
+ + handleChange("max_workers", Number.parseInt(e.target.value))} + className="w-32 px-3 py-2 text-sm border border-gray-300 rounded-md focus:outline-none focus:ring-2 focus:ring-nlr-blue" + /> +

+ Maximum concurrent workers (2-16). Higher values increase throughput but use more resources. +

+
+ + {/* Large Job Threshold */} +
+ + handleChange("large_job_threshold", Number.parseInt(e.target.value))} + className="w-32 px-3 py-2 text-sm border border-gray-300 rounded-md focus:outline-none focus:ring-2 focus:ring-nlr-blue" + /> +

+ Jobs with more files than this threshold use batch processing and database storage. +

+
+ + {/* Info banner */} +
+ +
+ Batch processing optimizes memory usage and performance for large uploads (1000+ files). + Results are stored in a database and retrieved on demand. +
+
+ + {/* Reset button */} +
+ +
+
+
+ ); +} From 42e938d570f0a1af94b7018a704bc7dd969e595a Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Mon, 2 Mar 2026 07:16:54 -0700 Subject: [PATCH 152/173] Frontend: Add list of active files --- .../src/components/upload/ActiveFilesList.tsx | 104 ++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 frontend/src/components/upload/ActiveFilesList.tsx diff --git a/frontend/src/components/upload/ActiveFilesList.tsx b/frontend/src/components/upload/ActiveFilesList.tsx new file mode 100644 index 0000000..9d92eb5 --- /dev/null +++ b/frontend/src/components/upload/ActiveFilesList.tsx @@ -0,0 +1,104 @@ +/** + * Compact list showing only files currently being uploaded. + * + * Displays up to 8 active files with: + * - Filename + * - File size + * - Upload progress bar + * - Status indicator + * + * Used during upload phase to show real-time activity without + * overwhelming the UI with thousands of rows. + */ + +import type { FileUploadState } from "../../types/api.ts"; +import { formatBytes } from "../../utils/format/bytes.ts"; +import ProgressBar from "../common/ProgressBar.tsx"; +import Spinner from "../common/Spinner.tsx"; +import { CheckIcon, XIcon, WarningIcon } from "../../utils/icons.tsx"; + +interface ActiveFilesListProps { + /** Currently active files (max 8) */ + files: FileUploadState[]; +} + +export default function ActiveFilesList({ files }: ActiveFilesListProps) { + if (files.length === 0) { + return ( +
+

No files currently uploading

+
+ ); + } + + return ( +
+
+

Currently Uploading

+ {files.length} active +
+ +
+ {files.map((file) => ( + + ))} +
+
+ ); +} + +function ActiveFileCard({ file }: { file: FileUploadState }) { + const statusIcon = getStatusIcon(file.status); + + return ( +
+
+
+
+

{file.filename}

+ {statusIcon} +
+

+ {formatBytes(file.file_size)} + {file.status === "uploading" && file.bytes_uploaded > 0 && ( + + {formatBytes(file.bytes_uploaded)} uploaded + + )} +

+
+
+ + {/* Progress bar for uploading files */} + {file.status === "uploading" && ( +
+ +
+ )} + + {/* Error message for failed files */} + {file.status === "failed" && file.error_message && ( +
+ {file.error_message} +
+ )} +
+ ); +} + +function getStatusIcon(status: string) { + switch (status) { + case "uploading": + return ; + case "completed": + return ; + case "failed": + return ; + case "skipped": + return ; + case "analyzing": + return ; + default: + return null; + } +} From c0e75bc2562100a02d2a836ef5b2809bd2ee263a Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Mon, 2 Mar 2026 07:17:48 -0700 Subject: [PATCH 153/173] Frontend: Add new batch progress component --- .../src/components/upload/BatchProgress.tsx | 131 ++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 frontend/src/components/upload/BatchProgress.tsx diff --git a/frontend/src/components/upload/BatchProgress.tsx b/frontend/src/components/upload/BatchProgress.tsx new file mode 100644 index 0000000..89aeeb1 --- /dev/null +++ b/frontend/src/components/upload/BatchProgress.tsx @@ -0,0 +1,131 @@ +/** + * Batch progress indicator for large upload jobs. + * + * Shows: + * - Current batch number (e.g., "Batch 5/200") + * - Progress bar for current batch + * - Cumulative job statistics + * - Active files being processed (max 8) + * + * Used during upload phase for jobs processed in batches. + */ + +import type { BatchState } from "../../types/api.ts"; +import ProgressBar from "../common/ProgressBar.tsx"; +import Spinner from "../common/Spinner.tsx"; +import { InfoIcon } from "../../utils/icons.tsx"; + +interface BatchProgressProps { + /** Current batch state */ + batchState: BatchState | null; + + /** Overall job progress (0-100) */ + jobProgressPercent: number; + + /** Total files completed across all batches */ + jobFilesCompleted: number; + + /** Total files in entire job */ + jobFilesTotal: number; + + /** Total files uploaded successfully across all batches */ + jobFilesUploaded: number; + + /** Total files failed across all batches */ + jobFilesFailed: number; + + /** Whether the job is actively running */ + isRunning: boolean; +} + +export default function BatchProgress({ + batchState, + jobProgressPercent, + jobFilesCompleted, + jobFilesTotal, + jobFilesUploaded, + jobFilesFailed, + isRunning, +}: BatchProgressProps) { + if (!batchState) { + return null; + } + + const batchProgressPercent = batchState.files_in_batch > 0 + ? (batchState.files_processed / batchState.files_in_batch) * 100 + : 0; + + const currentBatch = batchState.batch_id + 1; // 0-indexed to 1-indexed + const totalBatches = batchState.total_batches; + + return ( +
+ {/* Batch indicator */} +
+
+
+

+ Batch {currentBatch} of {totalBatches} +

+ {isRunning && batchState.status === "processing" && } +
+
+ {batchState.files_in_batch} files in this batch +
+
+ + {/* Batch progress */} + + + {/* Batch stats */} +
+ + Uploaded: {batchState.files_uploaded} | Failed: {batchState.files_failed} + + {batchState.status === "completed" && batchState.duration_seconds && ( + {batchState.duration_seconds.toFixed(1)}s + )} +
+
+ + {/* Overall job progress */} +
+
+

Overall Progress

+
+ + + + {/* Job stats */} +
+ + Uploaded: {jobFilesUploaded} | Failed: {jobFilesFailed} + + + {totalBatches - currentBatch} batch{totalBatches - currentBatch !== 1 ? "es" : ""} remaining + +
+
+ + {/* Info banner */} + {totalBatches > 10 && ( +
+ +
+ Large job detected: Files are being processed in batches + to optimize memory usage and performance. Full results will be available + after completion. +
+
+ )} +
+ ); +} From 9d910cc4da558fddfefec49b199145cbf70b99b8 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Mon, 2 Mar 2026 07:18:04 -0700 Subject: [PATCH 154/173] Frontend: Add full screen cancel modal that locks screen --- .../src/components/upload/CancelScanModal.tsx | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 frontend/src/components/upload/CancelScanModal.tsx diff --git a/frontend/src/components/upload/CancelScanModal.tsx b/frontend/src/components/upload/CancelScanModal.tsx new file mode 100644 index 0000000..5545ef6 --- /dev/null +++ b/frontend/src/components/upload/CancelScanModal.tsx @@ -0,0 +1,63 @@ +/** + * Confirmation modal shown when the user tries to cancel a folder scan. + */ + +import Modal from "../common/Modal.tsx"; +import { WarningIcon } from "../../utils/icons.tsx"; + +interface CancelScanModalProps { + isOpen: boolean; + onClose: () => void; + onConfirm: () => void; + foldersScanned: number; + filesFound: number; +} + +export default function CancelScanModal({ + isOpen, + onClose, + onConfirm, + foldersScanned, + filesFound, +}: CancelScanModalProps) { + return ( + + + +
+ } + > +
+
+ +
+

+ Scanning has found {filesFound} file{filesFound !== 1 ? "s" : ""} in{" "} + {foldersScanned} folder{foldersScanned !== 1 ? "s" : ""} so far. +

+

+ Cancelling will stop the scan and you'll need to start over if you want to continue. +

+
+
+
+ + ); +} From 3426624c4036bafaca179aa00d32e00f9493292a Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Mon, 2 Mar 2026 07:18:31 -0700 Subject: [PATCH 155/173] Frontend: Add scan progress modal --- .../components/upload/ScanProgressModal.tsx | 169 ++++++++++++++++++ 1 file changed, 169 insertions(+) create mode 100644 frontend/src/components/upload/ScanProgressModal.tsx diff --git a/frontend/src/components/upload/ScanProgressModal.tsx b/frontend/src/components/upload/ScanProgressModal.tsx new file mode 100644 index 0000000..f0073c0 --- /dev/null +++ b/frontend/src/components/upload/ScanProgressModal.tsx @@ -0,0 +1,169 @@ +/** + * Modal overlay that shows folder scan progress. + * + * Displayed when scanning a folder with many files to give users + * clear feedback about what's happening and how long it might take. + */ + +import { useEffect, useRef, useState } from "react"; +import { formatBytes } from "../../utils/format/bytes.ts"; +import ProgressBar from "../common/ProgressBar.tsx"; +import Spinner from "../common/Spinner.tsx"; +import { XIcon } from "../../utils/icons.tsx"; +import CancelScanModal from "./CancelScanModal.tsx"; + +interface ScanProgressModalProps { + isOpen: boolean; + foldersScanned: number; + foldersTotal: number; + totalFiles: number; + totalSize: number; + folderPath: string; + onCancel: () => void; +} + +export default function ScanProgressModal({ + isOpen, + foldersScanned, + foldersTotal, + totalFiles, + totalSize, + folderPath, + onCancel, +}: ScanProgressModalProps) { + const backdropRef = useRef(null); + const [showCancelConfirm, setShowCancelConfirm] = useState(false); + + // Calculate scan progress percentage + const progressPercent = foldersTotal > 0 ? Math.round((foldersScanned / foldersTotal) * 100) : 0; + + const handleCancelClick = () => { + setShowCancelConfirm(true); + }; + + const handleConfirmCancel = () => { + setShowCancelConfirm(false); + onCancel(); + }; + + // Handle Escape key - show confirmation + useEffect(() => { + if (!isOpen) return; + + function handleKeyDown(e: KeyboardEvent) { + if (e.key === "Escape") handleCancelClick(); + } + + document.addEventListener("keydown", handleKeyDown); + return () => document.removeEventListener("keydown", handleKeyDown); + }, [isOpen]); + + if (!isOpen) return null; + + function handleBackdropClick(e: React.MouseEvent) { + if (e.target === backdropRef.current) handleCancelClick(); + } + + return ( + <> +
+
+
+ {/* Header */} +
+
+ +
+

Scanning Folder

+

+ Searching for MCAP files and checking upload status... +

+
+
+ +
+ + {/* Folder path */} +
+

Scanning

+

+ {folderPath} +

+
+ + {/* Progress bar */} + {foldersTotal > 0 && ( +
+
+ Progress + {progressPercent}% +
+ +

+ {foldersScanned} of {foldersTotal} folders scanned +

+
+ )} + + {/* Progress stats */} +
+ + + +
+ + {/* Info message */} +
+

+ Note: Scanning large folders with thousands of files may take a few + minutes. Each file is checked against the upload cache to determine if it's already been + uploaded to S3. +

+
+ + {/* Cancel button */} +
+ +
+
+
+
+ {/* Cancel confirmation modal */} + setShowCancelConfirm(false)} + onConfirm={handleConfirmCancel} + foldersScanned={foldersScanned} + filesFound={totalFiles} + /> + + ); +} + +function StatCard({ value, label }: { value: number | string; label: string }) { + return ( +
+
+ {typeof value === "number" ? value.toLocaleString() : value} +
+
{label}
+
+ ); +} From 1562209413d998b84d086ed59be0b987f1670867 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Mon, 2 Mar 2026 07:19:26 -0700 Subject: [PATCH 156/173] Frontend: Add data source provenance to file derived settings --- .../src/components/settings/SettingsForm.tsx | 141 +++++++++++------- 1 file changed, 84 insertions(+), 57 deletions(-) diff --git a/frontend/src/components/settings/SettingsForm.tsx b/frontend/src/components/settings/SettingsForm.tsx index 5f74fc2..9817cc8 100644 --- a/frontend/src/components/settings/SettingsForm.tsx +++ b/frontend/src/components/settings/SettingsForm.tsx @@ -1,10 +1,41 @@ import { useEffect, useState } from "react"; import { apiGet, apiPost } from "../../api/client.ts"; import { useAppStore } from "../../stores/appStore.ts"; -import type { AppSettings, ConnectionTestResult } from "../../types/api.ts"; +import type { AppSettings, ConnectionTestResult, ValueSource } from "../../types/api.ts"; +import { LockIcon } from "../../utils/icons.tsx"; const AWS_REGIONS = ["us-east-1", "us-east-2", "us-west-1", "us-west-2"]; +/** Shows where a setting value comes from and locks the field if it is env-overridden. */ +function SourceBadge({ source }: { source?: ValueSource }) { + if (!source) return null; + + if (source.source === "env") { + return ( + + + Locked — set by environment variable{" "} + {source.env_var} + + ); + } + + if (source.source === "settings_file" || source.source === "default_file") { + const filename = source.path?.split("/").pop() ?? source.path ?? ""; + const label = + source.source === "default_file" + ? `Default — ${filename}` + : `Saved in ${filename}`; + return ( + + {label} + + ); + } + + return Built-in default; +} + export default function SettingsForm() { const { settings, updateSettings } = useAppStore(); @@ -16,9 +47,7 @@ export default function SettingsForm() { // Connection test state const [testing, setTesting] = useState(false); - const [testResult, setTestResult] = useState( - null, - ); + const [testResult, setTestResult] = useState(null); // Load profiles on mount useEffect(() => { @@ -40,13 +69,17 @@ export default function SettingsForm() { display_name: settings.display_name, log_directory: settings.log_directory, }); - // Check if the current region is not in our standard list setCustomRegion(!AWS_REGIONS.includes(settings.aws_region)); setIsDirty(false); } }, [settings]); + function isLocked(field: keyof AppSettings): boolean { + return settings?.value_sources?.[field]?.source === "env"; + } + function handleChange(field: keyof AppSettings, value: string) { + if (isLocked(field)) return; setFormValues((prev) => ({ ...prev, [field]: value })); setIsDirty(true); setTestResult(null); @@ -66,14 +99,11 @@ export default function SettingsForm() { setTesting(true); setTestResult(null); try { - const result = await apiPost( - "/api/settings/validate", - { - aws_profile: formValues.aws_profile, - aws_region: formValues.aws_region, - s3_bucket: formValues.s3_bucket, - }, - ); + const result = await apiPost("/api/settings/validate", { + aws_profile: formValues.aws_profile, + aws_region: formValues.aws_region, + s3_bucket: formValues.s3_bucket, + }); setTestResult(result); } catch { setTestResult({ success: false, error: "Connection test failed" }); @@ -85,55 +115,54 @@ export default function SettingsForm() { async function handleSave() { setSaving(true); try { - await updateSettings(formValues); + // Filter out env-overridden keys — they can't be changed anyway + const vsrc = settings?.value_sources ?? {}; + const toSave = Object.fromEntries( + Object.entries(formValues).filter(([k]) => vsrc[k]?.source !== "env"), + ); + await updateSettings(toSave); setIsDirty(false); } finally { setSaving(false); } } + const inputBase = + "w-full rounded-md border border-gray-300 px-3 py-2 text-sm focus:border-nlr-blue focus:ring-1 focus:ring-nlr-blue focus:outline-none"; + const inputLocked = "bg-gray-50 text-gray-500 cursor-not-allowed"; + return (
-

- AWS Configuration -

+

AWS Configuration

{/* AWS Profile */}
-
{/* AWS Region */}
-
)} +
{/* S3 Bucket */}
-
{/* Default Upload Folder */} @@ -204,20 +235,17 @@ export default function SettingsForm() { id="default_upload_folder" type="text" value={formValues.default_upload_folder ?? ""} - onChange={(e) => - handleChange("default_upload_folder", e.target.value) - } + onChange={(e) => handleChange("default_upload_folder", e.target.value)} placeholder="/path/to/mcap/files" - className="w-full rounded-md border border-gray-300 px-3 py-2 text-sm focus:border-nlr-blue focus:ring-1 focus:ring-nlr-blue focus:outline-none" + disabled={isLocked("default_upload_folder")} + className={`${inputBase} ${isLocked("default_upload_folder") ? inputLocked : ""}`} /> +
{/* Display Name */}
-
{/* Log Directory */}
-
@@ -283,9 +312,7 @@ export default function SettingsForm() { {isDirty && ( - - Unsaved changes - + Unsaved changes )}
From e10e0dd3056292eff4d25d60f47db9990f7ec33f Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Mon, 2 Mar 2026 07:19:58 -0700 Subject: [PATCH 157/173] Frontend: Add analyzing progress to file table --- frontend/src/components/upload/UnifiedFileTable.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/components/upload/UnifiedFileTable.tsx b/frontend/src/components/upload/UnifiedFileTable.tsx index 86b897e..84ef595 100644 --- a/frontend/src/components/upload/UnifiedFileTable.tsx +++ b/frontend/src/components/upload/UnifiedFileTable.tsx @@ -380,7 +380,7 @@ function StatusBadge({ const labels: Record = { queued: "queued", - in_progress: `${Math.round(progressPercent)}%`, + in_progress: progressPercent > 0 ? `${Math.round(progressPercent)}%` : "analyzing", completed: "uploaded", skipped: "skipped", failed: "failed", From a0fee3c0c97b648afbc9f8eefc28d6711dcbeb2a Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Mon, 2 Mar 2026 07:20:29 -0700 Subject: [PATCH 158/173] Frontend: Refactor upload header with more actionable KPI --- .../src/components/upload/UploadHeader.tsx | 108 ++++++++++++++---- 1 file changed, 85 insertions(+), 23 deletions(-) diff --git a/frontend/src/components/upload/UploadHeader.tsx b/frontend/src/components/upload/UploadHeader.tsx index ab9a9ef..79575c6 100644 --- a/frontend/src/components/upload/UploadHeader.tsx +++ b/frontend/src/components/upload/UploadHeader.tsx @@ -22,7 +22,12 @@ interface UploadHeaderProps { phase: UploadPhase; // Review data - totals: { totalFiles: number; alreadyUploaded: number; totalSize: number }; + totals: { + toUpload: number; + uploadSize: number; + alreadyOnS3: number; + totalInFolder: number; + }; isScanning: boolean; foldersFound: number; @@ -60,7 +65,13 @@ export default function UploadHeader({ onFilterClick, }: UploadHeaderProps) { if (phase === "review") { - return ; + return ( + + ); } if (phase === "uploading") { @@ -89,25 +100,46 @@ function ReviewHeader({ isScanning, foldersFound, }: { - totals: { totalFiles: number; alreadyUploaded: number; totalSize: number }; + totals: { + toUpload: number; + uploadSize: number; + alreadyOnS3: number; + totalInFolder: number; + }; isScanning: boolean; foldersFound: number; }) { - const newFiles = totals.totalFiles - totals.alreadyUploaded; return (
- - - - + + + +
{isScanning && (
- Scanning folders... ({foldersFound} folder{foldersFound !== 1 ? "s" : ""} found so far) + Scanning folders... ({foldersFound} folder + {foldersFound !== 1 ? "s" : ""} found so far)
)} @@ -144,7 +176,9 @@ function UploadingHeader({

- {isRunning && filesProcessed < totalFiles ? "Uploading..." : "Upload Complete"} + {isRunning && filesProcessed < totalFiles + ? "Uploading..." + : "Upload Complete"}

{isRunning && filesProcessed < totalFiles && }
@@ -204,11 +238,12 @@ function SummaryHeader({ }) { if (!job) return null; - const headlineColor = job.files_failed > 0 - ? "text-yellow-600" - : job.cancelled - ? "text-gray-600" - : "text-green-600"; + const headlineColor = + job.files_failed > 0 + ? "text-yellow-600" + : job.cancelled + ? "text-gray-600" + : "text-green-600"; const headlineText = job.cancelled ? "Upload Cancelled" @@ -230,32 +265,57 @@ function SummaryHeader({ onClick={() => onFilterClick?.("completed")} className="text-left" > - + - - - + + +
{job.files_failed > 0 && (
- {job.files_failed} file{job.files_failed !== 1 ? "s" : ""} failed to upload. + {job.files_failed} file + {job.files_failed !== 1 ? "s" : ""} failed to upload. ); From 41862ff9152aa257a861f043b27762a60c0551fc Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Mon, 2 Mar 2026 07:21:48 -0700 Subject: [PATCH 159/173] Frontend: Add batching support to useUploadJob --- frontend/src/hooks/useUploadJob.ts | 105 ++++++++++++++++++++++++++++- 1 file changed, 103 insertions(+), 2 deletions(-) diff --git a/frontend/src/hooks/useUploadJob.ts b/frontend/src/hooks/useUploadJob.ts index 381ba59..7a1f667 100644 --- a/frontend/src/hooks/useUploadJob.ts +++ b/frontend/src/hooks/useUploadJob.ts @@ -15,7 +15,11 @@ import type { AnalysisCompleteEvent, AnalysisProgressEvent, AutoUploadStartingEvent, + BatchCompletedEvent, + BatchProgressEvent, + BatchStartedEvent, FileUploadState, + JobCompletedEvent, UploadJob, UploadJobProgress, } from "../types/api.ts"; @@ -56,6 +60,10 @@ type SSEEvent = | AnalysisProgressEvent | AnalysisCompleteEvent | AutoUploadStartingEvent + | BatchStartedEvent + | BatchProgressEvent + | BatchCompletedEvent + | JobCompletedEvent | UploadJobProgress | UploadJob; @@ -86,7 +94,15 @@ export function useUploadJob(options: UseUploadJobOptions = {}): UseUploadJobRes const [totalBytesFormatted, setTotalBytesFormatted] = useState(""); const [isCancelling, setIsCancelling] = useState(false); - const { setUploadJobId, setCompletedJob } = useUploadStore(); + const { + setUploadJobId, + setCompletedJob, + setCurrentBatch, + setTotalBatches, + setBatchState, + setIsBatchProcessing, + batchState, + } = useUploadStore(); // Callback refs — kept in sync with latest options via effect const onFileUpdateRef = useRef(options.onFileUpdate); @@ -147,6 +163,84 @@ export function useUploadJob(options: UseUploadJobOptions = {}): UseUploadJobRes // arrive via progress dicts. break; + case "batch_started": { + const evt = data as BatchStartedEvent; + setCurrentBatch(evt.batch_id); + setTotalBatches(evt.total_batches); + setIsBatchProcessing(true); + setBatchState({ + batch_id: evt.batch_id, + total_batches: evt.total_batches, + files_in_batch: evt.files_in_batch, + status: "processing", + files_processed: 0, + files_uploaded: 0, + files_failed: 0, + bytes_uploaded: 0, + started_at: new Date().toISOString(), + completed_at: null, + duration_seconds: null, + error_message: "", + }); + break; + } + + case "batch_progress": { + const evt = data as BatchProgressEvent; + setCurrentBatch(evt.batch_id); + // Limit active files to 8 items max + setActiveFiles(evt.active_files.slice(0, 8)); + setFilesProcessed(evt.job_files_completed); + setTotalFiles(evt.job_files_total); + setProgressPercent(evt.job_progress_percent); + + // Update batch state with current progress + if (batchState) { + setBatchState({ + ...batchState, + files_processed: evt.batch_files_completed, + }); + } + + // Notify unified table for each active file + if (onFileUpdateRef.current) { + for (const file of evt.active_files.slice(0, 8)) { + onFileUpdateRef.current(file); + } + } + break; + } + + case "batch_completed": { + const evt = data as BatchCompletedEvent; + if (batchState) { + setBatchState({ + ...batchState, + status: "completed", + files_uploaded: evt.files_uploaded, + files_failed: evt.files_failed, + completed_at: new Date().toISOString(), + }); + } + break; + } + + case "job_completed": { + // Job completed event - reset batch state + setIsRunning(false); + setIsCancelling(false); + setJobId(null); + setIsBatchProcessing(false); + setCurrentBatch(null); + setTotalBatches(null); + setBatchState(null); + + // For large jobs, fetch paginated results instead of receiving all files + // The full file list would be too large for SSE + // Client should call /api/upload/results/ for pagination + break; + } + default: break; } @@ -213,7 +307,14 @@ export function useUploadJob(options: UseUploadJobOptions = {}): UseUploadJobRes return; } }, - [setCompletedJob], + [ + setCompletedJob, + setCurrentBatch, + setTotalBatches, + setBatchState, + setIsBatchProcessing, + batchState, + ], ); const sseUrl = useMemo( From 7cd2158b28d452526070ec9deda5e23234fd11a4 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Mon, 2 Mar 2026 07:22:09 -0700 Subject: [PATCH 160/173] Frontend: Add PerformanceSection to Settings --- frontend/src/pages/SettingsPage.tsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frontend/src/pages/SettingsPage.tsx b/frontend/src/pages/SettingsPage.tsx index cf2a901..dfe8055 100644 --- a/frontend/src/pages/SettingsPage.tsx +++ b/frontend/src/pages/SettingsPage.tsx @@ -1,6 +1,7 @@ import { useEffect } from "react"; import CacheSection from "../components/settings/CacheSection.tsx"; import DangerZone from "../components/settings/DangerZone.tsx"; +import PerformanceSection from "../components/settings/PerformanceSection.tsx"; import SettingsForm from "../components/settings/SettingsForm.tsx"; import UpdateSection from "../components/settings/UpdateSection.tsx"; import { useAppStore } from "../stores/appStore.ts"; @@ -24,6 +25,7 @@ export default function SettingsPage() {

Settings

+ From 043b910e7e958f8993a24a5e1a615909de6b8e5d Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Mon, 2 Mar 2026 07:22:46 -0700 Subject: [PATCH 161/173] Frontend: Wire in batching to upload page --- frontend/src/pages/UploadPage.tsx | 162 ++++++++++++++++++++++-------- 1 file changed, 120 insertions(+), 42 deletions(-) diff --git a/frontend/src/pages/UploadPage.tsx b/frontend/src/pages/UploadPage.tsx index 6f27c7d..d4ff7a4 100644 --- a/frontend/src/pages/UploadPage.tsx +++ b/frontend/src/pages/UploadPage.tsx @@ -6,14 +6,17 @@ * with phase-aware header, toolbar, and footer. */ -import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { useCallback, useEffect, useMemo, useState } from "react"; import Spinner from "../components/common/Spinner.tsx"; +import ActiveFilesList from "../components/upload/ActiveFilesList.tsx"; +import BatchProgress from "../components/upload/BatchProgress.tsx"; import CancelConfirmModal from "../components/upload/CancelConfirmModal.tsx"; import ConfirmModal from "../components/upload/ConfirmModal.tsx"; import FolderBrowser from "../components/upload/FolderBrowser.tsx"; import type { FolderExclusions } from "../components/upload/FolderBrowser.tsx"; import ReviewToolbar, { StatusFilterBar } from "../components/upload/ReviewToolbar.tsx"; +import ScanProgressModal from "../components/upload/ScanProgressModal.tsx"; import Stepper from "../components/upload/Stepper.tsx"; import UnifiedFileTable from "../components/upload/UnifiedFileTable.tsx"; import UploadFooter from "../components/upload/UploadFooter.tsx"; @@ -34,8 +37,9 @@ export default function UploadPage() { folderPath, setFolderPath, scanFolders, - scanTotals, completedJob, + batchState, + isBatchProcessing, reset, } = useUploadStore(); @@ -47,6 +51,7 @@ export default function UploadPage() { isScanning, scanComplete, folders, + foldersTotal, totals, } = useFolderScan(); @@ -89,29 +94,21 @@ export default function UploadPage() { const [pendingSelectedPaths, setPendingSelectedPaths] = useState([]); const [selectedPaths, setSelectedPaths] = useState>(new Set()); + // Delay scan modal by 500 ms — fast/cached scans complete before the timer + // fires, so the modal never flashes for them. + const [showScanModal, setShowScanModal] = useState(false); + useEffect(() => { + if (!isScanning) { + setShowScanModal(false); + return; + } + const timer = setTimeout(() => setShowScanModal(true), 500); + return () => clearTimeout(timer); + }, [isScanning]); + // Derive phase from step const phase: UploadPhase = step <= 2 ? "review" : step === 3 ? "uploading" : "summary"; - // ── Sync FileStore from scan data as folders arrive ── - - const prevFoldersRef = useRef(folders); - if (folders !== prevFoldersRef.current) { - prevFoldersRef.current = folders; - if (folders.length > 0 && step >= 2) { - store.buildFromScan(folders); - // Auto-select new files - const newSelected = new Set(); - for (const folder of folders) { - for (const file of folder.files) { - if (!file.already_uploaded) { - newSelected.add(file.path); - } - } - } - setSelectedPaths(newSelected); - } - } - // ── Freeze/unfreeze sort on phase transitions ── useEffect(() => { @@ -121,17 +118,40 @@ export default function UploadPage() { // ── Step transitions ── - /** Step 1 -> 2: User selected a folder. Start scanning and advance. */ + /** Step 1: User selected a folder. Start scanning (stay on Step 1 while scanning). */ const handleFolderSelected = useCallback( async (path: string, exclusions?: FolderExclusions) => { store.clear(); setFolderPath(path); - setStep(2); + // Stay on Step 1 while scanning - auto-advance when complete await startScan(path, false, exclusions); }, - [setFolderPath, setStep, startScan, store], + [setFolderPath, startScan, store], ); + /** Auto-advance to Step 2 when scan completes. + * + * We populate the FileStore here, right before advancing, because by the time + * scanComplete becomes true the folders array reference has already settled — + * any render-time ref-comparison trick would have already consumed the change + * while step was still 1 and skipped the buildFromScan call. + */ + useEffect(() => { + if (step === 1 && scanComplete && folders.length > 0) { + store.buildFromScan(folders); + const newSelected = new Set(); + for (const folder of folders) { + for (const file of folder.files) { + if (!file.already_uploaded) { + newSelected.add(file.path); + } + } + } + setSelectedPaths(newSelected); + setStep(2); + } + }, [step, scanComplete, folders, store, setSelectedPaths, setStep]); + /** Step 2: User clicks "Start Upload" — show confirmation modal. */ const handleStartUploadClick = useCallback(() => { setPendingSelectedPaths(Array.from(selectedPaths)); @@ -180,6 +200,15 @@ export default function UploadPage() { reset(); }, [reset, store]); + /** Cancel scan and return to Step 1 if needed. */ + const handleCancelScan = useCallback(async () => { + await cancelScan(); + // If we're on a later step during scan (shouldn't happen with new flow, but handle it) + if (step > 1) { + setStep(1); + } + }, [cancelScan, step, setStep]); + /** Back from step 2 -> step 1. */ const handleBack = useCallback(async () => { if (isScanning) { @@ -291,9 +320,30 @@ export default function UploadPage() { downloadUploadCsv(Array.from(store.getAllRows().values()), jobId); }, [completedJob, store]); - // ── Active totals for header ── - - const activeTotals = scanComplete ? totals : scanTotals; + // ── Review KPIs: what is actually going to happen when the user clicks Upload ── + // + // toUpload — new selected files (will be sent to S3) + // uploadSize — bytes of those new files + // alreadyOnS3 — selected files already there (will be skipped) + // totalInFolder — all files found in the scan (context denominator) + const activeTotals = useMemo(() => { + let toUpload = 0; + let uploadSize = 0; + let alreadyOnS3 = 0; + for (const file of allFiles) { + if (selectedPaths.has(file.path)) { + if (file.alreadyUploaded) { + alreadyOnS3++; + } else { + toUpload++; + uploadSize += file.size; + } + } + } + return { toUpload, uploadSize, alreadyOnS3, totalInFolder: allFiles.length }; + // `files` dep ensures we recompute when the store snapshot changes + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [allFiles, selectedPaths, files]); // ── Render ── @@ -371,20 +421,36 @@ export default function UploadPage() { /> )} - {/* The unified file table — stays mounted across phases */} - store.setSort(key)} - isSortFrozen={store.isFrozen()} - selectedPaths={selectedPaths} - onToggleFile={toggleFile} - onToggleAllFiltered={toggleAllFiltered} - headerChecked={headerChecked} - headerIndeterminate={headerIndeterminate} - /> + {/* Batch processing UI for upload phase with large jobs */} + {phase === "uploading" && isBatchProcessing ? ( + <> + + + + ) : ( + /* The unified file table — stays mounted across phases */ + store.setSort(key)} + isSortFrozen={store.isFrozen()} + selectedPaths={selectedPaths} + onToggleFile={toggleFile} + onToggleAllFiltered={toggleAllFiltered} + headerChecked={headerChecked} + headerIndeterminate={headerIndeterminate} + /> + )} {/* Bottom action buttons */} )} + + {/* Scan progress modal - only shown after 500 ms delay so fast/cached + scans never flash the modal at all. */} +
); } From 1a0f3ade9e80b68037af32470eb09c22edab0323 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Mon, 2 Mar 2026 07:23:10 -0700 Subject: [PATCH 162/173] Frontend: Add batching support to uploadStore --- frontend/src/stores/uploadStore.ts | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/frontend/src/stores/uploadStore.ts b/frontend/src/stores/uploadStore.ts index 5efb416..cc48b63 100644 --- a/frontend/src/stores/uploadStore.ts +++ b/frontend/src/stores/uploadStore.ts @@ -1,5 +1,5 @@ import { create } from "zustand"; -import type { ScannedFolder, UploadJob } from "../types/api.ts"; +import type { BatchState, ScannedFolder, UploadJob } from "../types/api.ts"; export type UploadStep = 1 | 2 | 3 | 4; @@ -17,6 +17,7 @@ interface UploadState { scanFolders: ScannedFolder[]; scanComplete: boolean; isScanning: boolean; + scanFoldersTotal: number; scanTotals: { totalFiles: number; alreadyUploaded: number; @@ -26,6 +27,7 @@ interface UploadState { addScanFolder: (folder: ScannedFolder) => void; setScanComplete: (complete: boolean) => void; setIsScanning: (scanning: boolean) => void; + setScanFoldersTotal: (total: number) => void; updateScanTotals: (totals: { totalFiles: number; alreadyUploaded: number; totalSize: number }) => void; // Upload job @@ -34,6 +36,16 @@ interface UploadState { setUploadJobId: (id: string | null) => void; setCompletedJob: (job: UploadJob | null) => void; + // Batch processing state + currentBatch: number | null; + totalBatches: number | null; + batchState: BatchState | null; + isBatchProcessing: boolean; + setCurrentBatch: (batch: number | null) => void; + setTotalBatches: (batches: number | null) => void; + setBatchState: (state: BatchState | null) => void; + setIsBatchProcessing: (processing: boolean) => void; + // Reset reset: () => void; } @@ -45,9 +57,14 @@ const initialState = { scanFolders: [], scanComplete: false, isScanning: false, + scanFoldersTotal: 0, scanTotals: { totalFiles: 0, alreadyUploaded: 0, totalSize: 0 }, uploadJobId: null, completedJob: null, + currentBatch: null, + totalBatches: null, + batchState: null, + isBatchProcessing: false, }; export const useUploadStore = create((set) => ({ @@ -61,10 +78,16 @@ export const useUploadStore = create((set) => ({ set((s) => ({ scanFolders: [...s.scanFolders, folder] })), setScanComplete: (scanComplete) => set({ scanComplete }), setIsScanning: (isScanning) => set({ isScanning }), + setScanFoldersTotal: (scanFoldersTotal) => set({ scanFoldersTotal }), updateScanTotals: (scanTotals) => set({ scanTotals }), setUploadJobId: (uploadJobId) => set({ uploadJobId }), setCompletedJob: (completedJob) => set({ completedJob }), + setCurrentBatch: (currentBatch) => set({ currentBatch }), + setTotalBatches: (totalBatches) => set({ totalBatches }), + setBatchState: (batchState) => set({ batchState }), + setIsBatchProcessing: (isBatchProcessing) => set({ isBatchProcessing }), + reset: () => set(initialState), })); From bd74dc4289d82b29078ada3f543d619635d2dfb4 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Mon, 2 Mar 2026 07:23:41 -0700 Subject: [PATCH 163/173] Frontend: Add batching and performance types --- frontend/src/types/api.ts | 84 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/frontend/src/types/api.ts b/frontend/src/types/api.ts index a2e6d2f..fb71ea2 100644 --- a/frontend/src/types/api.ts +++ b/frontend/src/types/api.ts @@ -117,6 +117,90 @@ export interface AutoUploadStartingEvent { job_id: string; } +// ── Batch processing types ── + +export interface BatchProcessingSettings { + enabled: boolean; + batch_size: number; + auto_tune_workers: boolean; + max_workers: number; + target_cpu_percent: number; + skip_mcap_validation: boolean; + use_database_for_large_jobs: boolean; + large_job_threshold: number; +} + +export interface BatchState { + batch_id: number; + total_batches: number; + files_in_batch: number; + status: "pending" | "processing" | "completed" | "failed" | "cancelled"; + files_processed: number; + files_uploaded: number; + files_failed: number; + bytes_uploaded: number; + started_at: string | null; + completed_at: string | null; + duration_seconds: number | null; + error_message: string; +} + +export interface BatchStartedEvent { + type: "batch_started"; + batch_id: number; + total_batches: number; + files_in_batch: number; +} + +export interface BatchProgressEvent { + type: "batch_progress"; + batch_id: number; + active_files: FileUploadState[]; // Max 8 items + batch_files_completed: number; + batch_files_total: number; + job_files_completed: number; + job_files_total: number; + job_progress_percent: number; +} + +export interface BatchCompletedEvent { + type: "batch_completed"; + batch_id: number; + files_uploaded: number; + files_failed: number; +} + +export interface JobCompletedEvent { + type: "job_completed"; + job_id: string; + status: "completed" | "failed" | "cancelled"; + total_files: number; + files_uploaded: number; + files_failed: number; + duration_seconds: number; +} + +export interface PaginatedResults { + job_id: string; + files: FileUploadState[]; + pagination: { + page: number; + per_page: number; + total_files: number; + total_pages: number; + has_next: boolean; + has_prev: boolean; + }; + job_metadata: { + job_id: string; + status: string; + total_files: number; + files_uploaded: number; + files_failed: number; + total_bytes: number; + }; +} + // ── Scan types ── export interface ScannedFileInfo { From e102fd6c51b903fa7079ecbf2c76a50618e37df0 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Mon, 2 Mar 2026 07:24:18 -0700 Subject: [PATCH 164/173] Frontend: Add settings provenance types --- frontend/src/types/api.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/frontend/src/types/api.ts b/frontend/src/types/api.ts index fb71ea2..a753444 100644 --- a/frontend/src/types/api.ts +++ b/frontend/src/types/api.ts @@ -332,6 +332,16 @@ export interface S3Breadcrumb { // ── Settings types ── +export type ValueSourceType = "builtin" | "default_file" | "settings_file" | "env"; + +export interface ValueSource { + source: ValueSourceType; + /** Absolute path to the file (present for default_file and settings_file). */ + path?: string; + /** Environment variable name (present for env). */ + env_var?: string; +} + export interface AppSettings { aws_profile: string; aws_region: string; @@ -339,6 +349,9 @@ export interface AppSettings { default_upload_folder: string; display_name: string; log_directory: string; + batch_processing?: BatchProcessingSettings; + /** Provenance metadata returned by the API — not sent on PUT. */ + value_sources?: Record; } export interface VersionInfo { From 1065310bb37b56486ce501e008f941f1b9519c6b Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Mon, 2 Mar 2026 07:24:38 -0700 Subject: [PATCH 165/173] Settings: Add default batch processing settings --- settings.default.json | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/settings.default.json b/settings.default.json index 7b271cb..a633a92 100644 --- a/settings.default.json +++ b/settings.default.json @@ -3,5 +3,15 @@ "aws_region": "us-west-2", "s3_bucket": "", "default_upload_folder": "", - "log_directory": "logs" + "log_directory": "logs", + "batch_processing": { + "enabled": true, + "batch_size": 100, + "auto_tune_workers": true, + "max_workers": 4, + "target_cpu_percent": 70.0, + "skip_mcap_validation": false, + "use_database_for_large_jobs": true, + "large_job_threshold": 1000 + } } From 79d1dd167db74beef2a26036b157732f0b491c3f Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Mon, 2 Mar 2026 07:24:57 -0700 Subject: [PATCH 166/173] Tests: Add logs and infra tests --- tests/test_log_service.py | 32 ++----- tests/test_sse_infrastructure.py | 158 +++++++++++++++++++++++++++++++ 2 files changed, 166 insertions(+), 24 deletions(-) create mode 100644 tests/test_sse_infrastructure.py diff --git a/tests/test_log_service.py b/tests/test_log_service.py index 9f5c264..132f856 100644 --- a/tests/test_log_service.py +++ b/tests/test_log_service.py @@ -113,9 +113,7 @@ def test_error_convenience(self, log_service: LogService, _mock_settings: Any) - entry = json.loads(log_file.read_text().strip()) assert entry["level"] == "ERROR" - def test_multiple_entries_appended( - self, log_service: LogService, _mock_settings: Any - ) -> None: + def test_multiple_entries_appended(self, log_service: LogService, _mock_settings: Any) -> None: """Test that multiple log calls append to the same file.""" with _mock_settings: log_service.info("app", "event1", "First") @@ -127,9 +125,7 @@ def test_multiple_entries_appended( lines = [line for line in log_file.read_text().strip().split("\n") if line] assert len(lines) == 3 - def test_no_metadata_omits_field( - self, log_service: LogService, _mock_settings: Any - ) -> None: + def test_no_metadata_omits_field(self, log_service: LogService, _mock_settings: Any) -> None: """Test that metadata field is omitted when not provided.""" with _mock_settings: log_service.info("app", "test", "No metadata") @@ -206,9 +202,7 @@ def test_read_entries_date_filter_hive( assert result["total"] == 1 assert result["entries"][0]["event"] == "event1" - def test_read_entries_invalid_date( - self, log_service: LogService, _mock_settings: Any - ) -> None: + def test_read_entries_invalid_date(self, log_service: LogService, _mock_settings: Any) -> None: """Test that an invalid date returns empty results.""" with _mock_settings: result = log_service.read_log_entries(date="not-a-date") @@ -284,9 +278,7 @@ def test_empty_stats(self, log_service: LogService, _mock_settings: Any) -> None assert stats["total_entries"] == 0 assert stats["file_count"] == 0 - def test_stats_includes_csv_count( - self, log_service: LogService, _mock_settings: Any - ) -> None: + def test_stats_includes_csv_count(self, log_service: LogService, _mock_settings: Any) -> None: """Test that stats include csv_count.""" with _mock_settings: log_service.info("app", "test", "Entry") @@ -386,9 +378,7 @@ def test_extract_date_from_non_hive_path(self) -> None: result = LogService._extract_date_from_hive_path(path) assert result is None - def test_log_writes_to_hive_path( - self, log_service: LogService, _mock_settings: Any - ) -> None: + def test_log_writes_to_hive_path(self, log_service: LogService, _mock_settings: Any) -> None: """Test that log() writes to a hive-partitioned events.jsonl.""" log_dir: Path = log_service._test_settings_mock.log_directory # type: ignore[attr-defined] @@ -471,9 +461,7 @@ def _make_mock_job(self) -> MagicMock: mock_job.files = [mock_file] return mock_job - def test_save_job_csv_creates_file( - self, log_service: LogService, _mock_settings: Any - ) -> None: + def test_save_job_csv_creates_file(self, log_service: LogService, _mock_settings: Any) -> None: """Test that save_job_csv creates a CSV at the correct hive path.""" log_dir: Path = log_service._test_settings_mock.log_directory # type: ignore[attr-defined] job_id = "a1b2c3d4-5678-9abc-def0-1234567890ab" @@ -489,9 +477,7 @@ def test_save_job_csv_creates_file( assert result_path.name.endswith(".csv") assert result_path.exists() - def test_save_job_csv_columns( - self, log_service: LogService, _mock_settings: Any - ) -> None: + def test_save_job_csv_columns(self, log_service: LogService, _mock_settings: Any) -> None: """Test that the CSV has all 14 expected columns.""" job_id = "test-job-id-1234" completed_at = datetime(2026, 2, 8, 12, 0, 0, tzinfo=UTC) @@ -522,9 +508,7 @@ def test_save_job_csv_columns( ] assert header == expected_columns - def test_save_job_csv_data_row( - self, log_service: LogService, _mock_settings: Any - ) -> None: + def test_save_job_csv_data_row(self, log_service: LogService, _mock_settings: Any) -> None: """Test that the CSV has a data row for each file.""" job_id = "test-job-id" completed_at = datetime(2026, 2, 8, 12, 0, 0, tzinfo=UTC) diff --git a/tests/test_sse_infrastructure.py b/tests/test_sse_infrastructure.py new file mode 100644 index 0000000..e608457 --- /dev/null +++ b/tests/test_sse_infrastructure.py @@ -0,0 +1,158 @@ +"""Tests for SSE infrastructure: event signaling, queue management, and resource cleanup.""" + +import threading +import time +from collections.abc import Generator +from typing import Any + +import pytest + +from app.routes.upload import ( + SSE_QUEUE_TTL_SECONDS, + _cleanup_old_sse_queues, + _sse_events, + _sse_queues, + _sse_timestamps, + send_sse_event, +) + + +@pytest.fixture +def clear_sse_state() -> Generator[None, None, None]: + """Clear SSE module state before each test.""" + _sse_queues.clear() + _sse_events.clear() + _sse_timestamps.clear() + yield + _sse_queues.clear() + _sse_events.clear() + _sse_timestamps.clear() + + +def test_send_sse_event_creates_timestamp(clear_sse_state: Any) -> None: + """Test that sending an event updates the timestamp.""" + from collections import deque + + job_id = "test-job-123" + + # Create a queue manually + queue = deque() + _sse_queues[job_id] = [queue] + + # Send event + send_sse_event(job_id, {"type": "test", "data": "hello"}) + + # Verify timestamp was created + assert job_id in _sse_timestamps + assert time.time() - _sse_timestamps[job_id] < 1 # Within 1 second + + +def test_send_sse_event_signals_waiting_threads(clear_sse_state: Any) -> None: + """Test that sending an event signals the threading.Event.""" + from collections import deque + + job_id = "test-job-456" + + # Create queue and event + queue = deque() + event = threading.Event() + _sse_queues[job_id] = [queue] + _sse_events[job_id] = event + + # Event should not be set initially + assert not event.is_set() + + # Send event + send_sse_event(job_id, {"type": "test"}) + + # Event should now be set + assert event.is_set() + assert len(queue) == 1 + + +def test_cleanup_removes_old_queues(clear_sse_state: Any) -> None: + """Test that cleanup removes expired queues.""" + from collections import deque + + # Create some queues with old timestamps + old_time = time.time() - SSE_QUEUE_TTL_SECONDS - 100 + recent_time = time.time() + + _sse_queues["old-job-1"] = [deque()] + _sse_timestamps["old-job-1"] = old_time + _sse_events["old-job-1"] = threading.Event() + + _sse_queues["old-job-2"] = [deque()] + _sse_timestamps["old-job-2"] = old_time + + _sse_queues["recent-job"] = [deque()] + _sse_timestamps["recent-job"] = recent_time + + # Run cleanup + removed = _cleanup_old_sse_queues() + + # Should remove 2 old jobs, keep recent one + assert removed == 2 + assert "old-job-1" not in _sse_queues + assert "old-job-1" not in _sse_events + assert "old-job-1" not in _sse_timestamps + assert "old-job-2" not in _sse_queues + assert "recent-job" in _sse_queues + + +def test_cleanup_with_no_expired_queues(clear_sse_state: Any) -> None: + """Test that cleanup does nothing when all queues are recent.""" + from collections import deque + + recent_time = time.time() + + _sse_queues["job-1"] = [deque()] + _sse_timestamps["job-1"] = recent_time + + _sse_queues["job-2"] = [deque()] + _sse_timestamps["job-2"] = recent_time + + # Run cleanup + removed = _cleanup_old_sse_queues() + + # Should remove nothing + assert removed == 0 + assert len(_sse_queues) == 2 + + +def test_event_driven_signaling(clear_sse_state: Any) -> None: + """Test that Event.wait() is more efficient than polling.""" + from collections import deque + + job_id = "test-job-signal" + queue = deque() + event = threading.Event() + + _sse_queues[job_id] = [queue] + _sse_events[job_id] = event + + # Simulate waiting thread + wait_result: list[float] = [] + + def waiter() -> None: + # This should block until event is set + start = time.time() + event.wait(timeout=2.0) + elapsed = time.time() - start + wait_result.append(elapsed) + + thread = threading.Thread(target=waiter) + thread.start() + + # Small delay to ensure thread is waiting + time.sleep(0.1) + + # Send event to wake thread + send_sse_event(job_id, {"type": "wake"}) + + # Wait for thread to finish + thread.join(timeout=3.0) + + # Thread should have woken up quickly (< 0.5s, not 2s timeout) + assert len(wait_result) == 1 + assert wait_result[0] < 0.5 # Should be nearly instant From 792d3f710575eb6c4977e4302c97c5cf02f6ac72 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Mon, 2 Mar 2026 07:25:30 -0700 Subject: [PATCH 167/173] Backend: Add sse registry/cleanup to init.py --- app/__init__.py | 51 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/app/__init__.py b/app/__init__.py index aaa06e4..d418c3a 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -1,11 +1,56 @@ """Flask application factory for MCAP S3 Uploader.""" +import atexit import os +import threading from flask import Flask from app.config import get_package_version, get_settings +# Background cleanup thread control +_cleanup_thread: threading.Thread | None = None +_cleanup_stop_event = threading.Event() + + +def _sse_cleanup_worker() -> None: + """Background worker that periodically cleans up stale SSE queues.""" + from app.routes.upload import _cleanup_old_sse_queues + + while not _cleanup_stop_event.wait(timeout=300): # Check every 5 minutes + try: + removed = _cleanup_old_sse_queues() + if removed > 0: + from app.services.log_service import get_log_service + + log = get_log_service() + log.info( + "sse", + "sse_cleanup", + f"Cleaned up {removed} stale SSE queues", + {"queues_removed": removed}, + ) + except Exception: + # Don't crash the cleanup thread on errors + pass + + +def _start_sse_cleanup() -> None: + """Start the background SSE cleanup thread.""" + global _cleanup_thread + if _cleanup_thread is None: + _cleanup_thread = threading.Thread( + target=_sse_cleanup_worker, daemon=True, name="SSECleanup" + ) + _cleanup_thread.start() + + +def _stop_sse_cleanup() -> None: + """Stop the background SSE cleanup thread.""" + _cleanup_stop_event.set() + if _cleanup_thread: + _cleanup_thread.join(timeout=2.0) + def create_app() -> Flask: """Create and configure the Flask application.""" @@ -39,6 +84,12 @@ def inject_display_name() -> dict[str, str]: app.register_blueprint(logs_bp, url_prefix="/api/logs") app.register_blueprint(delete_bp, url_prefix="/api/delete") + # Start background SSE cleanup thread + _start_sse_cleanup() + + # Register cleanup on shutdown + atexit.register(_stop_sse_cleanup) + # Log application startup from app.services.log_service import get_log_service From c513777e29cb45bf59cad4a5d84321732f2104c2 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Mon, 2 Mar 2026 09:04:14 -0700 Subject: [PATCH 168/173] Fix: Don't cancel futures --- app/services/upload_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/services/upload_manager.py b/app/services/upload_manager.py index 849b9bb..2237018 100644 --- a/app/services/upload_manager.py +++ b/app/services/upload_manager.py @@ -1367,8 +1367,8 @@ def byte_callback(uploaded: int, total: int) -> None: {"job_id": job_id, "error": str(e)}, ) finally: - # Wait for in-flight uploads; cancel_work_items prevents queued tasks from starting - upload_executor.shutdown(wait=True, cancel_futures=True) + # Wait for ALL uploads (in-flight + queued) to complete + upload_executor.shutdown(wait=True) # Mark any files still in non-terminal states as cancelled if job.cancelled: From 862b489faa1314144d79a111730baa73e61dd859 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Mon, 2 Mar 2026 09:37:34 -0700 Subject: [PATCH 169/173] Actions: Set js tests working dir to frontend --- .github/workflows/run_tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 5e7bb19..6b3b087 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -62,9 +62,12 @@ jobs: with: node-version: 22 cache: npm + cache-dependency-path: frontend/package-lock.json - name: Install dependencies run: npm ci + working-directory: frontend - name: Lint, type check, and test run: npm run check + working-directory: frontend From 732b1e003d54497d4578ec556fddd6a19b7fb980 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Mon, 2 Mar 2026 09:37:52 -0700 Subject: [PATCH 170/173] Pre Commit: Skip json files with comments --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 924d336..24add70 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,6 +9,7 @@ repos: - id: check-merge-conflict - id: check-yaml - id: check-json + exclude: ^frontend/tsconfig\. - id: no-commit-to-branch args: [--branch, main] From 4f6ccc3cb566d280e73fbfff796eb9b3707a61c3 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Mon, 2 Mar 2026 09:38:08 -0700 Subject: [PATCH 171/173] Dev: Add psutil type stubs --- requirements-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index 1b738ee..8a9e9e5 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,6 +2,7 @@ pytest>=8.0.0 pytest-cov>=4.1.0 moto[s3]>=5.0.0 mypy>=1.8.0 +types-psutil>=5.9.0 boto3-stubs[s3]>=1.34.0 ruff>=0.2.0 pre-commit>=3.6.0 From 210d152e266ee6a817ddb2f3322ccc2094e93444 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Mon, 2 Mar 2026 09:39:06 -0700 Subject: [PATCH 172/173] Release: Bump to 1.0.0 --- frontend/package.json | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/package.json b/frontend/package.json index 44356f3..2686499 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,7 +1,7 @@ { "name": "frontend", "private": true, - "version": "0.0.0", + "version": "1.0.0", "type": "module", "scripts": { "dev": "vite", diff --git a/pyproject.toml b/pyproject.toml index 6670e02..fca511f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "modaq-upload" -version = "0.2.2" +version = "1.0.0" description = "Python/Flask web application for uploading MODAQ data to Amazon AWS S3 buckets" requires-python = ">=3.11" authors = [ From 355d68d4dc199b4849f4c886feff8f8ecbd9d1d0 Mon Sep 17 00:00:00 2001 From: "Simms, Andrew" Date: Mon, 2 Mar 2026 09:42:10 -0700 Subject: [PATCH 173/173] Tests: Add shell html index for flask to test against --- tests/conftest.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 0bf19e8..e3896aa 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -31,6 +31,17 @@ def app() -> Generator[Flask, None, None]: ) temp_settings = f.name + # Ensure frontend/dist/index.html exists so SPA routes can serve it + from app.routes.main import FRONTEND_DIST + + dist_created = False + index_path = os.path.join(FRONTEND_DIST, "index.html") + if not os.path.exists(index_path): + os.makedirs(FRONTEND_DIST, exist_ok=True) + dist_created = True + with open(index_path, "w") as f: + f.write('
') + # Create the app (settings will be loaded from default) _ = SETTINGS_FILE # Reference to avoid unused import warning @@ -41,6 +52,8 @@ def app() -> Generator[Flask, None, None]: # Cleanup os.unlink(temp_settings) + if dist_created: + os.unlink(index_path) @pytest.fixture