diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml new file mode 100644 index 0000000..5e7bb19 --- /dev/null +++ b/.github/workflows/run_tests.yml @@ -0,0 +1,70 @@ +name: Run Tests + +on: + # Runs when commits are pushed directly to these branches + # Note: we are purposly skipping push in favor of branch protection rules + # that require PRs to be opened against these branches. This ensures that all + # changes are reviewed and pass tests before being merged. This avoids + # running tests on every push to feature branches, which can be noisy and + # inefficient. Instead, tests will only run when a PR is opened or updated + # that targets the main or develop branches. + # push: + # branches: [main, develop] + # Runs when a PR is opened/updated that wants to merge INTO these branches + pull_request: + branches: [main, develop] + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - uses: pre-commit/action@v3.0.1 + + python: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: pip + cache-dependency-path: | + requirements.txt + requirements-dev.txt + + - name: Install dependencies + run: | + pip install -r requirements.txt + pip install -r requirements-dev.txt + + - name: Lint + run: ruff check app/ tests/ + + - name: Type check + run: mypy app/ + + - name: Test + run: pytest tests/ -v + + javascript: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + + - name: Install dependencies + run: npm ci + + - name: Lint, type check, and test + run: npm run check diff --git a/.gitignore b/.gitignore index 77ff673..447eb83 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,9 @@ settings.json # SQLite cache modaq_upload_cache.db +# Log files +logs/ + # Python __pycache__/ *.py[cod] diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..924d336 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,19 @@ +repos: + # Standard safety hooks + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: detect-private-key + - id: check-added-large-files + args: [--maxkb=1000] + - id: check-merge-conflict + - id: check-yaml + - id: check-json + - id: no-commit-to-branch + args: [--branch, main] + + # Comprehensive secret detection + - repo: https://github.com/gitleaks/gitleaks + rev: v8.21.2 + hooks: + - id: gitleaks diff --git a/app/__init__.py b/app/__init__.py index 38559d8..7c69fc9 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -4,7 +4,7 @@ from flask import Flask -from app.config import get_settings +from app.config import get_package_version, get_settings def create_app() -> Flask: @@ -26,6 +26,7 @@ def inject_display_name() -> dict[str, str]: # Register blueprints from app.routes.files import files_bp + from app.routes.logs import logs_bp from app.routes.main import main_bp from app.routes.settings import settings_bp from app.routes.upload import upload_bp @@ -34,5 +35,17 @@ def inject_display_name() -> dict[str, str]: app.register_blueprint(upload_bp, url_prefix="/api/upload") app.register_blueprint(files_bp, url_prefix="/api/files") app.register_blueprint(settings_bp, url_prefix="/api/settings") + app.register_blueprint(logs_bp, url_prefix="/api/logs") + + # Log application startup + from app.services.log_service import get_log_service + + log = get_log_service() + log.info( + "app", + "app_started", + f"Application started (v{get_package_version()})", + {"version": get_package_version()}, + ) return app diff --git a/app/config.py b/app/config.py index 72cb7d5..432314c 100644 --- a/app/config.py +++ b/app/config.py @@ -1,5 +1,6 @@ """Configuration management for modaq_upload""" +import functools import json import os import subprocess @@ -28,26 +29,28 @@ ENV_S3_BUCKET = "MODAQ_S3_BUCKET" ENV_DEFAULT_UPLOAD_FOLDER = "MODAQ_DEFAULT_UPLOAD_FOLDER" ENV_DISPLAY_NAME = "MODAQ_DISPLAY_NAME" +ENV_LOG_DIRECTORY = "MODAQ_LOG_DIRECTORY" -def get_package_version() -> str: - """Get the package version from pyproject.toml.""" +@functools.cache +def _read_pyproject_field(field: str, default: str) -> str: + """Read a field from pyproject.toml [project] section (cached).""" try: with open(PYPROJECT_FILE, "rb") as f: pyproject = tomllib.load(f) - return str(pyproject.get("project", {}).get("version", "0.0.0")) + return str(pyproject.get("project", {}).get(field, default)) except Exception: - return "0.0.0" + return default + + +def get_package_version() -> str: + """Get the package version from pyproject.toml.""" + return _read_pyproject_field("version", "0.0.0") def get_package_name() -> str: """Get the package name from pyproject.toml.""" - try: - with open(PYPROJECT_FILE, "rb") as f: - pyproject = tomllib.load(f) - return str(pyproject.get("project", {}).get("name", "modaq-uploader")) - except Exception: - return "modaq-uploader" + return _read_pyproject_field("name", "modaq-uploader") class Settings: @@ -79,6 +82,7 @@ def _load_settings(self) -> None: "s3_bucket": "", "default_upload_folder": "", "display_name": "MODAQ Uploader", + "log_directory": "logs", } # Load from settings.default.json if it exists @@ -98,6 +102,7 @@ def _load_settings(self) -> None: "s3_bucket": os.environ.get(ENV_S3_BUCKET), "default_upload_folder": os.environ.get(ENV_DEFAULT_UPLOAD_FOLDER), "display_name": os.environ.get(ENV_DISPLAY_NAME), + "log_directory": os.environ.get(ENV_LOG_DIRECTORY), } # Only apply non-None environment values @@ -141,27 +146,36 @@ def reload(self) -> None: @property def aws_profile(self) -> str: """Get the AWS profile name.""" - return str(self._settings.get("aws_profile", "default")) + return str(self._settings["aws_profile"]) @property def aws_region(self) -> str: """Get the AWS region.""" - return str(self._settings.get("aws_region", "us-west-2")) + return str(self._settings["aws_region"]) @property def s3_bucket(self) -> str: """Get the S3 bucket name.""" - return str(self._settings.get("s3_bucket", "")) + return str(self._settings["s3_bucket"]) @property def default_upload_folder(self) -> str: """Get the default upload folder path.""" - return str(self._settings.get("default_upload_folder", "")) + return str(self._settings["default_upload_folder"]) @property def display_name(self) -> str: """Get the display name for the application.""" - return str(self._settings.get("display_name", "MODAQ Uploader")) + return str(self._settings["display_name"]) + + @property + def log_directory(self) -> Path: + """Get the log directory path (resolved to absolute path relative to BASE_DIR).""" + log_dir = str(self._settings["log_directory"]) + path = Path(log_dir) + if not path.is_absolute(): + path = BASE_DIR / path + return path def get_settings() -> Settings: @@ -229,35 +243,11 @@ def update_application(self) -> dict[str, Any]: "modaq_toolkit": {"success": False, "output": ""}, } - try: - # Git pull - git_result = subprocess.run( - ["git", "pull"], - cwd=self.base_dir, - capture_output=True, - text=True, - check=True, - ) - results["git_pull"] = { - "success": True, - "output": git_result.stdout + git_result.stderr, - } - - # Pip install requirements - pip_result = subprocess.run( - [sys.executable, "-m", "pip", "install", "-r", "requirements.txt"], - cwd=self.base_dir, - capture_output=True, - text=True, - check=True, - ) - results["pip_install"] = { - "success": True, - "output": pip_result.stdout + pip_result.stderr, - } - - # Update modaq_toolkit specifically (force reinstall to get latest) - modaq_result = subprocess.run( + steps: list[tuple[str, list[str]]] = [ + ("git_pull", ["git", "pull"]), + ("pip_install", [sys.executable, "-m", "pip", "install", "-r", "requirements.txt"]), + ( + "modaq_toolkit", [ sys.executable, "-m", @@ -267,34 +257,28 @@ def update_application(self) -> dict[str, Any]: "--force-reinstall", "git+https://github.com/MODAQ2/MODAQ_toolkit.git", ], - cwd=self.base_dir, - capture_output=True, - text=True, - check=True, - ) - results["modaq_toolkit"] = { - "success": True, - "output": modaq_result.stdout + modaq_result.stderr, - } - - except subprocess.CalledProcessError as e: - # Record which step failed - cmd_name = " ".join(e.cmd) if isinstance(e.cmd, list) else str(e.cmd) - if "git" in cmd_name: - results["git_pull"] = { - "success": False, - "output": e.stdout + e.stderr if e.stdout else str(e), - } - elif "modaq" in cmd_name.lower() or "MODAQ" in cmd_name: - results["modaq_toolkit"] = { - "success": False, - "output": e.stdout + e.stderr if e.stdout else str(e), + ), + ] + + for step_name, cmd in steps: + try: + result = subprocess.run( + cmd, + cwd=self.base_dir, + capture_output=True, + text=True, + check=True, + ) + results[step_name] = { + "success": True, + "output": result.stdout + result.stderr, } - else: - results["pip_install"] = { + except subprocess.CalledProcessError as e: + results[step_name] = { "success": False, "output": e.stdout + e.stderr if e.stdout else str(e), } + break return results diff --git a/app/routes/files.py b/app/routes/files.py index a102c74..3b122e0 100644 --- a/app/routes/files.py +++ b/app/routes/files.py @@ -2,7 +2,7 @@ from pathlib import Path -from flask import Blueprint, Response, jsonify, request +from flask import Blueprint, Response, g, jsonify, request from app.config import get_settings from app.services import s3_service @@ -10,6 +10,16 @@ files_bp = Blueprint("files", __name__) +@files_bp.before_request +def _require_bucket() -> tuple[Response, int] | None: + """Ensure S3 bucket is configured before handling any request.""" + settings = get_settings() + if not settings.s3_bucket: + return jsonify({"error": "S3 bucket not configured"}), 400 + g.settings = settings + return None + + @files_bp.route("/list", methods=["GET"]) def list_files() -> tuple[Response, int]: """List files and folders in S3 bucket. @@ -21,23 +31,18 @@ def list_files() -> tuple[Response, int]: Returns: JSON response with folders and files """ - settings = get_settings() - - if not settings.s3_bucket: - return jsonify({"error": "S3 bucket not configured"}), 400 - prefix = request.args.get("prefix", "") delimiter = request.args.get("delimiter", "/") try: client = s3_service.create_s3_client( - settings.aws_profile, - settings.aws_region, + g.settings.aws_profile, + g.settings.aws_region, ) result = s3_service.list_bucket_objects( client, - settings.s3_bucket, + g.settings.s3_bucket, prefix=prefix, delimiter=delimiter, ) @@ -71,24 +76,19 @@ def get_file_info() -> tuple[Response, int]: Returns: JSON response with object metadata """ - settings = get_settings() - - if not settings.s3_bucket: - return jsonify({"error": "S3 bucket not configured"}), 400 - key = request.args.get("key", "") if not key: return jsonify({"error": "Object key required"}), 400 try: client = s3_service.create_s3_client( - settings.aws_profile, - settings.aws_region, + g.settings.aws_profile, + g.settings.aws_region, ) result = s3_service.get_object_metadata( client, - settings.s3_bucket, + g.settings.s3_bucket, key, ) @@ -112,11 +112,6 @@ def search_files() -> tuple[Response, int]: Returns: JSON response with matching files """ - settings = get_settings() - - if not settings.s3_bucket: - return jsonify({"error": "S3 bucket not configured"}), 400 - query = request.args.get("query", "").lower() prefix = request.args.get("prefix", "") @@ -125,14 +120,14 @@ def search_files() -> tuple[Response, int]: try: client = s3_service.create_s3_client( - settings.aws_profile, - settings.aws_region, + g.settings.aws_profile, + g.settings.aws_region, ) # List all objects with prefix (no delimiter to get all files) result = s3_service.list_bucket_objects( client, - settings.s3_bucket, + g.settings.s3_bucket, prefix=prefix, delimiter="", # No delimiter to get all nested files max_keys=10000, @@ -196,7 +191,7 @@ def browse_local() -> tuple[Response, int]: # Build response folders: list[dict[str, str | int]] = [] - files: list[dict[str, str | int]] = [] + files: list[dict[str, str | int | float]] = [] mcap_count = 0 try: @@ -223,11 +218,13 @@ def browse_local() -> tuple[Response, int]: elif entry.is_file(): if entry.suffix == ".mcap": mcap_count += 1 + file_stat = entry.stat() files.append( { "name": entry.name, "path": str(entry), - "size": entry.stat().st_size, + "size": file_stat.st_size, + "mtime": file_stat.st_mtime, } ) except PermissionError: diff --git a/app/routes/logs.py b/app/routes/logs.py new file mode 100644 index 0000000..6017f08 --- /dev/null +++ b/app/routes/logs.py @@ -0,0 +1,173 @@ +"""Logs API routes for modaq_upload""" + +import csv +from pathlib import Path + +from flask import Blueprint, Response, jsonify, request, send_file + +from app.config import get_settings +from app.services import s3_service +from app.services.log_service import get_log_service + +logs_bp = Blueprint("logs", __name__) + + +@logs_bp.route("/entries", methods=["GET"]) +def get_log_entries() -> tuple[Response, int]: + """Query log entries with filtering and pagination. + + Query params: + date: Filter by date (YYYY-MM-DD) + level: Filter by level (INFO/WARNING/ERROR) + category: Filter by category (upload/analysis/settings/app/sync) + search: Full-text search in message and event + offset: Pagination offset (default 0) + limit: Pagination limit (default 100) + + Returns: + JSON with entries, total, offset, limit + """ + log = get_log_service() + + date = request.args.get("date") + level = request.args.get("level") + category = request.args.get("category") + search = request.args.get("search") + offset = request.args.get("offset", "0") + limit = request.args.get("limit", "100") + + try: + offset_int = max(0, int(offset)) + limit_int = max(1, min(1000, int(limit))) + except ValueError: + offset_int = 0 + limit_int = 100 + + result = log.read_log_entries( + date=date, + level=level, + category=category, + search=search, + offset=offset_int, + limit=limit_int, + ) + + return jsonify(result), 200 + + +@logs_bp.route("/files", methods=["GET"]) +def get_log_files() -> tuple[Response, int]: + """List all log files with metadata. + + Returns: + JSON with list of log files (date, filename, size) + """ + log = get_log_service() + files = log.list_log_files() + return jsonify({"files": files}), 200 + + +@logs_bp.route("/stats", methods=["GET"]) +def get_log_stats() -> tuple[Response, int]: + """Get aggregate log statistics. + + Returns: + JSON with counts by level/category, date range, totals + """ + log = get_log_service() + stats = log.get_log_stats() + return jsonify(stats), 200 + + +@logs_bp.route("/sync", methods=["POST"]) +def sync_logs() -> tuple[Response, int]: + """Trigger S3 sync of log files. + + Returns: + JSON with sync results (synced, skipped, errors) + """ + settings = get_settings() + + if not settings.s3_bucket: + return jsonify({"success": False, "error": "S3 bucket not configured"}), 400 + + log = get_log_service() + log.info("sync", "log_sync_started", "Starting log sync to S3") + + try: + client = s3_service.create_s3_client(settings.aws_profile, settings.aws_region) + result = log.sync_logs_to_s3(client, settings.s3_bucket) + return jsonify(result), 200 + except Exception as e: + log.error("sync", "log_sync_failed", f"Log sync failed: {e}", {"error": str(e)}) + return jsonify({"success": False, "error": str(e)}), 200 + + +def _resolve_csv_path(relative_path: str) -> Path | None: + """Resolve a relative CSV path safely within the log directory. + + Returns the absolute path if valid, or None if the path is invalid or + escapes the log directory. + """ + settings = get_settings() + log_dir = settings.log_directory + if not log_dir.is_absolute(): + from app.config import BASE_DIR + + log_dir = BASE_DIR / log_dir + + resolved = (log_dir / relative_path).resolve() + try: + resolved.relative_to(log_dir.resolve()) + except ValueError: + return None + if not resolved.is_file() or resolved.suffix != ".csv": + return None + return resolved + + +@logs_bp.route("/csv-download", methods=["GET"]) +def csv_download() -> tuple[Response, int] | Response: + """Serve a CSV file for browser download. + + Query params: + path: Relative path within the log directory + (e.g. csv/year=2026/month=02/day=08/upload-summary-143022-abcd1234.csv) + """ + relative_path = request.args.get("path", "") + if not relative_path: + return jsonify({"error": "Missing path parameter"}), 400 + + resolved = _resolve_csv_path(relative_path) + if resolved is None: + return jsonify({"error": "Invalid path"}), 400 + + return send_file(resolved, as_attachment=True, download_name=resolved.name) + + +@logs_bp.route("/csv-preview", methods=["GET"]) +def csv_preview() -> tuple[Response, int]: + """Parse a CSV and return JSON for in-page viewing. + + Query params: + path: Relative path within the log directory + + Returns: + JSON with columns list and rows list of dicts + """ + relative_path = request.args.get("path", "") + if not relative_path: + return jsonify({"error": "Missing path parameter"}), 400 + + resolved = _resolve_csv_path(relative_path) + if resolved is None: + return jsonify({"error": "Invalid path"}), 400 + + try: + with open(resolved, encoding="utf-8", newline="") as f: + reader = csv.DictReader(f) + columns = reader.fieldnames or [] + rows = list(reader) + return jsonify({"columns": columns, "rows": rows}), 200 + except Exception as e: + return jsonify({"error": f"Failed to read CSV: {e}"}), 500 diff --git a/app/routes/main.py b/app/routes/main.py index 291039a..700e234 100644 --- a/app/routes/main.py +++ b/app/routes/main.py @@ -21,3 +21,9 @@ def files() -> str: def settings() -> str: """Render the settings page.""" return render_template("settings.html") + + +@main_bp.route("/logs") +def logs() -> str: + """Render the logs viewer page.""" + return render_template("logs.html") diff --git a/app/routes/settings.py b/app/routes/settings.py index 7b06f88..bc1e1a6 100644 --- a/app/routes/settings.py +++ b/app/routes/settings.py @@ -5,6 +5,7 @@ from app.config import get_package_version, get_settings, get_updater from app.services import s3_service from app.services.cache_service import get_cache_service +from app.services.log_service import get_log_service settings_bp = Blueprint("settings", __name__) @@ -40,7 +41,10 @@ def update_settings() -> tuple[Response, int]: settings = get_settings() # Validate settings - allowed_keys = {"aws_profile", "aws_region", "s3_bucket", "default_upload_folder"} + allowed_keys = { + "aws_profile", "aws_region", "s3_bucket", "default_upload_folder", "display_name", + "log_directory", + } filtered_data = {k: v for k, v in data.items() if k in allowed_keys} if not filtered_data: @@ -48,6 +52,14 @@ def update_settings() -> tuple[Response, int]: settings.update(filtered_data) + log = get_log_service() + log.info( + "settings", + "settings_updated", + f"Updated settings: {', '.join(filtered_data.keys())}", + {"changed_keys": list(filtered_data.keys())}, + ) + return jsonify(settings.all()), 200 @@ -90,11 +102,18 @@ def validate_connection() -> tuple[Response, int]: if not bucket: return jsonify({"error": "S3 bucket not specified"}), 400 + log = get_log_service() try: client = s3_service.create_s3_client(profile, region) result = s3_service.validate_bucket_access(client, bucket) if result["success"]: + log.info( + "settings", + "connection_test", + f"Connection test succeeded for bucket '{bucket}'", + {"bucket": bucket, "profile": profile, "region": region, "success": True}, + ) return jsonify( { "success": True, @@ -102,6 +121,18 @@ def validate_connection() -> tuple[Response, int]: } ), 200 else: + log.warning( + "settings", + "connection_test", + f"Connection test failed for bucket '{bucket}': {result['error']}", + { + "bucket": bucket, + "profile": profile, + "region": region, + "success": False, + "error": result["error"], + }, + ) return jsonify( { "success": False, @@ -110,6 +141,12 @@ def validate_connection() -> tuple[Response, int]: ), 200 except Exception as e: + log.error( + "settings", + "connection_test", + f"Connection test error for bucket '{bucket}': {e}", + {"bucket": bucket, "profile": profile, "region": region, "error": str(e)}, + ) return jsonify( { "success": False, diff --git a/app/routes/upload.py b/app/routes/upload.py index f8d3cdb..0010c50 100644 --- a/app/routes/upload.py +++ b/app/routes/upload.py @@ -1,10 +1,11 @@ """Upload API routes for modaq_upload""" import json -import os import tempfile import threading -from collections.abc import Generator +import time +from collections import deque +from collections.abc import Callable, Generator from pathlib import Path from typing import Any @@ -21,7 +22,7 @@ upload_bp = Blueprint("upload", __name__) # Store for SSE clients per job -_sse_queues: dict[str, list[Any]] = {} +_sse_queues: dict[str, list[deque[dict[str, Any]]]] = {} _sse_lock = threading.Lock() @@ -33,6 +34,27 @@ def send_sse_event(job_id: str, data: dict[str, Any]) -> None: q.append(data) +def _make_analysis_callback( + job_id: str, +) -> Callable[[UploadJob, FileUploadState], None]: + """Create an analysis progress callback that sends SSE events.""" + + def callback(job: UploadJob, file_state: FileUploadState) -> None: + send_sse_event( + job_id, + { + "type": "analysis_progress", + "job_id": job.job_id, + "job_status": job.status.value, + "file": file_state.to_dict(), + "total_files": len(job.files), + "analysis_complete": job.status.value in ("ready", "failed"), + }, + ) + + return callback + + @upload_bp.route("/analyze", methods=["POST"]) def analyze_files() -> tuple[Response, int]: """Analyze uploaded files and prepare for upload. @@ -58,11 +80,10 @@ def analyze_files() -> tuple[Response, int]: for uploaded_file in uploaded_files: if uploaded_file.filename: # Filename may include subdirectory path from folder selection - temp_path = os.path.join(temp_dir, uploaded_file.filename) - # Create parent directories if they don't exist - os.makedirs(os.path.dirname(temp_path), exist_ok=True) + temp_path = Path(temp_dir) / uploaded_file.filename + temp_path.parent.mkdir(parents=True, exist_ok=True) uploaded_file.save(temp_path) - file_paths.append(temp_path) + file_paths.append(str(temp_path)) # Handle JSON with file paths (for folder selection / bulk mode) elif request.is_json: @@ -76,19 +97,7 @@ def analyze_files() -> tuple[Response, int]: # Create job with temp_dir tracked for cleanup job = manager.create_job(file_paths, temp_dir=temp_dir) - - def analysis_progress_callback(job: UploadJob, file_state: FileUploadState) -> None: - """Send analysis progress updates via SSE.""" - send_sse_event( - job.job_id, - { - "type": "analysis_progress", - "job_id": job.job_id, - "job_status": job.status.value, - "file": file_state.to_dict(), - "analysis_complete": job.status.value in ("ready", "failed"), - }, - ) + analysis_progress_callback = _make_analysis_callback(job.job_id) # Start analysis in background thread def run_analysis() -> None: @@ -186,7 +195,7 @@ def get_progress(job_id: str) -> Response: def generate() -> Generator[str, None, None]: # Create a queue for this client - queue: list[Any] = [] + queue: deque[dict[str, Any]] = deque() with _sse_lock: if job_id not in _sse_queues: _sse_queues[job_id] = [] @@ -202,7 +211,7 @@ def generate() -> Generator[str, None, None]: while True: # Check for updates while queue: - data = queue.pop(0) + data = queue.popleft() yield f"data: {json.dumps(data)}\n\n" # Check if job is complete @@ -210,8 +219,6 @@ def generate() -> Generator[str, None, None]: return # Small delay to prevent busy waiting - import time - time.sleep(0.1) # Check if job still exists @@ -268,19 +275,11 @@ def get_active_job() -> tuple[Response, int]: manager = get_upload_manager() # Find the most recent job that is still active (not completed/failed/cancelled) - active_statuses = { - UploadStatus.PENDING, - UploadStatus.ANALYZING, - UploadStatus.READY, - UploadStatus.UPLOADING, - } - + active_jobs = manager.get_active_jobs() active_job: UploadJob | None = None - for job in manager.jobs.values(): - if job.status in active_statuses: - # Return the most recently created active job - if active_job is None or job.job_id > active_job.job_id: - active_job = job + for job in active_jobs: + if active_job is None or job.job_id > active_job.job_id: + active_job = job if active_job: return jsonify( @@ -345,14 +344,20 @@ def scan_folder() -> tuple[Response, int]: # Recursively find all .mcap files files: list[dict[str, Any]] = [] + total_size = 0 try: for mcap_path in folder_path.rglob("*.mcap"): if mcap_path.is_file(): + stat = mcap_path.stat() + file_size = stat.st_size + total_size += file_size files.append( { "path": str(mcap_path.absolute()), "filename": mcap_path.name, - "size": mcap_path.stat().st_size, + "size": file_size, + "mtime": stat.st_mtime, + "relative_path": str(mcap_path.relative_to(folder_path)), } ) except PermissionError as e: @@ -364,6 +369,7 @@ def scan_folder() -> tuple[Response, int]: "folder_path": str(folder_path.absolute()), "files": files, "total_count": len(files), + "total_size": total_size, } ), 200 @@ -393,6 +399,7 @@ def bulk_analyze() -> tuple[Response, int]: file_paths: list[str] = data["file_paths"] auto_upload: bool = data.get("auto_upload", False) pre_filter_only: bool = data.get("pre_filter_only", False) + skip_duplicates: bool = data.get("skip_duplicates", True) if not file_paths: return jsonify({"error": "No files provided"}), 400 @@ -400,8 +407,10 @@ def bulk_analyze() -> tuple[Response, int]: settings = get_settings() manager = get_upload_manager() - # Run pre-filtering - files_to_analyze, pre_filter_stats = manager.pre_filter_files(file_paths, settings.s3_bucket) + # Run pre-filtering (with S3 fallback for cache misses) + files_to_analyze, pre_filter_stats = manager.pre_filter_files( + file_paths, settings.s3_bucket, settings.aws_profile, settings.aws_region + ) if pre_filter_only: return jsonify( @@ -412,22 +421,13 @@ def bulk_analyze() -> tuple[Response, int]: } ), 200 + # When force-reuploading, analyze ALL files (not just non-duplicates) + job_files = file_paths if not skip_duplicates else files_to_analyze + # Create job with files that need analysis (no temp_dir - direct file access) - job = manager.create_job(files_to_analyze, auto_upload=auto_upload) + job = manager.create_job(job_files, auto_upload=auto_upload) job.pre_filter_stats = pre_filter_stats - - def analysis_progress_callback(job: UploadJob, file_state: FileUploadState) -> None: - """Send analysis progress updates via SSE.""" - send_sse_event( - job.job_id, - { - "type": "analysis_progress", - "job_id": job.job_id, - "job_status": job.status.value, - "file": file_state.to_dict(), - "analysis_complete": job.status.value in ("ready", "failed"), - }, - ) + analysis_progress_callback = _make_analysis_callback(job.job_id) def upload_progress_callback(job: UploadJob) -> None: """Send upload progress updates via SSE.""" @@ -455,8 +455,8 @@ def run_bulk_analysis() -> None: }, ) - # Auto-upload if enabled and there are valid files - if final_job.auto_upload and final_job.has_valid_uploadable_files: + # Auto-upload if enabled and analysis succeeded + if final_job.auto_upload and final_job.status == UploadStatus.READY: send_sse_event( job.job_id, { @@ -469,9 +469,12 @@ def run_bulk_analysis() -> None: settings.aws_profile, settings.aws_region, settings.s3_bucket, - skip_duplicates=True, + skip_duplicates=skip_duplicates, progress_callback=upload_progress_callback, ) + elif final_job.auto_upload: + # All files failed analysis — send terminal status so frontend doesn't hang + send_sse_event(job.job_id, final_job.to_dict()) thread = threading.Thread(target=run_bulk_analysis, daemon=True) thread.start() diff --git a/app/services/cache_service.py b/app/services/cache_service.py index a18add1..e925709 100644 --- a/app/services/cache_service.py +++ b/app/services/cache_service.py @@ -10,31 +10,16 @@ class CacheService: - """Singleton cache service with thread-safe SQLite access for S3 file tracking.""" + """Cache service with thread-safe SQLite access for S3 file tracking.""" CACHE_FILE = "modaq_upload_cache.db" CACHE_TTL_SECONDS = 3600 # 1 hour default TTL - _instance: "CacheService | None" = None - _lock = threading.Lock() - - def __new__(cls) -> "CacheService": - """Ensure singleton instance.""" - with cls._lock: - if cls._instance is None: - cls._instance = super().__new__(cls) - cls._instance._initialized = False - return cls._instance - def __init__(self) -> None: """Initialize the cache database.""" - if getattr(self, "_initialized", False): - return - self._db_path = Path(self.CACHE_FILE) self._local = threading.local() self._init_db() - self._initialized = True def _get_connection(self) -> sqlite3.Connection: """Get thread-local database connection.""" @@ -75,6 +60,10 @@ def _init_db(self) -> None: CREATE INDEX IF NOT EXISTS idx_bucket ON s3_files(bucket) """) + cursor.execute(""" + CREATE INDEX IF NOT EXISTS idx_bucket_filename ON s3_files(bucket, filename, file_size) + """) + # Metadata table for tracking sync status per bucket cursor.execute(""" CREATE TABLE IF NOT EXISTS cache_metadata ( @@ -132,6 +121,42 @@ def check_exists_cached( return bool(row["file_exists"]) + def check_exists_by_filename( + self, + bucket: str, + filename: str, + file_size: int, + ) -> bool | None: + """Check if a file exists in the cache by filename and size. + + Fallback lookup when the S3 path isn't known (e.g. pre-filter can't + generate the same path as the actual MCAP analysis). No TTL is applied + because if we uploaded a file, that fact doesn't expire. + + Args: + bucket: S3 bucket name + filename: Original filename + file_size: File size in bytes + + Returns: + True if any cache entry for this filename+size says it exists, + None if no matching entry found + """ + conn = self._get_connection() + cursor = conn.cursor() + + cursor.execute( + """ + SELECT 1 FROM s3_files + WHERE bucket = ? AND filename = ? AND file_size = ? AND file_exists = 1 + LIMIT 1 + """, + (bucket, filename, file_size), + ) + + row = cursor.fetchone() + return True if row else None + def update_cache( self, bucket: str, diff --git a/app/services/log_service.py b/app/services/log_service.py new file mode 100644 index 0000000..2d06715 --- /dev/null +++ b/app/services/log_service.py @@ -0,0 +1,530 @@ +"""JSONL logging service for application events. + +Writes one JSON object per line to hive-partitioned daily .jsonl files. +DuckDB-compatible: SELECT * FROM read_json_auto('logs/json/**/events.jsonl', hive_partitioning=true) +""" + +import csv +import io +import json +import re +import threading +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from app.config import get_settings + + +class LogService: + """JSONL log service with thread-safe file writes.""" + + def __init__(self) -> None: + """Initialize the log service.""" + self._write_lock = threading.Lock() + + def _get_log_dir(self) -> Path: + """Get the configured log directory, creating it if needed.""" + settings = get_settings() + log_dir = settings.log_directory + log_dir.mkdir(parents=True, exist_ok=True) + return log_dir + + def _get_hive_dir(self, subdir: str, dt: datetime) -> Path: + """Build a hive-partitioned directory path and create it. + + Args: + subdir: Top-level subdirectory ('json' or 'csv') + dt: Datetime to partition by + + Returns: + Path like logs/json/year=2026/month=02/day=08/ + """ + log_dir = self._get_log_dir() + hive_dir = ( + log_dir + / subdir + / f"year={dt.year:04d}" + / f"month={dt.month:02d}" + / f"day={dt.day:02d}" + ) + hive_dir.mkdir(parents=True, exist_ok=True) + return hive_dir + + @staticmethod + def _extract_date_from_hive_path(path: Path) -> str | None: + """Extract YYYY-MM-DD date string from a hive-partitioned path. + + Looks for year=YYYY/month=MM/day=DD components in the path. + + Returns: + Date string like '2026-02-08', or None if not found + """ + path_str = str(path) + match = re.search(r"year=(\d{4})/month=(\d{2})/day=(\d{2})", path_str) + if match: + return f"{match.group(1)}-{match.group(2)}-{match.group(3)}" + return None + + def _get_current_log_file(self) -> Path: + """Get the path to today's events log file (hive-partitioned).""" + now = datetime.now(UTC) + hive_dir = self._get_hive_dir("json", now) + return hive_dir / "events.jsonl" + + def log( + self, + level: str, + category: str, + event: str, + message: str, + metadata: dict[str, Any] | None = None, + ) -> None: + """Append a log entry to the current day's JSONL file. + + Args: + level: Log level (INFO, WARNING, ERROR) + category: Event category (upload, analysis, settings, app, sync) + event: Machine-readable event name (snake_case) + message: Human-readable message + metadata: Optional additional data + """ + entry: dict[str, Any] = { + "timestamp": datetime.now(UTC).isoformat(), + "level": level.upper(), + "category": category, + "event": event, + "message": message, + } + if metadata: + entry["metadata"] = metadata + + line = json.dumps(entry, default=str) + + with self._write_lock: + log_file = self._get_current_log_file() + with open(log_file, "a", encoding="utf-8") as f: + f.write(line + "\n") + + def info( + self, + category: str, + event: str, + message: str, + metadata: dict[str, Any] | None = None, + ) -> None: + """Log an INFO-level event.""" + self.log("INFO", category, event, message, metadata) + + def warning( + self, + category: str, + event: str, + message: str, + metadata: dict[str, Any] | None = None, + ) -> None: + """Log a WARNING-level event.""" + self.log("WARNING", category, event, message, metadata) + + def error( + self, + category: str, + event: str, + message: str, + metadata: dict[str, Any] | None = None, + ) -> None: + """Log an ERROR-level event.""" + self.log("ERROR", category, event, message, metadata) + + def save_job_jsonl( + self, + job_id: str, + job_dict: dict[str, Any], + completed_at: datetime, + ) -> Path: + """Write a per-job JSONL summary file. + + Args: + job_id: The upload job ID + job_dict: Full job completion summary dict + completed_at: When the job completed + + Returns: + Path to the written file + """ + hive_dir = self._get_hive_dir("json", completed_at) + out_path = hive_dir / f"{job_id}.jsonl" + line = json.dumps(job_dict, default=str) + with self._write_lock: + with open(out_path, "w", encoding="utf-8") as f: + f.write(line + "\n") + return out_path + + def save_job_csv( + self, + job_id: str, + job: Any, + completed_at: datetime, + ) -> Path: + """Write a per-job CSV upload summary. + + Args: + job_id: The upload job ID + job: UploadJob instance with file states + completed_at: When the job completed + + Returns: + Path to the written CSV file + """ + from app.services.utils import format_file_size + + hive_dir = self._get_hive_dir("csv", completed_at) + time_str = completed_at.strftime("%H%M%S") + short_id = job_id[:8] + out_path = hive_dir / f"upload-summary-{time_str}-{short_id}.csv" + + columns = [ + "job_id", + "filename", + "file_size_bytes", + "file_size_formatted", + "s3_path", + "status", + "data_start_time", + "upload_started_at", + "upload_completed_at", + "upload_duration_seconds", + "upload_speed_mbps", + "is_duplicate", + "is_valid", + "error_message", + ] + + buf = io.StringIO() + writer = csv.writer(buf) + writer.writerow(columns) + + for f in job.files: + duration = f.upload_duration_seconds + speed = ( + round(f.file_size / duration / 1024 / 1024 * 8, 2) + if duration and duration > 0 + else None + ) + writer.writerow([ + job_id, + f.filename, + f.file_size, + format_file_size(f.file_size), + f.s3_path, + f.status.value, + f.start_time.isoformat() if f.start_time else "", + f.upload_started_at.isoformat() if f.upload_started_at else "", + f.upload_completed_at.isoformat() if f.upload_completed_at else "", + f.upload_duration_seconds, + speed, + f.is_duplicate, + f.is_valid, + f.error_message, + ]) + + with self._write_lock: + with open(out_path, "w", encoding="utf-8", newline="") as fh: + fh.write(buf.getvalue()) + + return out_path + + def list_log_files(self) -> list[dict[str, Any]]: + """List all log files (JSONL + CSV) with metadata. + + Returns: + List of dicts with date, filename, path, size_bytes, relative_path, type + """ + log_dir = self._get_log_dir() + json_dir = log_dir / "json" + csv_dir = log_dir / "csv" + + result: list[dict[str, Any]] = [] + + # Collect JSONL files under json/ + if json_dir.exists(): + for f in sorted(json_dir.rglob("*.jsonl"), reverse=True): + date_str = self._extract_date_from_hive_path(f) + rel_path = f.relative_to(log_dir) + result.append({ + "date": date_str, + "filename": f.name, + "path": str(f), + "relative_path": str(rel_path), + "size_bytes": f.stat().st_size, + "type": "jsonl", + }) + + # Collect CSV files under csv/ + if csv_dir.exists(): + for f in sorted(csv_dir.rglob("*.csv"), reverse=True): + date_str = self._extract_date_from_hive_path(f) + rel_path = f.relative_to(log_dir) + result.append({ + "date": date_str, + "filename": f.name, + "path": str(f), + "relative_path": str(rel_path), + "size_bytes": f.stat().st_size, + "type": "csv", + }) + + return result + + def read_log_entries( + self, + date: str | None = None, + level: str | None = None, + category: str | None = None, + search: str | None = None, + offset: int = 0, + limit: int = 100, + ) -> dict[str, Any]: + """Read and filter log entries with pagination. + + Args: + date: Filter by date (YYYY-MM-DD). None = all dates. + level: Filter by level (INFO/WARNING/ERROR) + category: Filter by category + search: Full-text search in message and event fields + offset: Number of entries to skip + limit: Maximum entries to return + + Returns: + Dict with entries, total count, offset, limit + """ + log_dir = self._get_log_dir() + json_dir = log_dir / "json" + + # Determine which event files to read + if date: + # Parse date and construct hive path directly + try: + dt = datetime.strptime(date, "%Y-%m-%d") + except ValueError: + return {"entries": [], "total": 0, "offset": offset, "limit": limit} + hive_path = ( + json_dir + / f"year={dt.year:04d}" + / f"month={dt.month:02d}" + / f"day={dt.day:02d}" + / "events.jsonl" + ) + files = [hive_path] if hive_path.exists() else [] + else: + if json_dir.exists(): + files = sorted(json_dir.rglob("events.jsonl"), reverse=True) + else: + files = [] + + # Read and filter entries + all_entries: list[dict[str, Any]] = [] + for log_file in files: + try: + with open(log_file, encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + + # Apply filters + if level and entry.get("level", "").upper() != level.upper(): + continue + if category and entry.get("category") != category: + continue + if search: + search_lower = search.lower() + msg = entry.get("message", "").lower() + evt = entry.get("event", "").lower() + if search_lower not in msg and search_lower not in evt: + continue + + all_entries.append(entry) + except OSError: + continue + + # Sort by timestamp descending (newest first) + all_entries.sort(key=lambda e: e.get("timestamp", ""), reverse=True) + + total = len(all_entries) + paginated = all_entries[offset : offset + limit] + + return { + "entries": paginated, + "total": total, + "offset": offset, + "limit": limit, + } + + def get_log_stats(self) -> dict[str, Any]: + """Get aggregate statistics across all log files. + + Returns: + Dict with counts by level/category, date range, totals + """ + log_dir = self._get_log_dir() + json_dir = log_dir / "json" + csv_dir = log_dir / "csv" + + level_counts: dict[str, int] = {} + category_counts: dict[str, int] = {} + total_entries = 0 + total_size = 0 + today_count = 0 + today_str = datetime.now(UTC).strftime("%Y-%m-%d") + dates: list[str] = [] + + event_files = sorted(json_dir.rglob("events.jsonl")) if json_dir.exists() else [] + + for log_file in event_files: + total_size += log_file.stat().st_size + date_str = self._extract_date_from_hive_path(log_file) + if date_str and date_str not in dates: + dates.append(date_str) + is_today = date_str == today_str + + try: + with open(log_file, encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + + total_entries += 1 + if is_today: + today_count += 1 + + lvl = entry.get("level", "UNKNOWN") + level_counts[lvl] = level_counts.get(lvl, 0) + 1 + + cat = entry.get("category", "unknown") + category_counts[cat] = category_counts.get(cat, 0) + 1 + except OSError: + continue + + dates.sort() + + # Count CSV files + csv_count = len(list(csv_dir.rglob("*.csv"))) if csv_dir.exists() else 0 + + return { + "total_entries": total_entries, + "today_entries": today_count, + "total_size_bytes": total_size, + "level_counts": level_counts, + "category_counts": category_counts, + "date_range": { + "earliest": dates[0] if dates else None, + "latest": dates[-1] if dates else None, + }, + "file_count": len(event_files), + "csv_count": csv_count, + } + + def sync_logs_to_s3( + self, + s3_client: Any, + bucket: str, + prefix: str = "logs/", + ) -> dict[str, Any]: + """Upload new/changed log files (JSONL + CSV) to S3. + + Tracks sync state in .sync_state.json to only upload changed files. + Uses relative paths as both the sync state key and S3 key suffix. + + Args: + s3_client: boto3 S3 client + bucket: S3 bucket name + prefix: S3 key prefix for log files + + Returns: + Dict with sync results + """ + log_dir = self._get_log_dir() + sync_state_file = log_dir / ".sync_state.json" + + # Load previous sync state + sync_state: dict[str, int] = {} + if sync_state_file.exists(): + try: + with open(sync_state_file, encoding="utf-8") as f: + sync_state = json.load(f) + except (json.JSONDecodeError, OSError): + sync_state = {} + + # Discover all JSONL under json/ and CSV under csv/ + files: list[Path] = [] + json_dir = log_dir / "json" + csv_dir = log_dir / "csv" + if json_dir.exists(): + files.extend(json_dir.rglob("*.jsonl")) + if csv_dir.exists(): + files.extend(csv_dir.rglob("*.csv")) + + synced = 0 + skipped = 0 + errors: list[str] = [] + + for log_file in files: + rel_path = str(log_file.relative_to(log_dir)) + current_size = log_file.stat().st_size + last_synced_size = sync_state.get(rel_path, 0) + + if current_size == last_synced_size: + skipped += 1 + continue + + s3_key = f"{prefix}{rel_path}" + try: + s3_client.upload_file(str(log_file), bucket, s3_key) + sync_state[rel_path] = current_size + synced += 1 + except Exception as e: + errors.append(f"{rel_path}: {e}") + + # Save updated sync state + try: + with open(sync_state_file, "w", encoding="utf-8") as f: + json.dump(sync_state, f, indent=2) + except OSError: + pass + + self.info( + "sync", + "log_sync_completed", + f"Synced {synced} log files to S3", + {"synced": synced, "skipped": skipped, "errors": len(errors)}, + ) + + return { + "success": len(errors) == 0, + "synced": synced, + "skipped": skipped, + "errors": errors, + "total_files": len(files), + } + + +# Module-level singleton accessor +_log_service: LogService | None = None + + +def get_log_service() -> LogService: + """Get the singleton LogService instance.""" + global _log_service + if _log_service is None: + _log_service = LogService() + return _log_service diff --git a/app/services/mcap_service.py b/app/services/mcap_service.py index 375180c..705d7c6 100644 --- a/app/services/mcap_service.py +++ b/app/services/mcap_service.py @@ -1,11 +1,23 @@ """MCAP service for parsing MCAP files and extracting metadata.""" +import logging import re from datetime import UTC, datetime from pathlib import Path import pandas as pd +from app.services.utils import format_file_size + +logger = logging.getLogger(__name__) + +__all__ = ["format_file_size"] + + +def to_naive_utc(dt: datetime) -> datetime: + """Strip timezone for comparison (normalize to naive UTC).""" + return dt.replace(tzinfo=None) if dt.tzinfo else dt + def _extract_timestamp_from_filename(filename: str) -> datetime | None: """Try to extract a timestamp from the filename. @@ -92,12 +104,12 @@ def _find_datetime_in_dataframes(dataframes: dict[str, pd.DataFrame]) -> datetim # Check if it's a valid timestamp (after 1980) if topic_time is not None: # Make timezone-naive for comparison - check_time = topic_time.replace(tzinfo=None) if topic_time.tzinfo else topic_time - if check_time > epoch_cutoff.replace(tzinfo=None): - if earliest_time is None or check_time < earliest_time.replace(tzinfo=None): + check_time = to_naive_utc(topic_time) + if check_time > to_naive_utc(epoch_cutoff): + if earliest_time is None or check_time < to_naive_utc(earliest_time): earliest_time = topic_time except Exception: - pass + logger.debug("Failed to extract datetime from index of topic", exc_info=True) # Then, check columns for datetime types for col in df.columns: @@ -116,13 +128,9 @@ def _find_datetime_in_dataframes(dataframes: dict[str, pd.DataFrame]) -> datetim else: topic_time = pd.Timestamp(first_val).to_pydatetime() - check_time = ( - topic_time.replace(tzinfo=None) if topic_time.tzinfo else topic_time - ) - if check_time > epoch_cutoff.replace(tzinfo=None): - if earliest_time is None or check_time < earliest_time.replace( - tzinfo=None - ): + check_time = to_naive_utc(topic_time) + if check_time > to_naive_utc(epoch_cutoff): + if earliest_time is None or check_time < to_naive_utc(earliest_time): earliest_time = topic_time # Check for timestamp-like column names @@ -139,22 +147,18 @@ def _find_datetime_in_dataframes(dataframes: dict[str, pd.DataFrame]) -> datetim elif first_val > 1e12: # Milliseconds topic_time = pd.Timestamp(first_val, unit="ms").to_pydatetime() else: # Seconds - topic_time = datetime.fromtimestamp(first_val) - - check_time = ( - topic_time.replace(tzinfo=None) - if topic_time.tzinfo - else topic_time - ) - if check_time > epoch_cutoff.replace(tzinfo=None): - if earliest_time is None or check_time < earliest_time.replace( - tzinfo=None + topic_time = datetime.fromtimestamp(first_val, tz=UTC) + + check_time = to_naive_utc(topic_time) + if check_time > to_naive_utc(epoch_cutoff): + if earliest_time is None or check_time < to_naive_utc( + earliest_time ): earliest_time = topic_time except Exception: - pass + logger.debug("Failed to parse timestamp column %s", col, exc_info=True) except Exception: - continue + logger.debug("Failed to inspect column %s", col, exc_info=True) return earliest_time @@ -183,7 +187,7 @@ def extract_start_time(file_path: Path | str) -> datetime: raise FileNotFoundError(f"MCAP file not found: {path}") earliest_time: datetime | None = None - epoch_cutoff = datetime(1980, 1, 1) + epoch_cutoff = datetime(1980, 1, 1, tzinfo=UTC) try: parser = MCAPParser(path) @@ -196,7 +200,7 @@ def extract_start_time(file_path: Path | str) -> datetime: ) earliest_time = _find_datetime_in_dataframes(dataframes) except Exception: - pass + logger.debug("Stage2 datetime extraction failed", exc_info=True) # If no valid time found, try without conversion if earliest_time is None: @@ -204,7 +208,7 @@ def extract_start_time(file_path: Path | str) -> datetime: dataframes = parser.get_dataframes(process_stage2=False) earliest_time = _find_datetime_in_dataframes(dataframes) except Exception: - pass + logger.debug("Raw dataframe extraction failed", exc_info=True) except ImportError as e: raise ImportError( @@ -212,12 +216,11 @@ def extract_start_time(file_path: Path | str) -> datetime: "Install with: pip install git+https://github.com/MODAQ2/MODAQ_toolkit.git" ) from e except Exception: - pass # Will fall back to filename parsing + logger.debug("MCAP parser initialization failed for %s", path, exc_info=True) # Validate the timestamp - must be after 1980 if earliest_time is not None: - check_time = earliest_time.replace(tzinfo=None) if earliest_time.tzinfo else earliest_time - if check_time < epoch_cutoff: + if to_naive_utc(earliest_time) < to_naive_utc(epoch_cutoff): earliest_time = None # Invalid timestamp, try filename # Fallback: try to extract from filename @@ -247,6 +250,7 @@ def generate_s3_path(start_time: datetime, filename: str) -> str: minute_bucket = (start_time.minute // 10) * 10 path = ( + f"data/" f"year={start_time.year:04d}/" f"month={start_time.month:02d}/" f"day={start_time.day:02d}/" @@ -288,18 +292,3 @@ def get_file_info(file_path: Path | str) -> dict[str, str | int | None]: return info -def format_file_size(size_bytes: int) -> str: - """Format a file size in bytes to a human-readable string. - - Args: - size_bytes: Size in bytes - - Returns: - Human-readable size string (e.g., "1.5 GB") - """ - size_float = float(size_bytes) - for unit in ["B", "KB", "MB", "GB", "TB"]: - if abs(size_float) < 1024.0: - return f"{size_float:.1f} {unit}" - size_float = size_float / 1024.0 - return f"{size_float:.1f} PB" diff --git a/app/services/s3_service.py b/app/services/s3_service.py index bcd461d..f50740e 100644 --- a/app/services/s3_service.py +++ b/app/services/s3_service.py @@ -255,6 +255,15 @@ def validate_bucket_access(client: S3Client, bucket: str) -> dict[str, Any]: } +def get_s3_client_from_settings() -> tuple[S3Client, str]: + """Create S3 client from current app settings. Returns (client, bucket).""" + from app.config import get_settings + + settings = get_settings() + client = create_s3_client(settings.aws_profile, settings.aws_region) + return client, settings.s3_bucket + + def get_object_metadata(client: S3Client, bucket: str, key: str) -> dict[str, Any]: """Get metadata for an S3 object. diff --git a/app/services/upload_manager.py b/app/services/upload_manager.py index e32c5b0..0f7ffea 100644 --- a/app/services/upload_manager.py +++ b/app/services/upload_manager.py @@ -1,21 +1,26 @@ """Upload manager for orchestrating file uploads to S3.""" +import logging import shutil import threading import uuid from collections.abc import Callable from concurrent.futures import ThreadPoolExecutor, as_completed from dataclasses import dataclass, field -from datetime import datetime +from datetime import UTC, datetime from enum import Enum from pathlib import Path from typing import Any from app.services import mcap_service, s3_service from app.services.cache_service import get_cache_service +from app.services.log_service import get_log_service +from app.services.utils import format_file_size + +logger = logging.getLogger(__name__) # Timestamps before this date are considered invalid (1970/epoch issues) -EPOCH_CUTOFF = datetime(1980, 1, 1) +EPOCH_CUTOFF = datetime(1980, 1, 1, tzinfo=UTC) class UploadStatus(Enum): @@ -62,7 +67,7 @@ def to_dict(self) -> dict[str, Any]: "filename": self.filename, "local_path": self.local_path, "file_size": self.file_size, - "file_size_formatted": mcap_service.format_file_size(self.file_size), + "file_size_formatted": format_file_size(self.file_size), "status": self.status.value, "s3_path": self.s3_path, "start_time": self.start_time.isoformat() if self.start_time else None, @@ -95,14 +100,14 @@ class UploadJob: job_id: str files: list[FileUploadState] = field(default_factory=list) status: UploadStatus = UploadStatus.PENDING - created_at: datetime = field(default_factory=datetime.now) + created_at: datetime = field(default_factory=lambda: datetime.now(UTC)) started_at: datetime | None = None completed_at: datetime | None = None cancelled: bool = False auto_upload: bool = False # Auto-start upload when analysis completes temp_dir: str | None = None # Temp directory for cleanup pre_filter_stats: dict[str, Any] = field(default_factory=dict) # Pre-filter statistics - _lock: threading.Lock = field(default_factory=threading.Lock, repr=False) + lock: threading.Lock = field(default_factory=threading.Lock, repr=False) @property def total_bytes(self) -> int: @@ -139,7 +144,7 @@ def eta_seconds(self) -> int | None: if not self.started_at or self.uploaded_bytes == 0: return None - elapsed = (datetime.now() - self.started_at).total_seconds() + elapsed = (datetime.now(UTC) - self.started_at).total_seconds() if elapsed <= 0: return None @@ -178,6 +183,13 @@ def average_upload_speed_mbps(self) -> float | None: return round(self.successfully_uploaded_bytes / duration / 1024 / 1024 * 8, 2) return None + def resolve_analysis_status(self) -> None: + """Set job status based on file analysis results.""" + if any(f.status == UploadStatus.READY for f in self.files): + self.status = UploadStatus.READY + else: + self.status = UploadStatus.FAILED + def to_dict(self) -> dict[str, Any]: """Convert to dictionary for JSON serialization.""" duration = self.total_upload_duration_seconds @@ -191,11 +203,11 @@ def to_dict(self) -> dict[str, Any]: "files_skipped": sum(1 for f in self.files if f.status == UploadStatus.SKIPPED), "files_uploaded": sum(1 for f in self.files if f.status == UploadStatus.COMPLETED), "total_bytes": self.total_bytes, - "total_bytes_formatted": mcap_service.format_file_size(self.total_bytes), + "total_bytes_formatted": format_file_size(self.total_bytes), "uploaded_bytes": self.uploaded_bytes, - "uploaded_bytes_formatted": mcap_service.format_file_size(self.uploaded_bytes), + "uploaded_bytes_formatted": format_file_size(self.uploaded_bytes), "successfully_uploaded_bytes": self.successfully_uploaded_bytes, - "successfully_uploaded_bytes_formatted": mcap_service.format_file_size( + "successfully_uploaded_bytes_formatted": format_file_size( self.successfully_uploaded_bytes ), "progress_percent": self.progress_percent, @@ -255,6 +267,19 @@ def create_job( with self._lock: self.jobs[job_id] = job + log = get_log_service() + log.info( + "upload", + "upload_job_created", + f"Created upload job with {len(job.files)} files", + { + "job_id": job_id, + "total_files": len(job.files), + "total_bytes": job.total_bytes, + "auto_upload": auto_upload, + }, + ) + return job def get_job(self, job_id: str) -> UploadJob | None: @@ -319,12 +344,7 @@ def analyze_job( file_state.error_message = str(e) # Update job status - if all(f.status == UploadStatus.READY for f in job.files): - job.status = UploadStatus.READY - elif any(f.status == UploadStatus.READY for f in job.files): - job.status = UploadStatus.READY - else: - job.status = UploadStatus.FAILED + job.resolve_analysis_status() return job @@ -334,6 +354,9 @@ def _analyze_single_file( s3_client: Any, s3_bucket: str, use_cache: bool = True, + job_id: str = "", + progress_callback: Callable[["UploadJob", FileUploadState], None] | None = None, + job: "UploadJob | None" = None, ) -> FileUploadState: """Analyze a single file - extract timestamp and check for duplicates. @@ -342,19 +365,25 @@ def _analyze_single_file( s3_client: S3 client for duplicate checking s3_bucket: S3 bucket name use_cache: Whether to use the cache for duplicate checking + job_id: The parent job ID (for logging) + progress_callback: Optional callback fired when file starts analyzing + job: The parent UploadJob (needed for callback) Returns: The updated FileUploadState """ + log = get_log_service() file_state.status = UploadStatus.ANALYZING + if progress_callback and job: + progress_callback(job, file_state) try: # Extract timestamp from MCAP start_time = mcap_service.extract_start_time(file_state.local_path) file_state.start_time = start_time # Check if timestamp is valid (after 1980) - check_time = start_time.replace(tzinfo=None) if start_time.tzinfo else start_time - file_state.is_valid = check_time >= EPOCH_CUTOFF + naive_start = mcap_service.to_naive_utc(start_time) + file_state.is_valid = naive_start >= EPOCH_CUTOFF.replace(tzinfo=None) # Generate S3 path s3_path = mcap_service.generate_s3_path(start_time, file_state.filename) @@ -387,10 +416,31 @@ def _analyze_single_file( file_state.status = UploadStatus.READY + log.info( + "analysis", + "file_analysis_completed", + f"Analyzed {file_state.filename}", + { + "job_id": job_id, + "filename": file_state.filename, + "file_size": file_state.file_size, + "s3_path": file_state.s3_path, + "is_duplicate": file_state.is_duplicate, + "is_valid": file_state.is_valid, + }, + ) + except Exception as e: file_state.status = UploadStatus.FAILED file_state.error_message = str(e) + log.error( + "analysis", + "file_analysis_failed", + f"Failed to analyze {file_state.filename}: {e}", + {"job_id": job_id, "filename": file_state.filename, "error": str(e)}, + ) + return file_state def analyze_job_async( @@ -415,17 +465,22 @@ def analyze_job_async( Returns: The updated UploadJob or None if not found """ + log = get_log_service() job = self.get_job(job_id) if not job: return None job.status = UploadStatus.ANALYZING - # Mark all files as analyzing - for file_state in job.files: - file_state.status = UploadStatus.ANALYZING + log.info( + "analysis", + "analysis_started", + f"Starting analysis of {len(job.files)} files", + {"job_id": job_id, "total_files": len(job.files)}, + ) - # Send initial progress + # Keep files as PENDING; workers will mark ANALYZING when they start. + # Send initial callbacks so frontend knows the job started and can show queue. if progress_callback: for file_state in job.files: progress_callback(job, file_state) @@ -452,6 +507,9 @@ def analyze_job_async( s3_client, s3_bucket, use_cache, + job_id, + progress_callback, + job, ) futures[future] = file_state @@ -462,7 +520,7 @@ def analyze_job_async( # Result is already updated in-place, but get it to handle exceptions future.result() except Exception as e: - with job._lock: + with job.lock: file_state.status = UploadStatus.FAILED file_state.error_message = str(e) @@ -471,13 +529,25 @@ def analyze_job_async( progress_callback(job, file_state) # Update job status - with job._lock: - if all(f.status == UploadStatus.READY for f in job.files): - job.status = UploadStatus.READY - elif any(f.status == UploadStatus.READY for f in job.files): - job.status = UploadStatus.READY - else: - job.status = UploadStatus.FAILED + with job.lock: + job.resolve_analysis_status() + + ready_count = sum(1 for f in job.files if f.status == UploadStatus.READY) + failed_count = sum(1 for f in job.files if f.status == UploadStatus.FAILED) + duplicate_count = sum(1 for f in job.files if f.is_duplicate) + + log.info( + "analysis", + "analysis_completed", + f"Analysis complete: {ready_count} ready, {failed_count} failed, " + f"{duplicate_count} duplicates", + { + "job_id": job_id, + "ready": ready_count, + "failed": failed_count, + "duplicates": duplicate_count, + }, + ) return job @@ -488,7 +558,7 @@ def start_upload( aws_region: str, s3_bucket: str, skip_duplicates: bool = True, - progress_callback: Any | None = None, + progress_callback: Callable[["UploadJob"], None] | None = None, ) -> None: """Start uploading files in a job. @@ -505,7 +575,7 @@ def start_upload( return job.status = UploadStatus.UPLOADING - job.started_at = datetime.now() + job.started_at = datetime.now(UTC) # Create S3 client try: @@ -518,6 +588,8 @@ def start_upload( file_state.error_message = f"Failed to create S3 client: {e}" return + log = get_log_service() + # Filter files to upload files_to_upload = [] for file_state in job.files: @@ -527,12 +599,28 @@ def start_upload( if skip_duplicates and file_state.is_duplicate: file_state.status = UploadStatus.SKIPPED file_state.bytes_uploaded = file_state.file_size + log.info( + "upload", + "file_upload_skipped", + f"Skipped duplicate: {file_state.filename}", + {"job_id": job_id, "filename": file_state.filename, "reason": "duplicate"}, + ) continue # Skip files with invalid timestamps if not file_state.is_valid: file_state.status = UploadStatus.SKIPPED file_state.error_message = "Invalid timestamp (pre-1980)" + log.warning( + "upload", + "file_upload_skipped", + f"Skipped invalid timestamp: {file_state.filename}", + { + "job_id": job_id, + "filename": file_state.filename, + "reason": "invalid_timestamp", + }, + ) continue files_to_upload.append(file_state) @@ -544,27 +632,45 @@ def start_upload( if job.cancelled: break - def make_callback( + def make_upload_task( fs: FileUploadState, - ) -> Any: - def callback(uploaded: int, total: int) -> None: - with job._lock: - fs.bytes_uploaded = uploaded + ) -> Callable[[], Any]: + def upload_task() -> Any: + # Mark UPLOADING inside the worker so files stay READY until picked up + with job.lock: + fs.status = UploadStatus.UPLOADING + fs.upload_started_at = datetime.now(UTC) + log.info( + "upload", + "file_upload_started", + f"Uploading {fs.filename}", + { + "job_id": job_id, + "filename": fs.filename, + "file_size": fs.file_size, + "s3_path": fs.s3_path, + }, + ) if progress_callback: progress_callback(job) - return callback + def byte_callback(uploaded: int, total: int) -> None: + with job.lock: + fs.bytes_uploaded = uploaded + if progress_callback: + progress_callback(job) - file_state.status = UploadStatus.UPLOADING - file_state.upload_started_at = datetime.now() - future = executor.submit( - s3_service.upload_file_with_progress, - s3_client, - file_state.local_path, - s3_bucket, - file_state.s3_path, - make_callback(file_state), - ) + return s3_service.upload_file_with_progress( + s3_client, + fs.local_path, + s3_bucket, + fs.s3_path, + byte_callback, + ) + + return upload_task + + future = executor.submit(make_upload_task(file_state)) futures[future] = file_state # Process results as they complete @@ -575,10 +681,22 @@ def callback(uploaded: int, total: int) -> None: file_state = futures[future] try: result = future.result() - file_state.upload_completed_at = datetime.now() + file_state.upload_completed_at = datetime.now(UTC) if result["success"]: file_state.status = UploadStatus.COMPLETED file_state.bytes_uploaded = file_state.file_size + log.info( + "upload", + "file_upload_completed", + f"Uploaded {file_state.filename}", + { + "job_id": job_id, + "filename": file_state.filename, + "file_size": file_state.file_size, + "upload_duration_seconds": file_state.upload_duration_seconds, + "s3_path": file_state.s3_path, + }, + ) # Update cache to mark file as existing try: cache = get_cache_service() @@ -590,20 +708,36 @@ def callback(uploaded: int, total: int) -> None: file_size=file_state.file_size, ) except Exception: - pass # Cache update failure shouldn't fail the upload + logger.debug("Cache update failed after upload", exc_info=True) else: file_state.status = UploadStatus.FAILED file_state.error_message = result.get("error", "Unknown error") + log.error( + "upload", + "file_upload_failed", + f"Failed to upload {file_state.filename}: {file_state.error_message}", + { + "job_id": job_id, + "filename": file_state.filename, + "error": file_state.error_message, + }, + ) except Exception as e: - file_state.upload_completed_at = datetime.now() + file_state.upload_completed_at = datetime.now(UTC) file_state.status = UploadStatus.FAILED file_state.error_message = str(e) + log.error( + "upload", + "file_upload_failed", + f"Failed to upload {file_state.filename}: {e}", + {"job_id": job_id, "filename": file_state.filename, "error": str(e)}, + ) if progress_callback: progress_callback(job) # Update final job status - job.completed_at = datetime.now() + job.completed_at = datetime.now(UTC) if job.cancelled: job.status = UploadStatus.CANCELLED elif all(f.status in (UploadStatus.COMPLETED, UploadStatus.SKIPPED) for f in job.files): @@ -616,6 +750,71 @@ def callback(uploaded: int, total: int) -> None: # Clean up temp directory when upload completes self.cleanup_temp_dir(job_id) + uploaded_count = sum(1 for f in job.files if f.status == UploadStatus.COMPLETED) + skipped_count = sum(1 for f in job.files if f.status == UploadStatus.SKIPPED) + failed_count = sum(1 for f in job.files if f.status == UploadStatus.FAILED) + + # Build per-file summary for logging + file_summary = [ + { + "filename": f.filename, + "s3_path": f.s3_path, + "status": f.status.value, + "file_size": f.file_size, + "duration_seconds": f.upload_duration_seconds, + } + for f in job.files + ] + + log.info( + "upload", + "upload_job_completed", + f"Upload job completed: {uploaded_count} uploaded, " + f"{skipped_count} skipped, {failed_count} failed", + { + "job_id": job_id, + "status": job.status.value, + "uploaded": uploaded_count, + "skipped": skipped_count, + "failed": failed_count, + "total_bytes_uploaded": job.successfully_uploaded_bytes, + "duration_seconds": job.total_upload_duration_seconds, + "avg_speed_mbps": job.average_upload_speed_mbps, + "files": file_summary, + }, + ) + + # Save per-job JSONL summary + try: + completed_at = job.completed_at or datetime.now(UTC) + log.save_job_jsonl(job_id, { + "timestamp": completed_at.isoformat(), + "event": "upload_job_completed", + "job_id": job_id, + "status": job.status.value, + "uploaded": uploaded_count, + "skipped": skipped_count, + "failed": failed_count, + "total_bytes_uploaded": job.successfully_uploaded_bytes, + "duration_seconds": job.total_upload_duration_seconds, + "avg_speed_mbps": job.average_upload_speed_mbps, + "files": file_summary, + }, completed_at) + except Exception: + logger.warning("Failed to save job JSONL summary", exc_info=True) + + # Save upload summary CSV + try: + log.save_job_csv(job_id, job, completed_at) + except Exception: + logger.warning("Failed to save job CSV summary", exc_info=True) + + # Auto-sync logs to S3 after job completion + try: + log.sync_logs_to_s3(s3_client, s3_bucket) + except Exception: + logger.debug("Log sync to S3 failed", exc_info=True) + if progress_callback: progress_callback(job) @@ -640,6 +839,14 @@ def cancel_job(self, job_id: str) -> bool: # Clean up temp directory when job is cancelled self.cleanup_temp_dir(job_id) + log = get_log_service() + log.warning( + "upload", + "upload_job_cancelled", + f"Upload job {job_id} cancelled", + {"job_id": job_id}, + ) + return True def cleanup_temp_dir(self, job_id: str) -> bool: @@ -662,22 +869,27 @@ def cleanup_temp_dir(self, job_id: str) -> bool: job.temp_dir = None return True except Exception: - pass + logger.warning("Failed to clean up temp dir: %s", temp_path, exc_info=True) return False def pre_filter_files( self, file_paths: list[str], s3_bucket: str, + aws_profile: str = "default", + aws_region: str = "us-west-2", ) -> tuple[list[str], dict[str, Any]]: """Pre-filter files using cache and filename timestamp extraction. This is a fast pre-filtering step that avoids expensive MCAP parsing by extracting timestamps from filenames and checking the cache. + On cache miss, falls back to S3 HEAD checks. Args: file_paths: List of file paths to filter s3_bucket: S3 bucket name for cache lookup + aws_profile: AWS profile for S3 fallback checks + aws_region: AWS region for S3 fallback checks Returns: Tuple of (files_to_analyze, stats_dict) @@ -686,11 +898,15 @@ def pre_filter_files( cache = get_cache_service() files_to_analyze: list[str] = [] file_statuses: list[dict[str, Any]] = [] + # Track cache misses that have valid S3 paths for batch S3 check + cache_miss_indices: list[int] = [] + cache_miss_s3_paths: list[str] = [] stats: dict[str, Any] = { "total": len(file_paths), "cache_hits": 0, "cache_skipped": 0, + "s3_hits": 0, "no_timestamp": 0, "to_analyze": 0, } @@ -700,13 +916,26 @@ def pre_filter_files( if not path.exists(): continue + stat = path.stat() file_status: dict[str, Any] = { "path": file_path, "filename": path.name, - "size": path.stat().st_size, + "size": stat.st_size, + "mtime": stat.st_mtime, "already_uploaded": False, } + # First: check cache by filename+size (works regardless of timestamp source) + filename_result = cache.check_exists_by_filename( + s3_bucket, path.name, stat.st_size + ) + if filename_result is True: + stats["cache_hits"] += 1 + stats["cache_skipped"] += 1 + file_status["already_uploaded"] = True + file_statuses.append(file_status) + continue + # Try to extract timestamp from filename (fast, no file I/O) timestamp = mcap_service._extract_timestamp_from_filename(path.name) @@ -719,8 +948,9 @@ def pre_filter_files( # Generate S3 path from filename timestamp s3_path = mcap_service.generate_s3_path(timestamp, path.name) + file_status["s3_path"] = s3_path - # Check cache + # Check cache by S3 path cache_result = cache.check_exists_cached(s3_bucket, s3_path) if cache_result is True: @@ -728,18 +958,61 @@ def pre_filter_files( stats["cache_hits"] += 1 stats["cache_skipped"] += 1 file_status["already_uploaded"] = True - else: - # Not in cache or doesn't exist, need analysis - if cache_result is not None: - stats["cache_hits"] += 1 + elif cache_result is False: + # Cache says it doesn't exist + stats["cache_hits"] += 1 files_to_analyze.append(file_path) + else: + # Cache miss (None) — need S3 check + cache_miss_indices.append(len(file_statuses)) + cache_miss_s3_paths.append(s3_path) file_statuses.append(file_status) + # Batch S3 HEAD checks for cache misses + if cache_miss_s3_paths: + try: + s3_client = s3_service.create_s3_client(aws_profile, aws_region) + + def check_s3(s3_path: str) -> bool: + return s3_service.check_file_exists(s3_client, s3_bucket, s3_path) + + with ThreadPoolExecutor(max_workers=self.max_workers) as executor: + results = list(executor.map(check_s3, cache_miss_s3_paths)) + + for idx, s3_path, exists in zip( + cache_miss_indices, cache_miss_s3_paths, results, strict=True + ): + # Update cache with result + fs = file_statuses[idx] + cache.update_cache( + s3_bucket, s3_path, exists, fs["filename"], fs["size"] + ) + if exists: + stats["s3_hits"] += 1 + fs["already_uploaded"] = True + else: + files_to_analyze.append(fs["path"]) + except Exception: + # S3 check failed — fall back to full analysis for cache misses + for idx in cache_miss_indices: + files_to_analyze.append(file_statuses[idx]["path"]) + stats["to_analyze"] = len(files_to_analyze) stats["file_statuses"] = file_statuses return files_to_analyze, stats + def get_active_jobs(self) -> list[UploadJob]: + """Get all currently active (non-terminal) jobs.""" + active_statuses = { + UploadStatus.PENDING, + UploadStatus.ANALYZING, + UploadStatus.READY, + UploadStatus.UPLOADING, + } + with self._lock: + return [j for j in self.jobs.values() if j.status in active_statuses] + def cleanup_old_jobs(self, max_age_seconds: int = 3600) -> int: """Remove completed jobs older than max_age_seconds. @@ -749,7 +1022,7 @@ def cleanup_old_jobs(self, max_age_seconds: int = 3600) -> int: Returns: Number of jobs removed """ - now = datetime.now() + now = datetime.now(UTC) removed = 0 with self._lock: diff --git a/app/services/utils.py b/app/services/utils.py new file mode 100644 index 0000000..93bf86d --- /dev/null +++ b/app/services/utils.py @@ -0,0 +1,18 @@ +"""Shared utility functions for app services.""" + + +def format_file_size(size_bytes: int) -> str: + """Format a file size in bytes to a human-readable string. + + Args: + size_bytes: Size in bytes + + Returns: + Human-readable size string (e.g., "1.5 GB") + """ + size_float = float(size_bytes) + for unit in ["B", "KB", "MB", "GB", "TB"]: + if abs(size_float) < 1024.0: + return f"{size_float:.1f} {unit}" + size_float = size_float / 1024.0 + return f"{size_float:.1f} PB" diff --git a/app/static/images/nrel-logo@2x-01.png b/app/static/images/nlr-logo@2x-01.png similarity index 100% rename from app/static/images/nrel-logo@2x-01.png rename to app/static/images/nlr-logo@2x-01.png diff --git a/app/static/js/app.js b/app/static/js/app.js index fd52f12..08d325b 100644 --- a/app/static/js/app.js +++ b/app/static/js/app.js @@ -11,6 +11,7 @@ import { loadHeaderVersion, openAboutModal, } from './modules/about.js'; +import { hideEl } from './modules/dom.js'; import { goToStep } from './modules/stepper.js'; // Global initialization (runs on every page) @@ -31,6 +32,8 @@ document.addEventListener('click', (e) => { } else if (action === 'go-to-step') { const step = Number(/** @type {HTMLElement} */ (target).dataset.step); if (step) goToStep(step); + } else if (action === 'close-confirm-modal') { + hideEl('confirm-upload-modal'); } }); @@ -43,4 +46,6 @@ if (page === 'upload') { import('./modules/file-browser.js').then(({ initFileBrowser }) => initFileBrowser()); } else if (page === 'settings') { import('./modules/settings.js').then(({ initSettings }) => initSettings()); +} else if (page === 'logs') { + import('./modules/logs.js').then(({ initLogs }) => initLogs()); } diff --git a/app/static/js/modules/about.js b/app/static/js/modules/about.js index 183fc79..a206e82 100644 --- a/app/static/js/modules/about.js +++ b/app/static/js/modules/about.js @@ -1,73 +1,52 @@ /** * About modal functionality. */ -import state from "./state.js"; +import { apiGet } from './api.js'; +import { hideEl, setText, showEl } from './dom.js'; +import state from './state.js'; /** * Load version info from the API and populate header badge. */ -export function loadHeaderVersion() { - fetch("/api/settings/version") - .then((r) => r.json()) - .then((data) => { - state.appVersionData = data; - const version = data.version || "0.0.0"; - - const headerVersion = document.getElementById("header-version"); - if (headerVersion) { - headerVersion.textContent = version; - } - }) - .catch(() => { - const headerVersion = document.getElementById("header-version"); - if (headerVersion) { - headerVersion.textContent = "?"; - } - }); +export async function loadHeaderVersion() { + try { + const data = await apiGet('/api/settings/version'); + state.appVersionData = data; + setText('header-version', data.version || '0.0.0'); + } catch { + setText('header-version', '?'); + } } export function openAboutModal() { - const modal = document.getElementById("about-modal"); - if (modal) { - modal.classList.remove("hidden"); - document.body.style.overflow = "hidden"; - - if (state.appVersionData) { - const aboutVersion = document.getElementById("about-version"); - const aboutCommit = document.getElementById("about-commit"); - const aboutBranch = document.getElementById("about-branch"); + showEl('about-modal'); + document.body.style.overflow = 'hidden'; - if (aboutVersion) - aboutVersion.textContent = state.appVersionData.version || "0.0.0"; - if (aboutCommit) - aboutCommit.textContent = state.appVersionData.commit || "-"; - if (aboutBranch) - aboutBranch.textContent = state.appVersionData.branch || "-"; - } + if (state.appVersionData) { + setText('about-version', state.appVersionData.version || '0.0.0'); + setText('about-commit', state.appVersionData.commit || '-'); + setText('about-branch', state.appVersionData.branch || '-'); } } export function closeAboutModal() { - const modal = document.getElementById("about-modal"); - if (modal) { - modal.classList.add("hidden"); - document.body.style.overflow = ""; - } + hideEl('about-modal'); + document.body.style.overflow = ''; } /** * Initialize about modal event listeners. */ export function initAboutModal() { - document.addEventListener("keydown", (e) => { - if (e.key === "Escape") { + document.addEventListener('keydown', (e) => { + if (e.key === 'Escape') { closeAboutModal(); } }); - const modal = document.getElementById("about-modal"); + const modal = document.getElementById('about-modal'); if (modal) { - modal.addEventListener("click", (e) => { + modal.addEventListener('click', (e) => { if (e.target === modal) { closeAboutModal(); } diff --git a/app/static/js/modules/analysis.js b/app/static/js/modules/analysis.js index b2bf3f6..acc4e18 100644 --- a/app/static/js/modules/analysis.js +++ b/app/static/js/modules/analysis.js @@ -1,50 +1,81 @@ /** - * File analysis/validation (Step 3) and handleFiles entry point. + * File analysis/validation and combined upload progress (Step 3). */ +import { apiGet } from './api.js'; +import { hideEl, setText, showEl } from './dom.js'; +import { formatBytes } from './formatters.js'; +import { showNotification } from './notify.js'; import state from './state.js'; -import { STEP_3_VALIDATED, setUploadStep, showUploadSteps } from './stepper.js'; -import { resetUpload } from './upload-control.js'; -import { connectProgressStream, showCompletionSummary, updateProgressUI } from './upload-exec.js'; -import { formatBytes, showNotification } from './utils.js'; +import { setUploadStep } from './stepper.js'; +import { showCompletionSummary, updateProgressUI } from './upload-exec.js'; + +// Progress weighting: analysis is typically slower than upload +const ANALYSIS_WEIGHT = 70; +const UPLOAD_WEIGHT = 30; + +/** + * Cache of last-known status per filename to avoid redundant DOM rebuilds. + * @type {Map} + */ +const fileRowStatusCache = new Map(); + +/** Pending file updates to flush in the next animation frame. @type {Map} */ +const pendingFileUpdates = new Map(); + +/** Whether a requestAnimationFrame is already scheduled. */ +let rafScheduled = false; + +/** Pending overall progress data to flush. @type {any} */ +let pendingProgressData = null; + +/** Current phase of the upload flow: 'analysis' or 'upload'. */ +let currentPhase = 'analysis'; + +/** + * Reset internal state (call when starting a new job or resetting). + */ +export function resetAnalysisState() { + fileRowStatusCache.clear(); + pendingFileUpdates.clear(); + pendingProgressData = null; + rafScheduled = false; + currentPhase = 'analysis'; +} /** * Check for an active upload job and restore UI state. */ export async function checkForActiveJob() { try { - const response = await fetch('/api/upload/active'); - if (!response.ok) return; - - const data = await response.json(); + const data = await apiGet('/api/upload/active'); if (!data.job_id) return; state.currentJobId = data.job_id; const job = data.job; - if (job.status === 'analyzing') { + if (job.status === 'analyzing' || job.status === 'uploading') { setUploadStep(3); - document.getElementById('drop-zone')?.classList.add('hidden'); - document.getElementById('analysis-section')?.classList.remove('hidden'); - initializeAnalysisTableFromJob(job); - connectAnalysisProgressStream(state.currentJobId); + hideEl('folder-browser-panel'); + showEl('upload-section'); + + if (job.status === 'uploading') { + setText('upload-phase-label', 'Uploading files...'); + } + + connectCombinedProgressStream(state.currentJobId); } else if (job.status === 'ready') { setUploadStep(3); - document.getElementById('drop-zone')?.classList.add('hidden'); - document.getElementById('analysis-section')?.classList.remove('hidden'); - displayAnalysisResults(job); - } else if (job.status === 'uploading') { - setUploadStep(4); - document.getElementById('drop-zone')?.classList.add('hidden'); - document.getElementById('progress-section')?.classList.remove('hidden'); - connectProgressStream(); + hideEl('folder-browser-panel'); + showEl('upload-section'); + connectCombinedProgressStream(state.currentJobId); } else if ( job.status === 'completed' || job.status === 'failed' || job.status === 'cancelled' ) { - setUploadStep(5); - document.getElementById('drop-zone')?.classList.add('hidden'); - document.getElementById('completion-section')?.classList.remove('hidden'); + setUploadStep(4); + hideEl('folder-browser-panel'); + showEl('completion-section'); showCompletionSummary(job); } } catch (error) { @@ -53,259 +84,21 @@ export async function checkForActiveJob() { } /** - * Initialize analysis table from an existing job (for state restoration). - * @param {any} job - */ -function initializeAnalysisTableFromJob(job) { - const tbody = document.getElementById('file-table-body'); - if (!tbody) return; - - tbody.innerHTML = job.files - .map((/** @type {any} */ file) => { - let statusBadge; - if (file.status === 'analyzing') { - statusBadge = ` - - Analyzing - `; - } else if (file.status === 'failed') { - statusBadge = - 'Failed'; - } else if (file.is_duplicate) { - statusBadge = - 'Duplicate'; - } else { - statusBadge = - 'Ready'; - } - - const startTimeDisplay = file.start_time - ? new Date(file.start_time).toLocaleString('en-US', { - month: 'short', - day: 'numeric', - hour: '2-digit', - minute: '2-digit', - }) - : '-'; - - return ` - - -
- - - - ${file.filename} -
- - ${file.file_size_formatted || '-'} - ${startTimeDisplay} - ${file.s3_path || '-'} - ${statusBadge} - - `; - }) - .join(''); - - const totalFiles = document.getElementById('total-files'); - if (totalFiles) totalFiles.textContent = String(job.files.length); - - const uploadBtn = /** @type {HTMLButtonElement | null} */ (document.getElementById('upload-btn')); - if (uploadBtn) uploadBtn.disabled = true; -} - -/** - * Handle files from drag-drop or file input. - * @param {File[]} files - */ -export async function handleFiles(files) { - const formData = new FormData(); - for (const file of files) { - formData.append('files', file); - } - - showUploadSteps(3); - - document.getElementById('drop-zone')?.classList.add('hidden'); - document.getElementById('analysis-section')?.classList.remove('hidden'); - - initializeAnalysisTable(files); - - try { - const response = await fetch('/api/upload/analyze', { - method: 'POST', - body: formData, - }); - - const data = await response.json(); - - if (!response.ok && response.status !== 202) { - throw new Error(data.error || 'Analysis failed'); - } - - state.currentJobId = data.job_id; - - if (response.status === 202) { - connectAnalysisProgressStream(data.job_id); - } else { - displayAnalysisResults(data); - } - } catch (error) { - showNotification(/** @type {Error} */ (error).message, 'error'); - resetUpload(); - } -} - -/** - * Initialize the analysis table with placeholder rows. - * @param {File[]} files - */ -function initializeAnalysisTable(files) { - const tbody = document.getElementById('file-table-body'); - if (!tbody) return; - - tbody.innerHTML = files - .map( - (file) => ` - - -
- - - - ${file.name} -
- - ${formatBytes(file.size)} - - - - - - - - Analyzing - - - - `, - ) - .join(''); - - const totalFiles = document.getElementById('total-files'); - if (totalFiles) totalFiles.textContent = String(files.length); - - const totalSize = document.getElementById('total-size'); - if (totalSize) totalSize.textContent = formatBytes(files.reduce((sum, f) => sum + f.size, 0)); - - const duplicateCount = document.getElementById('duplicate-count'); - if (duplicateCount) duplicateCount.textContent = '0'; - - const uploadBtn = /** @type {HTMLButtonElement | null} */ (document.getElementById('upload-btn')); - if (uploadBtn) uploadBtn.disabled = true; - - const analysisCompleted = document.getElementById('analysis-completed'); - if (analysisCompleted) analysisCompleted.textContent = '0'; - - const analysisTotal = document.getElementById('analysis-total'); - if (analysisTotal) analysisTotal.textContent = String(files.length); - - const analysisPercent = document.getElementById('analysis-percent'); - if (analysisPercent) analysisPercent.textContent = '0%'; - - const analysisProgressBar = /** @type {HTMLElement | null} */ ( - document.getElementById('analysis-progress-bar') - ); - if (analysisProgressBar) analysisProgressBar.style.width = '0%'; - - document.getElementById('analysis-progress')?.classList.remove('hidden'); -} - -/** - * Initialize analysis table from file paths (folder upload). - * @param {string[]} filePaths - */ -export function initializeAnalysisTableFromPaths(filePaths) { - const tbody = document.getElementById('file-table-body'); - if (!tbody) return; - - tbody.innerHTML = filePaths - .map((path) => { - const filename = path.split('/').pop(); - return ` - - -
- - - - ${filename} -
- - - - - - - - - - - Analyzing - - - - `; - }) - .join(''); - - const totalFiles = document.getElementById('total-files'); - if (totalFiles) totalFiles.textContent = String(filePaths.length); - - const totalSize = document.getElementById('total-size'); - if (totalSize) totalSize.textContent = '-'; - - const duplicateCount = document.getElementById('duplicate-count'); - if (duplicateCount) duplicateCount.textContent = '0'; - - const uploadBtn = /** @type {HTMLButtonElement | null} */ (document.getElementById('upload-btn')); - if (uploadBtn) uploadBtn.disabled = true; - - const analysisCompleted = document.getElementById('analysis-completed'); - if (analysisCompleted) analysisCompleted.textContent = '0'; - - const analysisTotal = document.getElementById('analysis-total'); - if (analysisTotal) analysisTotal.textContent = String(filePaths.length); - - const analysisPercent = document.getElementById('analysis-percent'); - if (analysisPercent) analysisPercent.textContent = '0%'; - - const analysisProgressBar = /** @type {HTMLElement | null} */ ( - document.getElementById('analysis-progress-bar') - ); - if (analysisProgressBar) analysisProgressBar.style.width = '0%'; - - document.getElementById('analysis-progress')?.classList.remove('hidden'); -} - -/** - * Connect to SSE stream for analysis progress updates. + * Connect to SSE stream for combined validation + upload progress. * @param {string | null} jobId */ -export function connectAnalysisProgressStream(jobId) { +export function connectCombinedProgressStream(jobId) { if (state.eventSource) { state.eventSource.close(); } + resetAnalysisState(); + state.eventSource = new EventSource(`/api/upload/progress/${jobId}`); + let analysisTotal = 0; + let analysisCompleted = 0; + state.eventSource.onmessage = (event) => { const data = JSON.parse(event.data); @@ -315,12 +108,32 @@ export function connectAnalysisProgressStream(jobId) { return; } + // Analysis progress: queue per-file update for next frame if (data.type === 'analysis_progress') { - updateAnalysisRow(data.file); + queueFileUpdate(data.file); + + // Set total from the first event that carries it + if (data.total_files && analysisTotal === 0) { + analysisTotal = data.total_files; + setText('files-total', data.total_files); + } + + // Only count terminal statuses for progress bar (not pending/analyzing) + if (data.file.status !== 'pending' && data.file.status !== 'analyzing') { + analysisCompleted++; + } + + // Update progress bar for analysis phase (use float, let CSS transition smooth it) + if (analysisTotal > 0) { + const percent = (analysisCompleted / analysisTotal) * ANALYSIS_WEIGHT; + setProgressBar(percent); + } } + // Analysis complete: transition to upload phase if (data.type === 'analysis_complete') { - displayAnalysisResults(data.job); + setProgressBar(ANALYSIS_WEIGHT); + setPhaseLabel('Preparing upload...'); if (!data.auto_upload) { state.eventSource?.close(); @@ -328,22 +141,33 @@ export function connectAnalysisProgressStream(jobId) { } } + // Auto-upload starting if (data.type === 'auto_upload_starting') { - showNotification('Auto-upload starting...', 'info'); - setUploadStep(4); - document.getElementById('analysis-section')?.classList.add('hidden'); - document.getElementById('progress-section')?.classList.remove('hidden'); + currentPhase = 'upload'; + setPhaseLabel('Uploading files...'); } + // Upload progress updates (job-level data without a type field) if (!data.type && data.job_id && data.status) { if (data.status === 'uploading') { - updateProgressUI(data); + // Remap upload progress into the UPLOAD_WEIGHT portion of the bar + const adjusted = { + ...data, + progress_percent: ANALYSIS_WEIGHT + (data.progress_percent / 100) * UPLOAD_WEIGHT, + }; + pendingProgressData = adjusted; + + // Queue per-file row updates + if (data.files) { + for (const file of data.files) { + queueFileUpdate(file); + } + } + scheduleRaf(); } else if (['completed', 'failed', 'cancelled'].includes(data.status)) { state.eventSource?.close(); state.eventSource = null; showCompletionSummary(data); - } else if (data.status === 'ready') { - displayAnalysisResults(data); } } }; @@ -358,226 +182,211 @@ export function connectAnalysisProgressStream(jobId) { } /** - * Update a single row in the analysis table. - * @param {any} fileData + * Set the overall progress bar value (CSS transition handles smoothing). + * @param {number} percent */ -function updateAnalysisRow(fileData) { - const row = /** @type {HTMLTableRowElement | null} */ ( - document.querySelector(`tr[data-filename="${fileData.filename}"]`) - ); - if (!row) return; - - let statusBadge; - if (fileData.status === 'failed') { - statusBadge = `Failed`; - } else if (fileData.is_duplicate) { - statusBadge = - 'Duplicate'; - } else if (fileData.is_valid === false) { - statusBadge = - 'Invalid'; - } else if (fileData.status === 'ready') { - statusBadge = - 'Ready'; - } else { - statusBadge = ` - - Analyzing - `; - } +function setProgressBar(percent) { + const progressBar = /** @type {HTMLElement | null} */ (document.getElementById('progress-bar')); + if (progressBar) progressBar.style.width = `${percent}%`; + setText('progress-percent', percent.toFixed(1)); +} - row.dataset.status = fileData.status; - row.dataset.isDuplicate = fileData.is_duplicate ? 'true' : 'false'; - row.dataset.isAnalyzed = - fileData.status === 'ready' || fileData.status === 'failed' ? 'true' : 'false'; - - const startTimeDisplay = fileData.start_time - ? new Date(fileData.start_time).toLocaleString('en-US', { - month: 'short', - day: 'numeric', - hour: '2-digit', - minute: '2-digit', - }) - : '-'; - - row.innerHTML = ` - -
- - - - ${fileData.filename} -
- - ${fileData.file_size_formatted} - ${startTimeDisplay} - ${fileData.s3_path || '-'} - ${statusBadge} - `; - - updateAnalysisProgress(); - updateAnalysisSummary(); - applyUploadedFilter(); +/** + * Update the phase label with an opacity fade transition. + * @param {string} text + */ +function setPhaseLabel(text) { + const phaseLabel = document.getElementById('upload-phase-label'); + if (!phaseLabel || phaseLabel.textContent === text) return; + phaseLabel.style.opacity = '0'; + setTimeout(() => { + phaseLabel.textContent = text; + phaseLabel.style.opacity = '1'; + }, 150); } -function updateAnalysisSummary() { - const rows = document.querySelectorAll('#file-table-body tr[data-filename]'); - let duplicateCount = 0; +/** + * Queue a file update for the next animation frame. + * @param {any} fileData + */ +function queueFileUpdate(fileData) { + pendingFileUpdates.set(fileData.filename, fileData); + scheduleRaf(); +} - for (const row of rows) { - const badge = row.querySelector('td:last-child span'); - if (badge?.textContent?.includes('Duplicate')) { - duplicateCount++; - } +/** Schedule a requestAnimationFrame if not already pending. */ +function scheduleRaf() { + if (!rafScheduled) { + rafScheduled = true; + requestAnimationFrame(flushUpdates); } - - const el = document.getElementById('duplicate-count'); - if (el) el.textContent = String(duplicateCount); } -function updateAnalysisProgress() { - const rows = document.querySelectorAll('#file-table-body tr[data-filename]'); - const total = rows.length; - let analyzed = 0; +/** Flush all pending updates in a single animation frame. */ +function flushUpdates() { + rafScheduled = false; - for (const row of rows) { - if (/** @type {HTMLElement} */ (row).dataset.isAnalyzed === 'true') { - analyzed++; - } + // Flush pending file row updates + for (const [, fileData] of pendingFileUpdates) { + updateUploadFileRow(fileData); } + pendingFileUpdates.clear(); - const percent = total > 0 ? Math.round((analyzed / total) * 100) : 0; - - const analysisCompleted = document.getElementById('analysis-completed'); - if (analysisCompleted) analysisCompleted.textContent = String(analyzed); - - const analysisTotal = document.getElementById('analysis-total'); - if (analysisTotal) analysisTotal.textContent = String(total); - - const analysisPercent = document.getElementById('analysis-percent'); - if (analysisPercent) analysisPercent.textContent = `${percent}%`; - - const analysisProgressBar = /** @type {HTMLElement | null} */ ( - document.getElementById('analysis-progress-bar') - ); - if (analysisProgressBar) analysisProgressBar.style.width = `${percent}%`; + // Recompute queue positions after row updates + recomputeQueuePositions(); - const progressSection = document.getElementById('analysis-progress'); - if (analyzed === total && total > 0) { - progressSection?.classList.add('hidden'); - } else { - progressSection?.classList.remove('hidden'); + // Flush pending overall progress + if (pendingProgressData) { + updateProgressUI(pendingProgressData); + pendingProgressData = null; } } -export function applyUploadedFilter() { - const hideUploadedEl = /** @type {HTMLInputElement | null} */ ( - document.getElementById('hide-uploaded') - ); - const hideUploaded = hideUploadedEl?.checked || false; - const rows = document.querySelectorAll('#file-table-body tr[data-filename]'); +/** + * Recompute "Queued (X of Y)" labels for pending/ready-queued files. + */ +function recomputeQueuePositions() { + const allRows = document.querySelectorAll('[data-upload-file]'); + // Determine which statuses count as "queued" based on current phase + const queuedStatuses = currentPhase === 'upload' ? ['ready'] : ['pending']; + + // First pass: count queued files + let totalQueued = 0; + for (const row of allRows) { + const filename = /** @type {HTMLElement} */ (row).getAttribute('data-upload-file'); + const status = fileRowStatusCache.get(filename || ''); + if (status && queuedStatuses.includes(status)) { + totalQueued++; + } + } - for (const row of rows) { - if (hideUploaded && /** @type {HTMLElement} */ (row).dataset.isDuplicate === 'true') { - row.classList.add('hidden'); - } else { - row.classList.remove('hidden'); + if (totalQueued === 0) return; + + // Second pass: assign positions + let pos = 1; + for (const row of allRows) { + const filename = /** @type {HTMLElement} */ (row).getAttribute('data-upload-file'); + const status = fileRowStatusCache.get(filename || ''); + if (status && queuedStatuses.includes(status)) { + const label = row.querySelector('[data-queue-label]'); + if (label) { + label.textContent = + currentPhase === 'upload' + ? `Queued for upload (${pos} of ${totalQueued})` + : `Queued (${pos} of ${totalQueued})`; + } + pos++; } } } /** - * Display final analysis results. - * @param {any} job + * Update a single file row with minimal DOM changes. + * Only rebuilds innerHTML on status transitions; for uploading progress, + * just updates the bar width and percentage text to avoid resetting spinners. + * @param {any} fileData */ -export function displayAnalysisResults(job) { - const tbody = document.getElementById('file-table-body'); - if (!tbody) return; - - let totalSize = 0; - let duplicateCount = 0; - let invalidCount = 0; - - tbody.innerHTML = job.files - .map((/** @type {any} */ file) => { - totalSize += file.file_size; - if (file.is_duplicate) duplicateCount++; - if (file.is_valid === false) invalidCount++; - - let statusBadge; - if (file.status === 'failed') { - statusBadge = `Failed`; - } else if (file.is_duplicate) { - statusBadge = - 'Duplicate'; - } else if (file.is_valid === false) { - statusBadge = - 'Invalid'; - } else { - statusBadge = - 'Ready'; - } +function updateUploadFileRow(fileData) { + const row = document.querySelector(`[data-upload-file="${fileData.filename}"]`); + if (!row) return; - const startTimeDisplay = file.start_time - ? new Date(file.start_time).toLocaleString('en-US', { - month: 'short', - day: 'numeric', - hour: '2-digit', - minute: '2-digit', - }) - : '-'; - - return ` - - -
- - - - ${file.filename} -
- - ${file.file_size_formatted} - ${startTimeDisplay} - ${file.s3_path || '-'} - ${statusBadge} - - `; - }) - .join(''); - - const totalFilesEl = document.getElementById('total-files'); - if (totalFilesEl) totalFilesEl.textContent = String(job.files.length); - - const totalSizeEl = document.getElementById('total-size'); - if (totalSizeEl) totalSizeEl.textContent = formatBytes(totalSize); - - const duplicateCountEl = document.getElementById('duplicate-count'); - if (duplicateCountEl) { - duplicateCountEl.textContent = - String(duplicateCount) + (invalidCount > 0 ? ` + ${invalidCount} invalid` : ''); + const cachedStatus = fileRowStatusCache.get(fileData.filename); + const newStatus = fileData.is_duplicate ? `${fileData.status}:dup` : fileData.status; + + // If status hasn't changed, only do micro-updates for uploading progress + if (cachedStatus === newStatus) { + if (fileData.status === 'uploading') { + const bar = /** @type {HTMLElement | null} */ (row.querySelector('[data-progress-bar]')); + if (bar) bar.style.width = `${fileData.progress_percent || 0}%`; + const label = row.querySelector('[data-progress-label]'); + if (label) { + label.textContent = + fileData.progress_percent != null ? `${fileData.progress_percent.toFixed(0)}%` : ''; + } + } + return; } - document.getElementById('analysis-progress')?.classList.add('hidden'); - - const hasUploadableFiles = job.files.some( - (/** @type {any} */ f) => f.status === 'ready' && f.is_valid !== false && !f.is_duplicate, - ); - const uploadBtn = /** @type {HTMLButtonElement | null} */ (document.getElementById('upload-btn')); - if (uploadBtn) uploadBtn.disabled = !hasUploadableFiles; + // Status changed — full rebuild of the row + fileRowStatusCache.set(fileData.filename, newStatus); + row.innerHTML = buildFileRowHTML(fileData); +} - const descriptionEl = document.getElementById('step-description'); - if (descriptionEl) { - descriptionEl.textContent = STEP_3_VALIDATED; +/** + * Build the full HTML for a file row. + * @param {any} fileData + * @returns {string} + */ +function buildFileRowHTML(fileData) { + let statusIcon = ''; + let statusText = ''; + + if (fileData.status === 'completed') { + statusIcon = + ''; + statusText = 'Uploaded'; + } else if (fileData.status === 'failed') { + statusIcon = + ''; + statusText = `Failed`; + } else if (fileData.status === 'skipped') { + statusIcon = + ''; + statusText = 'Skipped'; + } else if (fileData.status === 'uploading') { + statusIcon = + ''; + const pct = fileData.progress_percent != null ? `${fileData.progress_percent.toFixed(0)}%` : ''; + statusText = ` +
+
+
+
+ ${pct} +
`; + } else if (fileData.status === 'analyzing') { + statusIcon = + ''; + statusText = 'Validating'; + } else if (fileData.status === 'ready') { + if (fileData.is_duplicate) { + statusIcon = + ''; + statusText = 'Duplicate'; + } else if (currentPhase === 'upload') { + // In upload phase, non-duplicate ready files are queued for upload + statusIcon = + '
'; + statusText = 'Queued for upload'; + } else { + statusIcon = + ''; + statusText = 'Validated'; + } + } else if (fileData.status === 'pending') { + statusIcon = + '
'; + statusText = 'Queued'; + } else { + statusText = `${fileData.status || 'Unknown'}`; } - applyUploadedFilter(); + return ` +
+ ${statusIcon || '
'} + ${fileData.filename} +
+
+ ${fileData.file_size_formatted ? `${fileData.file_size_formatted}` : ''} + ${statusText} +
+ `; +} + +/** + * @param {any} fileData + * @returns {string} + */ +export function formatFileSize(fileData) { + return fileData.file_size_formatted || formatBytes(fileData.file_size || 0); } diff --git a/app/static/js/modules/api.js b/app/static/js/modules/api.js new file mode 100644 index 0000000..ca68506 --- /dev/null +++ b/app/static/js/modules/api.js @@ -0,0 +1,51 @@ +/** + * Shared fetch wrappers for JSON API calls. + */ + +/** + * GET a JSON endpoint. Throws on non-ok responses. + * @param {string} url + * @returns {Promise} + */ +export async function apiGet(url) { + const response = await fetch(url); + const data = await response.json(); + if (!response.ok) throw new Error(data.error || `Request failed (${response.status})`); + return data; +} + +/** + * POST to a JSON endpoint. Throws on non-ok responses. + * @param {string} url + * @param {any} [body] - If provided, sent as JSON with Content-Type header. + * @returns {Promise} + */ +export async function apiPost(url, body) { + /** @type {RequestInit} */ + const options = { method: 'POST' }; + if (body !== undefined) { + options.headers = { 'Content-Type': 'application/json' }; + options.body = JSON.stringify(body); + } + const response = await fetch(url, options); + const data = await response.json(); + if (!response.ok) throw new Error(data.error || `Request failed (${response.status})`); + return data; +} + +/** + * PUT to a JSON endpoint. Throws on non-ok responses. + * @param {string} url + * @param {any} body + * @returns {Promise} + */ +export async function apiPut(url, body) { + const response = await fetch(url, { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }); + const data = await response.json(); + if (!response.ok) throw new Error(data.error || `Request failed (${response.status})`); + return data; +} diff --git a/app/static/js/modules/debounce.js b/app/static/js/modules/debounce.js new file mode 100644 index 0000000..d7b0b72 --- /dev/null +++ b/app/static/js/modules/debounce.js @@ -0,0 +1,16 @@ +/** + * Debounce utility. + */ + +/** + * @param {(...args: any[]) => void} fn + * @param {number} delay + * @returns {(...args: any[]) => void} + */ +export function debounce(fn, delay) { + let timeout = 0; + return (/** @type {any[]} */ ...args) => { + clearTimeout(timeout); + timeout = window.setTimeout(() => fn(...args), delay); + }; +} diff --git a/app/static/js/modules/dom.js b/app/static/js/modules/dom.js new file mode 100644 index 0000000..0af1309 --- /dev/null +++ b/app/static/js/modules/dom.js @@ -0,0 +1,73 @@ +/** + * DOM manipulation helpers. + */ + +/** + * Set the text content of an element by ID. + * @param {string} id + * @param {string | number} value + */ +export function setText(id, value) { + const el = document.getElementById(id); + if (el) el.textContent = String(value); +} + +/** + * Remove the `hidden` class from an element by ID. + * @param {string} id + */ +export function showEl(id) { + document.getElementById(id)?.classList.remove('hidden'); +} + +/** + * Add the `hidden` class to an element by ID. + * @param {string} id + */ +export function hideEl(id) { + document.getElementById(id)?.classList.add('hidden'); +} + +/** + * Disable a button, swap its label, run a callback, then restore. + * @param {HTMLButtonElement | null} btn + * @param {string} loadingText + * @param {() => Promise} callback + */ +/** + * Toggle the `hidden` class on an element by ID. + * @param {string} id + * @param {boolean} show - When true, remove `hidden`; when false, add it. + */ +export function toggleEl(id, show) { + document.getElementById(id)?.classList.toggle('hidden', !show); +} + +/** + * Append text to an element's textContent by ID. + * @param {string} id + * @param {string} value + */ +export function appendText(id, value) { + const el = document.getElementById(id); + if (el) el.textContent += String(value); +} + +/** + * Disable a button, swap its label, run a callback, then restore. + * @param {HTMLButtonElement | null} btn + * @param {string} loadingText + * @param {() => Promise} callback + */ +export async function withLoadingButton(btn, loadingText, callback) { + if (!btn) return; + const originalText = btn.textContent; + btn.disabled = true; + btn.textContent = loadingText; + try { + await callback(); + } finally { + btn.disabled = false; + btn.textContent = originalText; + } +} diff --git a/app/static/js/modules/file-browser.js b/app/static/js/modules/file-browser.js index 3d9528b..cfc58ea 100644 --- a/app/static/js/modules/file-browser.js +++ b/app/static/js/modules/file-browser.js @@ -1,8 +1,13 @@ /** * S3 file browser page functionality. */ +import { apiGet } from './api.js'; +import { debounce } from './debounce.js'; +import { hideEl, setText, showEl } from './dom.js'; +import { formatBytes } from './formatters.js'; +import { fileIcon, folderIcon } from './icons.js'; +import { showNotification } from './notify.js'; import state from './state.js'; -import { formatBytes, showNotification } from './utils.js'; export function initFileBrowser() { const refreshBtn = document.getElementById('refresh-btn'); @@ -18,32 +23,25 @@ export function initFileBrowser() { retryBtn?.addEventListener('click', () => loadFiles(state.currentPrefix)); closeSearchBtn?.addEventListener('click', hideSearchResults); - let searchTimeout = 0; - searchInput?.addEventListener('input', () => { - clearTimeout(searchTimeout); - searchTimeout = window.setTimeout(() => { - const query = searchInput.value.trim(); - if (query.length >= 2) { - searchFiles(query); - } else { - hideSearchResults(); - } - }, 300); - }); + const debouncedSearch = debounce(() => { + const query = searchInput?.value.trim() || ''; + if (query.length >= 2) { + searchFiles(query); + } else { + hideSearchResults(); + } + }, 300); + searchInput?.addEventListener('input', debouncedSearch); loadSettings().then(() => loadFiles('')); } async function loadSettings() { try { - const response = await fetch('/api/settings'); - const settings = await response.json(); - - const bucketName = document.getElementById('bucket-name'); - if (bucketName) bucketName.textContent = settings.s3_bucket || 'No bucket configured'; + const settings = await apiGet('/api/settings'); + setText('bucket-name', settings.s3_bucket || 'No bucket configured'); } catch (_error) { - const bucketName = document.getElementById('bucket-name'); - if (bucketName) bucketName.textContent = 'Error loading settings'; + setText('bucket-name', 'Error loading settings'); } } @@ -53,35 +51,32 @@ async function loadSettings() { async function loadFiles(prefix) { state.currentPrefix = prefix; - document.getElementById('loading-state')?.classList.remove('hidden'); - document.getElementById('error-state')?.classList.add('hidden'); - document.getElementById('empty-state')?.classList.add('hidden'); - document.getElementById('file-list')?.classList.add('hidden'); + showEl('loading-state'); + hideEl('error-state'); + hideEl('empty-state'); + hideEl('file-list'); try { - const response = await fetch(`/api/files/list?prefix=${encodeURIComponent(prefix)}`); - const data = await response.json(); + const data = await apiGet(`/api/files/list?prefix=${encodeURIComponent(prefix)}`); - if (!response.ok || !data.success) { + if (!data.success) { throw new Error(data.error || 'Failed to load files'); } - document.getElementById('loading-state')?.classList.add('hidden'); + hideEl('loading-state'); updateBreadcrumb(data.breadcrumbs || []); if (data.folders.length === 0 && data.files.length === 0) { - document.getElementById('empty-state')?.classList.remove('hidden'); + showEl('empty-state'); return; } displayFiles(data.folders, data.files); } catch (error) { - document.getElementById('loading-state')?.classList.add('hidden'); - document.getElementById('error-state')?.classList.remove('hidden'); - - const errorMessage = document.getElementById('error-message'); - if (errorMessage) errorMessage.textContent = /** @type {Error} */ (error).message; + hideEl('loading-state'); + showEl('error-state'); + setText('error-message', /** @type {Error} */ (error).message); } } @@ -93,12 +88,12 @@ function updateBreadcrumb(breadcrumbs) { if (!nav) return; nav.innerHTML = ` - Root + Root ${breadcrumbs .map( (b) => ` / - ${b.name} + ${b.name} `, ) .join('')} @@ -124,9 +119,7 @@ function displayFiles(folders, files) { ...folders.map( (folder) => `
- - - + ${folderIcon()} ${folder.name}/
`, @@ -135,10 +128,7 @@ function displayFiles(folders, files) { (file) => `
- - - + ${fileIcon()} ${file.name}
@@ -150,7 +140,7 @@ function displayFiles(folders, files) { ), ].join(''); - fileList.classList.remove('hidden'); + showEl('file-list'); for (const el of fileList.querySelectorAll('[data-prefix]')) { el.addEventListener('click', () => @@ -164,15 +154,9 @@ function displayFiles(folders, files) { */ async function searchFiles(query) { try { - const response = await fetch( + const data = await apiGet( `/api/files/search?query=${encodeURIComponent(query)}&prefix=${encodeURIComponent(state.currentPrefix)}`, ); - const data = await response.json(); - - if (!response.ok) { - throw new Error(data.error || 'Search failed'); - } - showSearchResults(data.files, query); } catch (error) { showNotification(/** @type {Error} */ (error).message, 'error'); @@ -196,10 +180,7 @@ function showSearchResults(files, _query) { (file) => `
- - - + ${fileIcon()}
${file.name}
${file.key}
@@ -211,11 +192,11 @@ function showSearchResults(files, _query) { ) .join(''); - container.classList.remove('hidden'); + showEl('search-results'); } function hideSearchResults() { - document.getElementById('search-results')?.classList.add('hidden'); + hideEl('search-results'); const searchInput = /** @type {HTMLInputElement | null} */ ( document.getElementById('search-input') diff --git a/app/static/js/modules/file-handler.js b/app/static/js/modules/file-handler.js deleted file mode 100644 index bc7b885..0000000 --- a/app/static/js/modules/file-handler.js +++ /dev/null @@ -1,106 +0,0 @@ -/** - * File extraction from drag-and-drop and file input. - */ - -/** - * Recursively read all files from a FileSystemDirectoryHandle. - * @param {FileSystemDirectoryHandle} dirHandle - * @param {string} [path] - * @returns {Promise} - */ -export async function readDirectoryRecursively(dirHandle, path = '') { - const files = []; - - // @ts-ignore — FileSystemDirectoryHandle async iteration not in all TS DOM libs - for await (const [name, handle] of dirHandle) { - if (handle.kind === 'file') { - const file = await handle.getFile(); - file.relativePath = path + name; - files.push(file); - } else if (handle.kind === 'directory') { - const subFiles = await readDirectoryRecursively(handle, `${path}${name}/`); - files.push(...subFiles); - } - } - - return files; -} - -/** - * Extract all files from dropped items, recursively scanning folders. - * @param {DataTransferItemList} items - * @returns {Promise} - */ -export async function extractFilesFromDrop(items) { - const files = []; - const entries = []; - - for (let i = 0; i < items.length; i++) { - const item = items[i]; - if (item.kind === 'file') { - const entry = item.webkitGetAsEntry ? item.webkitGetAsEntry() : null; - if (entry) { - entries.push(entry); - } else { - const file = item.getAsFile(); - if (file) files.push(file); - } - } - } - - for (const entry of entries) { - const entryFiles = await readEntryRecursively(entry); - files.push(...entryFiles); - } - - return files; -} - -/** - * Recursively read a FileSystemEntry (file or directory). - * @param {FileSystemEntry} entry - * @returns {Promise} - */ -async function readEntryRecursively(entry) { - if (entry.isFile) { - return new Promise((resolve) => { - /** @type {FileSystemFileEntry} */ (entry).file( - (file) => resolve([file]), - () => resolve([]), - ); - }); - } - - if (entry.isDirectory) { - const files = []; - const reader = /** @type {FileSystemDirectoryEntry} */ (entry).createReader(); - - const readAllEntries = async () => { - const allEntries = []; - const readBatch = () => - new Promise((resolve) => { - reader.readEntries( - (batch) => resolve(batch), - () => resolve([]), - ); - }); - - let batch; - do { - batch = await readBatch(); - allEntries.push(...batch); - } while (batch.length > 0); - - return allEntries; - }; - - const childEntries = await readAllEntries(); - for (const child of childEntries) { - const childFiles = await readEntryRecursively(child); - files.push(...childFiles); - } - return files; - } - - return []; -} diff --git a/app/static/js/modules/folder-browser.js b/app/static/js/modules/folder-browser.js index 3c7234d..55c0102 100644 --- a/app/static/js/modules/folder-browser.js +++ b/app/static/js/modules/folder-browser.js @@ -1,100 +1,68 @@ -import { connectAnalysisProgressStream, initializeAnalysisTableFromPaths } from './analysis.js'; +import { connectCombinedProgressStream } from './analysis.js'; +import { apiGet, apiPost } from './api.js'; +import { hideEl, setText, showEl } from './dom.js'; +import { formatBytes, formatMtime } from './formatters.js'; +import { fileIcon, folderIcon } from './icons.js'; +import { showNotification } from './notify.js'; +import { toggleSort, updateSortIndicators } from './sorting-helpers.js'; /** - * Folder browser modal and scan results for folder-based upload. + * Inline folder browser and scan results for folder-based upload. */ import state from './state.js'; import { setUploadStep, showUploadSteps } from './stepper.js'; -import { formatBytes, showNotification } from './utils.js'; /** - * Initialize folder browser modal handlers. + * Initialize the inline folder browser and auto-load initial folder. */ -export function initFolderBrowser() { - const modal = document.getElementById('folder-browser-modal'); - if (!modal) return; - - document.getElementById('close-folder-browser')?.addEventListener('click', closeFolderBrowser); - document.getElementById('cancel-folder-browser')?.addEventListener('click', closeFolderBrowser); - - modal.addEventListener('click', (e) => { - if (e.target === modal.querySelector('.fixed.inset-0')) { - closeFolderBrowser(); - } - }); - - document.addEventListener('keydown', (e) => { - if (e.key === 'Escape' && !modal.classList.contains('hidden')) { - closeFolderBrowser(); +export async function initFolderBrowser() { + const panel = document.getElementById('folder-browser-panel'); + if (!panel) return; + + document.getElementById('select-folder-btn')?.addEventListener('click', selectCurrentFolder); + + // Delegated click handler for folder navigation (shared across 3 containers) + /** @param {Event} e */ + function handleFolderNavClick(e) { + const target = /** @type {HTMLElement} */ (e.target).closest('[data-action="navigate-folder"]'); + if (target) { + loadFolderBrowser(/** @type {HTMLElement} */ (target).dataset.path || ''); } - }); - - document.getElementById('select-current-folder')?.addEventListener('click', selectCurrentFolder); - - // Delegated click handler for folder navigation - const folderList = document.getElementById('folder-list'); - if (folderList) { - folderList.addEventListener('click', (e) => { - const target = /** @type {HTMLElement} */ (e.target).closest( - '[data-action="navigate-folder"]', - ); - if (target) { - loadFolderBrowser(/** @type {HTMLElement} */ (target).dataset.path || ''); - } - }); } - const quickLinks = document.getElementById('folder-quick-links'); - if (quickLinks) { - quickLinks.addEventListener('click', (e) => { - const target = /** @type {HTMLElement} */ (e.target).closest( - '[data-action="navigate-folder"]', - ); - if (target) { - loadFolderBrowser(/** @type {HTMLElement} */ (target).dataset.path || ''); + document.getElementById('folder-list')?.addEventListener('click', handleFolderNavClick); + document.getElementById('folder-quick-links')?.addEventListener('click', handleFolderNavClick); + document.getElementById('folder-breadcrumb')?.addEventListener('click', handleFolderNavClick); + + // Sort header click handler (Review table) + const scanSection = document.getElementById('scan-results-section'); + if (scanSection) { + scanSection.addEventListener('click', (e) => { + const th = /** @type {HTMLElement} */ (e.target).closest('[data-sort]'); + if (th) { + sortReviewTable(/** @type {HTMLElement} */ (th).dataset.sort || 'filename'); } }); } - const breadcrumb = document.getElementById('folder-breadcrumb'); - if (breadcrumb) { - breadcrumb.addEventListener('click', (e) => { - const target = /** @type {HTMLElement} */ (e.target).closest( - '[data-action="navigate-folder"]', - ); - if (target) { - loadFolderBrowser(/** @type {HTMLElement} */ (target).dataset.path || ''); + // Sort header click handler (File browser table) + const fileTableHead = document.getElementById('file-table-head'); + if (fileTableHead) { + fileTableHead.addEventListener('click', (e) => { + const th = /** @type {HTMLElement} */ (e.target).closest('[data-file-sort]'); + if (th) { + sortBrowserFileTable(/** @type {HTMLElement} */ (th).dataset.fileSort || 'name'); } }); } -} - -export function closeFolderBrowser() { - const modal = document.getElementById('folder-browser-modal'); - if (modal) modal.classList.add('hidden'); - document.body.style.overflow = ''; -} - -/** - * Open the folder browser modal. - */ -export async function openFolderBrowser() { - const modal = document.getElementById('folder-browser-modal'); - if (modal) modal.classList.remove('hidden'); - document.body.style.overflow = 'hidden'; + // Auto-load: use last folder or default from settings const lastFolder = localStorage.getItem('lastUploadFolder'); - if (lastFolder) { loadFolderBrowser(lastFolder); } else { try { - const response = await fetch('/api/settings'); - const settings = await response.json(); - if (settings.default_upload_folder) { - loadFolderBrowser(settings.default_upload_folder); - } else { - loadFolderBrowser(''); - } + const settings = await apiGet('/api/settings'); + loadFolderBrowser(settings.default_upload_folder || ''); } catch (_error) { loadFolderBrowser(''); } @@ -103,17 +71,14 @@ export async function openFolderBrowser() { /** * Load folder contents from the backend. + * Uses raw fetch for complex retry-on-failure logic. * @param {string} path * @param {boolean} [isRetry] */ export async function loadFolderBrowser(path, isRetry = false) { - const folderList = document.getElementById('folder-list'); - const loadingState = document.getElementById('folder-loading'); - const errorState = document.getElementById('folder-error'); - - if (folderList) folderList.classList.add('hidden'); - if (errorState) errorState.classList.add('hidden'); - if (loadingState) loadingState.classList.remove('hidden'); + hideEl('folder-list'); + hideEl('folder-error'); + showEl('folder-loading'); try { const url = path ? `/api/files/browse?path=${encodeURIComponent(path)}` : '/api/files/browse'; @@ -138,7 +103,7 @@ export async function loadFolderBrowser(path, isRetry = false) { if (lastUsedFolder && lastUsedFolder !== data.current_path) { const lastFolderName = lastUsedFolder.split('/').pop() || lastUsedFolder; quickLinksHtml += ` - @@ -166,7 +131,7 @@ export async function loadFolderBrowser(path, isRetry = false) { .map( (/** @type {{ name: string, path: string }} */ crumb, /** @type {number} */ i) => ` ${i > 0 ? '/' : ''} - @@ -176,22 +141,35 @@ export async function loadFolderBrowser(path, isRetry = false) { } // Update MCAP count - const mcapCount = document.getElementById('folder-mcap-count'); - if (mcapCount) mcapCount.textContent = String(data.mcap_count); + setText('folder-mcap-count', data.mcap_count); - // Enable/disable select button - const selectBtn = /** @type {HTMLButtonElement | null} */ ( - document.getElementById('select-current-folder') + // Enable select button and store current path + const selectFolderBtn = /** @type {HTMLButtonElement | null} */ ( + document.getElementById('select-folder-btn') ); - if (selectBtn) { - selectBtn.disabled = false; - selectBtn.dataset.path = data.current_path; + if (selectFolderBtn) { + selectFolderBtn.disabled = false; + selectFolderBtn.dataset.path = data.current_path; + } + + // Update bottom summary with MCAP count + if (data.mcap_count > 0) { + setText( + 'folder-select-summary', + `${data.mcap_count} MCAP file${data.mcap_count === 1 ? '' : 's'} in this folder.`, + ); + } else { + setText( + 'folder-select-summary', + 'Navigate to the folder containing your MCAP files, then click Upload Folder.', + ); } // Render folder list - if (loadingState) loadingState.classList.add('hidden'); - if (folderList) folderList.classList.remove('hidden'); + hideEl('folder-loading'); + showEl('folder-list'); + const folderList = document.getElementById('folder-list'); if (folderList) { if (data.folders.length === 0 && data.files.length === 0) { folderList.innerHTML = @@ -214,9 +192,7 @@ export async function loadFolderBrowser(path, isRetry = false) {
- - - + ${folderIcon()} ${folder.name}
${ @@ -227,36 +203,27 @@ export async function loadFolderBrowser(path, isRetry = false) {
`, ), - ...data.files.slice(0, 10).map( - (/** @type {{ name: string, size: number }} */ file) => ` -
-
- - - - ${file.name} -
- ${formatBytes(file.size)} -
- `, - ), - data.files.length > 10 - ? ` -
- ... and ${data.files.length - 10} more MCAP files -
- ` - : '', ].join(''); } } - } catch (error) { - if (loadingState) loadingState.classList.add('hidden'); - if (errorState) errorState.classList.remove('hidden'); - const errorMessage = document.getElementById('folder-error-message'); - if (errorMessage) errorMessage.textContent = /** @type {Error} */ (error).message; + // Render MCAP file table + if (data.files.length > 0) { + state.browserFiles = data.files; + state.browserFileSortConfig = state.browserFileSortConfig || { + column: 'name', + ascending: true, + }; + renderBrowserFileTable(); + showEl('file-table-section'); + } else { + state.browserFiles = []; + hideEl('file-table-section'); + } + } catch (error) { + hideEl('folder-loading'); + showEl('folder-error'); + setText('folder-error-message', /** @type {Error} */ (error).message); } } @@ -264,29 +231,20 @@ export async function loadFolderBrowser(path, isRetry = false) { * Select the current folder and scan for MCAP files. */ async function selectCurrentFolder() { - const selectBtn = /** @type {HTMLButtonElement | null} */ ( - document.getElementById('select-current-folder') + const btn = /** @type {HTMLButtonElement | null} */ ( + document.getElementById('select-folder-btn') ); - if (!selectBtn) return; + if (!btn) return; - const folderPath = selectBtn.dataset.path; + const folderPath = btn.dataset.path; if (!folderPath) return; - selectBtn.disabled = true; - selectBtn.textContent = 'Scanning...'; + btn.disabled = true; + const btnSpan = btn.querySelector('span'); + if (btnSpan) btnSpan.textContent = 'Scanning...'; try { - const response = await fetch('/api/upload/scan-folder', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ folder_path: folderPath }), - }); - - const data = await response.json(); - - if (!response.ok) { - throw new Error(data.error || 'Failed to scan folder'); - } + const data = await apiPost('/api/upload/scan-folder', { folder_path: folderPath }); if (data.total_count === 0) { showNotification('No MCAP files found in this folder', 'error'); @@ -296,63 +254,68 @@ async function selectCurrentFolder() { state.selectedFolderPath = folderPath; localStorage.setItem('lastUploadFolder', folderPath); - closeFolderBrowser(); - - const prefilterResponse = await fetch('/api/upload/bulk-analyze', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - file_paths: data.files.map((/** @type {{ path: string }} */ f) => f.path), - pre_filter_only: true, - }), + const prefilterData = await apiPost('/api/upload/bulk-analyze', { + file_paths: data.files.map((/** @type {{ path: string }} */ f) => f.path), + pre_filter_only: true, }); - const prefilterData = await prefilterResponse.json(); - showScanResults(data, prefilterData.pre_filter_stats || {}); } catch (error) { showNotification(/** @type {Error} */ (error).message, 'error'); } finally { - selectBtn.disabled = false; - selectBtn.textContent = 'Select This Folder'; + btn.disabled = false; + const resetSpan = btn.querySelector('span'); + if (resetSpan) resetSpan.textContent = 'Upload This Folder'; } } /** - * Show scan results UI. + * Show scan results UI with sortable review table. * @param {any} scanData * @param {any} prefilterStats */ function showScanResults(scanData, prefilterStats) { - const scanSection = document.getElementById('scan-results-section'); - const dropZone = document.getElementById('drop-zone'); - showUploadSteps(2); - const selectedFolderPath = document.getElementById('selected-folder-path'); - if (selectedFolderPath) selectedFolderPath.textContent = scanData.folder_path; + // Store folder path for relative path computation + state.scanFolderPath = scanData.folder_path; + state.scanTotalSize = scanData.total_size || 0; - const scanTotal = document.getElementById('scan-total'); - if (scanTotal) scanTotal.textContent = String(scanData.total_count); + setText('selected-folder-path', scanData.folder_path); + setText('scan-total', scanData.total_count); + setText('scan-total-volume', formatBytes(state.scanTotalSize)); + setText('scan-already-uploaded', prefilterStats.cache_skipped || 0); - const scanToUpload = document.getElementById('scan-to-upload'); - if (scanToUpload) { - scanToUpload.textContent = String(prefilterStats.to_analyze || scanData.total_count); + // Merge scan data files with prefilter statuses + const fileStatuses = prefilterStats.file_statuses || []; + /** @type {Map} */ + const prefilterMap = new Map(); + for (const fs of fileStatuses) { + prefilterMap.set(fs.path, fs); } - const scanAlreadyUploaded = document.getElementById('scan-already-uploaded'); - if (scanAlreadyUploaded) { - scanAlreadyUploaded.textContent = String(prefilterStats.cache_skipped || 0); + /** @type {Array} */ + const mergedStatuses = []; + for (const scanFile of scanData.files) { + const pf = prefilterMap.get(scanFile.path); + mergedStatuses.push({ + path: scanFile.path, + filename: scanFile.filename, + size: scanFile.size, + mtime: scanFile.mtime || 0, + relative_path: scanFile.relative_path || scanFile.filename, + already_uploaded: pf ? pf.already_uploaded : false, + }); } - const fileStatuses = prefilterStats.file_statuses || []; - state.scanFilePaths = fileStatuses + state.scanFilePaths = mergedStatuses .filter((/** @type {any} */ f) => !f.already_uploaded) .map((/** @type {any} */ f) => f.path); - state.scanFileStatuses = fileStatuses; + state.scanFileStatuses = mergedStatuses; + state.reviewSortConfig = { column: 'filename', ascending: true }; - renderScanFileList(fileStatuses); + renderReviewTable(mergedStatuses); const hideUploadedCheckbox = /** @type {HTMLInputElement | null} */ ( document.getElementById('scan-hide-uploaded') @@ -362,58 +325,96 @@ function showScanResults(scanData, prefilterStats) { hideUploadedCheckbox.onchange = () => applyScanFileFilter(); } - const startBtn = /** @type {HTMLButtonElement | null} */ ( - document.getElementById('start-upload-btn') + const continueBtn = /** @type {HTMLButtonElement | null} */ ( + document.getElementById('continue-upload-btn') ); - if (startBtn) { - startBtn.disabled = (prefilterStats.to_analyze || scanData.total_count) === 0; + if (continueBtn) { + continueBtn.disabled = state.scanFileStatuses.length === 0; } - if (dropZone) dropZone.classList.add('hidden'); - if (scanSection) scanSection.classList.remove('hidden'); + hideEl('folder-browser-panel'); + showEl('scan-results-section'); } /** + * Render the sortable review table body. * @param {any[]} fileStatuses */ -function renderScanFileList(fileStatuses) { - const fileList = document.getElementById('scan-file-list'); - if (!fileList) return; +function renderReviewTable(fileStatuses) { + const tbody = document.getElementById('scan-file-list'); + if (!tbody) return; + + const { column, ascending } = state.reviewSortConfig; const sorted = [...fileStatuses].sort((a, b) => { - if (a.already_uploaded === b.already_uploaded) { - return a.filename.localeCompare(b.filename); + let cmp = 0; + if (column === 'filename') { + cmp = a.filename.localeCompare(b.filename); + } else if (column === 'path') { + const aDir = getDirectoryPart(a.relative_path); + const bDir = getDirectoryPart(b.relative_path); + cmp = aDir.localeCompare(bDir); + } else if (column === 'mtime') { + cmp = (a.mtime || 0) - (b.mtime || 0); + } else if (column === 'size') { + cmp = a.size - b.size; + } else if (column === 'status') { + cmp = Number(a.already_uploaded) - Number(b.already_uploaded); } - return a.already_uploaded ? 1 : -1; + return ascending ? cmp : -cmp; }); - fileList.innerHTML = sorted - .map( - (file) => ` -
-
- - ${ - file.already_uploaded - ? '' - : '' - } - - + tbody.innerHTML = sorted + .map((file) => { + const dirPath = getDirectoryPart(file.relative_path); + + return ` + + + ${file.filename} -
-
- ${formatBytes(file.size)} + + + ${dirPath || '.'} + + ${formatMtime(file.mtime)} + ${formatBytes(file.size)} + ${file.already_uploaded ? 'Uploaded' : 'To Upload'} -
-
- `, - ) + + + `; + }) .join(''); + + // Update sort indicators + updateSortIndicators('[data-sort]', '.sort-indicator', 'sort', column, ascending); + + // Re-apply filter + applyScanFileFilter(); +} + +/** + * Get the directory part of a relative path (everything before the filename). + * @param {string} relativePath + * @returns {string} + */ +function getDirectoryPart(relativePath) { + const lastSlash = relativePath.lastIndexOf('/'); + return lastSlash >= 0 ? relativePath.substring(0, lastSlash) : ''; +} + +/** + * Sort the review table by column. + * @param {string} column + */ +function sortReviewTable(column) { + toggleSort(state.reviewSortConfig, column); + renderReviewTable(state.scanFileStatuses); } function applyScanFileFilter() { @@ -421,66 +422,203 @@ function applyScanFileFilter() { document.getElementById('scan-hide-uploaded') ); const hideUploaded = hideUploadedEl?.checked || false; - const fileList = document.getElementById('scan-file-list'); - if (!fileList) return; + const tbody = document.getElementById('scan-file-list'); + if (!tbody) return; - const items = fileList.querySelectorAll('[data-already-uploaded]'); - for (const item of items) { - if (hideUploaded && /** @type {HTMLElement} */ (item).dataset.alreadyUploaded === 'true') { - item.classList.add('hidden'); + const rows = tbody.querySelectorAll('[data-already-uploaded]'); + for (const row of rows) { + if (hideUploaded && /** @type {HTMLElement} */ (row).dataset.alreadyUploaded === 'true') { + row.classList.add('hidden'); } else { - item.classList.remove('hidden'); + row.classList.remove('hidden'); } } } /** - * Start direct upload from scanned folder. + * Show the confirm upload modal. */ -export async function startDirectUpload() { - if (!state.scanFilePaths || state.scanFilePaths.length === 0) { +export function showConfirmModal() { + if (!state.scanFileStatuses || state.scanFileStatuses.length === 0) { showNotification('No files to upload', 'error'); return; } - const startBtn = /** @type {HTMLButtonElement | null} */ ( - document.getElementById('start-upload-btn') + // Calculate size of non-uploaded files (default view) + updateConfirmModalCounts(false); + + // Wire up force-reupload checkbox to update counts dynamically + const checkbox = /** @type {HTMLInputElement | null} */ ( + document.getElementById('force-reupload-checkbox') ); - if (startBtn) { - startBtn.disabled = true; - startBtn.textContent = 'Validating...'; + if (checkbox) { + checkbox.checked = false; + checkbox.onchange = () => updateConfirmModalCounts(checkbox.checked); } - try { - const response = await fetch('/api/upload/bulk-analyze', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - file_paths: state.scanFilePaths, - auto_upload: false, - }), - }); - - const data = await response.json(); + showEl('confirm-skip-note'); + showEl('confirm-upload-modal'); +} - if (!response.ok && response.status !== 202) { - throw new Error(data.error || 'Failed to start upload'); +/** + * Update confirm modal file count/size based on force-reupload state. + * @param {boolean} forceReupload + */ +function updateConfirmModalCounts(forceReupload) { + let uploadSize = 0; + let uploadCount = 0; + for (const file of state.scanFileStatuses) { + if (forceReupload || !file.already_uploaded) { + uploadSize += file.size; + uploadCount++; } + } - state.currentJobId = data.job_id; + setText('confirm-file-count', uploadCount); + setText('confirm-total-size', formatBytes(uploadSize)); + + if (forceReupload) { + hideEl('confirm-skip-note'); + } else { + showEl('confirm-skip-note'); + } +} + +/** + * Start the combined validate + upload flow. + */ +export async function startCombinedUpload() { + // Close confirm modal + hideEl('confirm-upload-modal'); + + // Check force-reupload state + const forceCheckbox = /** @type {HTMLInputElement | null} */ ( + document.getElementById('force-reupload-checkbox') + ); + const forceReupload = forceCheckbox?.checked || false; + + // Determine which file paths to send + const filePaths = forceReupload + ? state.scanFileStatuses.map((/** @type {any} */ f) => f.path) + : state.scanFilePaths; + + if (!filePaths || filePaths.length === 0) { + showNotification('No files to upload', 'error'); + return; + } + + setUploadStep(3); + + hideEl('scan-results-section'); + showEl('upload-section'); + + // Set phase label + setText('upload-phase-label', 'Validating files...'); + + // Initialize file list with pending status + initUploadFileList(filePaths); + + try { + /** @type {Record} */ + const requestBody = { + file_paths: filePaths, + auto_upload: true, + }; + if (forceReupload) { + requestBody.skip_duplicates = false; + } - setUploadStep(3); + const data = await apiPost('/api/upload/bulk-analyze', requestBody); - document.getElementById('scan-results-section')?.classList.add('hidden'); - document.getElementById('analysis-section')?.classList.remove('hidden'); + state.currentJobId = data.job_id; + setText('files-total', data.total_files); - initializeAnalysisTableFromPaths(state.scanFilePaths); - connectAnalysisProgressStream(data.job_id); + connectCombinedProgressStream(data.job_id); } catch (error) { showNotification(/** @type {Error} */ (error).message, 'error'); - if (startBtn) { - startBtn.disabled = false; - startBtn.textContent = 'Validate Files'; + } +} + +/** + * Render the sortable file table in the folder browser. + */ +function renderBrowserFileTable() { + const tbody = document.getElementById('file-table-body'); + if (!tbody || !state.browserFiles) return; + + const { column, ascending } = state.browserFileSortConfig; + + const sorted = [...state.browserFiles].sort((a, b) => { + let cmp = 0; + if (column === 'name') { + cmp = a.name.localeCompare(b.name); + } else if (column === 'mtime') { + cmp = (a.mtime || 0) - (b.mtime || 0); + } else if (column === 'size') { + cmp = a.size - b.size; } + return ascending ? cmp : -cmp; + }); + + tbody.innerHTML = sorted + .map( + (/** @type {{ name: string, size: number, mtime: number }} */ file) => ` + + +
+ ${fileIcon('h-4 w-4 text-gray-400 mr-2 flex-shrink-0')} + ${file.name} +
+ + ${formatMtime(file.mtime)} + ${formatBytes(file.size)} + + `, + ) + .join(''); + + // Update sort indicators + updateSortIndicators('[data-file-sort]', '.file-sort-indicator', 'fileSort', column, ascending); +} + +/** + * Sort the browser file table by column. + * @param {string} column + */ +function sortBrowserFileTable(column) { + if (!state.browserFileSortConfig) { + state.browserFileSortConfig = { column, ascending: true }; + } else { + toggleSort(state.browserFileSortConfig, column); } + renderBrowserFileTable(); +} + +/** + * Initialize the upload file list with queued status. + * @param {string[]} filePaths + */ +function initUploadFileList(filePaths) { + const listEl = document.getElementById('upload-file-list'); + if (!listEl) return; + + const total = filePaths.length; + listEl.innerHTML = filePaths + .map((path, index) => { + const filename = path.split('/').pop() || path; + return ` +
+
+
+ + + +
+ ${filename} +
+ Queued (${index + 1} of ${total}) +
+ `; + }) + .join(''); } diff --git a/app/static/js/modules/utils.js b/app/static/js/modules/formatters.js similarity index 60% rename from app/static/js/modules/utils.js rename to app/static/js/modules/formatters.js index b16c98e..d3ceaa8 100644 --- a/app/static/js/modules/utils.js +++ b/app/static/js/modules/formatters.js @@ -1,5 +1,5 @@ /** - * Pure utility functions for formatting and notifications. + * Pure formatting functions for bytes, time, and dates. */ /** @@ -44,23 +44,17 @@ export function formatDuration(seconds) { } /** - * @param {string} message - * @param {'info' | 'error' | 'success'} [type] + * Format a Unix epoch (seconds) into a locale date string. + * @param {number | null | undefined} epochSeconds + * @returns {string} */ -export function showNotification(message, type = 'info') { - const notification = document.createElement('div'); - notification.className = `fixed top-4 right-4 px-6 py-3 rounded-md shadow-lg z-50 transition-opacity duration-300 ${ - type === 'error' - ? 'bg-red-500 text-white' - : type === 'success' - ? 'bg-green-500 text-white' - : 'bg-nrel-blue text-white' - }`; - notification.textContent = message; - document.body.appendChild(notification); - - setTimeout(() => { - notification.classList.add('opacity-0'); - setTimeout(() => notification.remove(), 300); - }, 5000); +export function formatMtime(epochSeconds) { + if (!epochSeconds) return '-'; + return new Date(epochSeconds * 1000).toLocaleString('en-US', { + month: 'short', + day: 'numeric', + year: 'numeric', + hour: '2-digit', + minute: '2-digit', + }); } diff --git a/app/static/js/modules/icons.js b/app/static/js/modules/icons.js new file mode 100644 index 0000000..d9bc569 --- /dev/null +++ b/app/static/js/modules/icons.js @@ -0,0 +1,26 @@ +/** + * Reusable SVG icon templates. + */ + +const FILE_ICON_PATH = + 'M9 12h6m-6 4h6m2 5H7a2 2 0 01-2-2V5a2 2 0 012-2h5.586a1 1 0 01.707.293l5.414 5.414a1 1 0 01.293.707V19a2 2 0 01-2 2z'; + +const FOLDER_ICON_PATH = 'M2 6a2 2 0 012-2h5l2 2h5a2 2 0 012 2v6a2 2 0 01-2 2H4a2 2 0 01-2-2V6z'; + +/** + * File document SVG icon. + * @param {string} [cls] - CSS classes (default: 'h-5 w-5 text-gray-400 mr-3') + * @returns {string} + */ +export function fileIcon(cls = 'h-5 w-5 text-gray-400 mr-3') { + return ``; +} + +/** + * Folder SVG icon. + * @param {string} [cls] - CSS classes (default: 'h-5 w-5 text-nlr-yellow mr-3') + * @returns {string} + */ +export function folderIcon(cls = 'h-5 w-5 text-nlr-yellow mr-3') { + return ``; +} diff --git a/app/static/js/modules/logs.js b/app/static/js/modules/logs.js new file mode 100644 index 0000000..1a65cfc --- /dev/null +++ b/app/static/js/modules/logs.js @@ -0,0 +1,425 @@ +/** + * Logs viewer page module. + * + * Loads log entries from /api/logs/entries with filtering, pagination, + * expandable detail rows, and S3 sync trigger. + */ +import { apiGet, apiPost } from './api.js'; +import { debounce } from './debounce.js'; +import { hideEl, setText, toggleEl, withLoadingButton } from './dom.js'; +import { showNotification } from './notify.js'; +import state from './state.js'; + +/** + * Format an ISO timestamp for display. + * @param {string} iso + * @returns {string} + */ +function formatTimestamp(iso) { + try { + const d = new Date(iso); + return d.toLocaleString(undefined, { + year: 'numeric', + month: '2-digit', + day: '2-digit', + hour: '2-digit', + minute: '2-digit', + second: '2-digit', + }); + } catch { + return iso; + } +} + +/** + * Return Tailwind classes for a log level badge. + * @param {string} level + * @returns {string} + */ +function levelBadgeClass(level) { + const upper = level.toUpperCase(); + if (upper === 'ERROR') return 'bg-red-100 text-red-800'; + if (upper === 'WARNING') return 'bg-yellow-100 text-yellow-800'; + return 'bg-blue-100 text-blue-800'; +} + +/** + * Get the filter DOM elements (shared by clearFilters and applyFilters). + */ +function getFilterEls() { + return { + date: /** @type {HTMLInputElement | null} */ (document.getElementById('filter-date')), + level: /** @type {HTMLSelectElement | null} */ (document.getElementById('filter-level')), + category: /** @type {HTMLSelectElement | null} */ (document.getElementById('filter-category')), + search: /** @type {HTMLInputElement | null} */ (document.getElementById('filter-search')), + }; +} + +/** + * Render the log table body from an array of entries. + * @param {Array>} entries + */ +function renderLogTable(entries) { + const tbody = document.getElementById('log-table-body'); + if (!tbody) return; + + if (entries.length === 0) { + tbody.innerHTML = + 'No log entries found'; + return; + } + + let html = ''; + for (const entry of entries) { + const ts = formatTimestamp(/** @type {string} */ (entry.timestamp) || ''); + const level = /** @type {string} */ (entry.level) || 'INFO'; + const category = /** @type {string} */ (entry.category) || ''; + const event = /** @type {string} */ (entry.event) || ''; + const message = /** @type {string} */ (entry.message) || ''; + const metadata = /** @type {Record | undefined} */ (entry.metadata); + const hasMetadata = metadata && Object.keys(metadata).length > 0; + + const rowClass = hasMetadata ? 'cursor-pointer hover:bg-gray-50' : ''; + const toggleAttr = hasMetadata ? 'data-log-toggle' : ''; + + html += ` + ${ts} + ${level} + ${category} + ${event} + ${message} + `; + + if (hasMetadata) { + const metaRows = Object.entries(metadata) + .map( + ([k, v]) => + `${k}${typeof v === 'object' ? JSON.stringify(v) : String(v)}`, + ) + .join(''); + + html += ` + + ${metaRows}
+ + `; + } + } + + tbody.innerHTML = html; + + // Wire toggle for expandable rows + for (const row of tbody.querySelectorAll('[data-log-toggle]')) { + row.addEventListener('click', () => { + const detail = row.nextElementSibling; + if (detail?.classList.contains('log-detail-row')) { + detail.classList.toggle('hidden'); + } + }); + } +} + +/** Fetch and render log entries using current filters + pagination. */ +export async function loadLogEntries() { + const params = new URLSearchParams(); + const { logFilters, logPagination } = state; + + if (logFilters.date) params.set('date', logFilters.date); + if (logFilters.level) params.set('level', logFilters.level); + if (logFilters.category) params.set('category', logFilters.category); + if (logFilters.search) params.set('search', logFilters.search); + params.set('offset', String(logPagination.offset)); + params.set('limit', String(logPagination.limit)); + + try { + const data = await apiGet(`/api/logs/entries?${params}`); + + renderLogTable(data.entries || []); + + // Update count label + const total = data.total || 0; + const from = total > 0 ? data.offset + 1 : 0; + const to = Math.min(data.offset + data.limit, total); + setText('log-count-label', `Showing ${from}-${to} of ${total}`); + + // Pagination buttons + const prevBtn = /** @type {HTMLButtonElement | null} */ ( + document.getElementById('prev-page-btn') + ); + const nextBtn = /** @type {HTMLButtonElement | null} */ ( + document.getElementById('next-page-btn') + ); + if (prevBtn) prevBtn.disabled = data.offset <= 0; + if (nextBtn) nextBtn.disabled = data.offset + data.limit >= total; + } catch { + setText('log-count-label', 'Failed to load log entries'); + } +} + +/** Fetch and render log stats into the stat cards. */ +export async function loadLogStats() { + try { + const data = await apiGet('/api/logs/stats'); + + setText('stat-total', data.total_entries ?? 0); + setText('stat-today', data.today_entries ?? 0); + setText('stat-errors', data.level_counts?.ERROR ?? 0); + setText('stat-files', data.file_count ?? 0); + } catch { + // Silently fail — stats are non-critical + } +} + +/** POST /api/logs/sync and show notification. */ +export async function syncLogs() { + const btn = /** @type {HTMLButtonElement | null} */ (document.getElementById('sync-logs-btn')); + + await withLoadingButton(btn, 'Syncing...', async () => { + try { + const data = await apiPost('/api/logs/sync'); + + if (data.success) { + showNotification(`Synced ${data.synced} log files to S3`, 'success'); + } else { + showNotification(`Sync failed: ${data.error || 'Unknown error'}`, 'error'); + } + } catch (err) { + showNotification(`Sync error: ${err}`, 'error'); + } + }); +} + +/** Track which CSV is currently previewed (to toggle on re-click). */ +let currentPreviewPath = ''; + +/** + * Extract time (HH:MM:SS) from a CSV filename like upload-summary-143022-abcd1234.csv. + * @param {string} filename + * @returns {string} + */ +function extractTimeFromFilename(filename) { + const match = filename.match(/upload-summary-(\d{2})(\d{2})(\d{2})/); + if (match) return `${match[1]}:${match[2]}:${match[3]}`; + return '-'; +} + +/** Fetch CSV file list and render the Upload Summaries table. */ +export async function loadCsvFiles() { + try { + const data = await apiGet('/api/logs/files'); + /** @type {Array<{date: string|null, filename: string, relative_path: string, size_bytes: number, type: string}>} */ + const csvFiles = (data.files || []).filter( + (/** @type {{type: string}} */ f) => f.type === 'csv', + ); + + // Update badge + setText('csv-count-badge', csvFiles.length); + toggleEl('csv-count-badge', csvFiles.length > 0); + + const tbody = document.getElementById('csv-table-body'); + if (!tbody) return; + + if (csvFiles.length === 0) { + tbody.innerHTML = + 'No upload summaries found'; + return; + } + + let html = ''; + for (const file of csvFiles) { + const date = file.date || '-'; + const time = extractTimeFromFilename(file.filename); + const escapedPath = file.relative_path.replace(/"/g, '"'); + + html += ` + ${date} + ${time} + ${file.filename} + + + + + `; + } + tbody.innerHTML = html; + + // Wire download buttons + for (const btn of tbody.querySelectorAll('.csv-download-btn')) { + btn.addEventListener('click', () => { + const path = /** @type {HTMLElement} */ (btn).dataset.path || ''; + downloadCsv(path); + }); + } + + // Wire view buttons + for (const btn of tbody.querySelectorAll('.csv-view-btn')) { + btn.addEventListener('click', () => { + const path = /** @type {HTMLElement} */ (btn).dataset.path || ''; + previewCsv(path); + }); + } + } catch { + // Non-critical — silently fail + } +} + +/** + * Trigger a browser download for a CSV file. + * @param {string} path - Relative path within the log directory + */ +function downloadCsv(path) { + window.location.href = `/api/logs/csv-download?path=${encodeURIComponent(path)}`; +} + +/** + * Fetch and render a CSV preview inline. Toggle on re-click. + * @param {string} path - Relative path within the log directory + */ +async function previewCsv(path) { + const panel = document.getElementById('csv-preview-panel'); + const content = document.getElementById('csv-preview-content'); + if (!panel || !content) return; + + // Toggle off if clicking the same file + if (currentPreviewPath === path && !panel.classList.contains('hidden')) { + hideEl('csv-preview-panel'); + currentPreviewPath = ''; + return; + } + + currentPreviewPath = path; + content.innerHTML = '

Loading...

'; + panel.classList.remove('hidden'); + + const filename = path.split('/').pop() || path; + setText('csv-preview-title', `Preview: ${filename}`); + + try { + const data = await apiGet(`/api/logs/csv-preview?path=${encodeURIComponent(path)}`); + + if (data.error) { + content.innerHTML = `

${data.error}

`; + return; + } + + const columns = /** @type {string[]} */ (data.columns || []); + const rows = /** @type {Array>} */ (data.rows || []); + + if (columns.length === 0) { + content.innerHTML = '

Empty CSV

'; + return; + } + + let tableHtml = ''; + tableHtml += ''; + for (const col of columns) { + tableHtml += ``; + } + tableHtml += ''; + for (const row of rows) { + tableHtml += ''; + for (const col of columns) { + const val = row[col] ?? ''; + tableHtml += ``; + } + tableHtml += ''; + } + tableHtml += '
${col}
${val}
'; + content.innerHTML = tableHtml; + } catch { + content.innerHTML = '

Failed to load CSV preview

'; + } +} + +/** Reset all filters and reload. */ +export function clearFilters() { + state.logFilters = { date: null, level: null, category: null, search: '' }; + state.logPagination = { offset: 0, limit: 100 }; + + const { date, level, category, search } = getFilterEls(); + if (date) date.value = ''; + if (level) level.value = ''; + if (category) category.value = ''; + if (search) search.value = ''; + + loadLogEntries(); + loadLogStats(); +} + +/** Read current filter values from the DOM into state and reload. */ +function applyFilters() { + const { date, level, category, search } = getFilterEls(); + + state.logFilters.date = date?.value || null; + state.logFilters.level = level?.value || null; + state.logFilters.category = category?.value || null; + state.logFilters.search = search?.value || ''; + state.logPagination.offset = 0; + + loadLogEntries(); +} + +/** Initialize the logs page: load data and wire event handlers. */ +export function initLogs() { + // Reset state + state.logFilters = { date: null, level: null, category: null, search: '' }; + state.logPagination = { offset: 0, limit: 100 }; + + // Initial load + loadLogEntries(); + loadLogStats(); + loadCsvFiles(); + + // CSV section toggle + const csvToggle = document.getElementById('csv-section-toggle'); + const csvBody = document.getElementById('csv-section-body'); + const csvIcon = document.getElementById('csv-toggle-icon'); + if (csvToggle && csvBody) { + csvToggle.addEventListener('click', () => { + csvBody.classList.toggle('hidden'); + csvIcon?.classList.toggle('rotate-90'); + }); + } + + // CSV preview close button + document.getElementById('csv-preview-close')?.addEventListener('click', () => { + hideEl('csv-preview-panel'); + currentPreviewPath = ''; + }); + + // Filter controls + const { date: dateEl, level: levelEl, category: catEl, search: searchEl } = getFilterEls(); + const clearBtn = document.getElementById('clear-filters-btn'); + const syncBtn = document.getElementById('sync-logs-btn'); + const prevBtn = document.getElementById('prev-page-btn'); + const nextBtn = document.getElementById('next-page-btn'); + + if (dateEl) dateEl.addEventListener('change', applyFilters); + if (levelEl) levelEl.addEventListener('change', applyFilters); + if (catEl) catEl.addEventListener('change', applyFilters); + + // Debounced search + if (searchEl) { + searchEl.addEventListener('input', debounce(applyFilters, 300)); + } + + if (clearBtn) clearBtn.addEventListener('click', clearFilters); + if (syncBtn) syncBtn.addEventListener('click', syncLogs); + + if (prevBtn) { + prevBtn.addEventListener('click', () => { + state.logPagination.offset = Math.max( + 0, + state.logPagination.offset - state.logPagination.limit, + ); + loadLogEntries(); + }); + } + + if (nextBtn) { + nextBtn.addEventListener('click', () => { + state.logPagination.offset += state.logPagination.limit; + loadLogEntries(); + }); + } +} diff --git a/app/static/js/modules/notify.js b/app/static/js/modules/notify.js new file mode 100644 index 0000000..e121a5b --- /dev/null +++ b/app/static/js/modules/notify.js @@ -0,0 +1,25 @@ +/** + * Toast notification system. + */ + +/** + * @param {string} message + * @param {'info' | 'error' | 'success'} [type] + */ +export function showNotification(message, type = 'info') { + const notification = document.createElement('div'); + notification.className = `fixed top-4 right-4 px-6 py-3 rounded-md shadow-lg z-50 transition-opacity duration-300 ${ + type === 'error' + ? 'bg-red-500 text-white' + : type === 'success' + ? 'bg-green-500 text-white' + : 'bg-nlr-blue text-white' + }`; + notification.textContent = message; + document.body.appendChild(notification); + + setTimeout(() => { + notification.classList.add('opacity-0'); + setTimeout(() => notification.remove(), 300); + }, 5000); +} diff --git a/app/static/js/modules/settings.js b/app/static/js/modules/settings.js index c26ec8b..06c3ea5 100644 --- a/app/static/js/modules/settings.js +++ b/app/static/js/modules/settings.js @@ -1,8 +1,10 @@ -import state from './state.js'; +import { apiGet, apiPost, apiPut } from './api.js'; +import { appendText, setText, showEl, withLoadingButton } from './dom.js'; +import { showNotification } from './notify.js'; /** * Settings page functionality. */ -import { showNotification } from './utils.js'; +import state from './state.js'; const AWS_REGION_OPTIONS = new Set([ 'us-west-2', @@ -90,8 +92,7 @@ export function initSettings() { async function loadCurrentSettings() { try { - const response = await fetch('/api/settings'); - const settings = await response.json(); + const settings = await apiGet('/api/settings'); setAwsRegion(settings.aws_region || 'us-west-2'); @@ -111,8 +112,7 @@ async function loadCurrentSettings() { async function loadAwsProfiles() { try { - const response = await fetch('/api/settings/profiles'); - const data = await response.json(); + const data = await apiGet('/api/settings/profiles'); const select = /** @type {HTMLSelectElement | null} */ (document.getElementById('aws-profile')); if (!select) return; @@ -131,29 +131,18 @@ async function loadAwsProfiles() { async function loadVersionInfo() { try { - const response = await fetch('/api/settings/version'); - const data = await response.json(); - - const gitBranch = document.getElementById('git-branch'); - if (gitBranch) gitBranch.textContent = data.branch || '-'; - - const gitCommit = document.getElementById('git-commit'); - if (gitCommit) gitCommit.textContent = data.commit || '-'; - - const gitDate = document.getElementById('git-date'); - if (gitDate) gitDate.textContent = data.last_updated || '-'; + const data = await apiGet('/api/settings/version'); - const pkgVersion = document.getElementById('pkg-version'); - if (pkgVersion) pkgVersion.textContent = data.version || '-'; + setText('git-branch', data.branch || '-'); + setText('git-commit', data.commit || '-'); + setText('git-date', data.last_updated || '-'); + setText('pkg-version', data.version || '-'); - const versionInfo = document.getElementById('version-info'); - if (versionInfo) { - let versionText = data.version || '0.0.0'; - if (data.commit) { - versionText += ` (${data.commit})`; - } - versionInfo.textContent = versionText; + let versionText = data.version || '0.0.0'; + if (data.commit) { + versionText += ` (${data.commit})`; } + setText('version-info', versionText); } catch (error) { console.error('Failed to load version info:', error); } @@ -170,17 +159,7 @@ async function saveSettings() { }; try { - const response = await fetch('/api/settings', { - method: 'PUT', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(settings), - }); - - if (!response.ok) { - const data = await response.json(); - throw new Error(data.error || 'Failed to save settings'); - } - + await apiPut('/api/settings', settings); showNotification('Settings saved successfully', 'success'); } catch (error) { showNotification(/** @type {Error} */ (error).message, 'error'); @@ -194,8 +173,6 @@ async function testConnection() { const status = document.getElementById('connection-status'); if (!btn || !status) return; - btn.disabled = true; - btn.textContent = 'Testing...'; status.textContent = 'Testing connection...'; status.className = 'mt-1 text-sm text-gray-500'; @@ -205,29 +182,16 @@ async function testConnection() { s3_bucket: /** @type {HTMLInputElement} */ (document.getElementById('s3-bucket'))?.value, }; - try { - const response = await fetch('/api/settings/validate', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(settings), - }); - - const data = await response.json(); - - if (data.success) { - status.textContent = data.message; - status.className = 'mt-1 text-sm text-green-600'; - } else { - status.textContent = data.error; + await withLoadingButton(btn, 'Testing...', async () => { + try { + const data = await apiPost('/api/settings/validate', settings); + status.textContent = data.success ? data.message : data.error; + status.className = `mt-1 text-sm ${data.success ? 'text-green-600' : 'text-red-600'}`; + } catch (error) { + status.textContent = /** @type {Error} */ (error).message; status.className = 'mt-1 text-sm text-red-600'; } - } catch (error) { - status.textContent = /** @type {Error} */ (error).message; - status.className = 'mt-1 text-sm text-red-600'; - } - - btn.disabled = false; - btn.textContent = 'Test Connection'; + }); } async function checkForUpdates() { @@ -240,94 +204,82 @@ async function checkForUpdates() { ); if (!btn || !status) return; - btn.disabled = true; - btn.textContent = 'Checking...'; - - try { - const response = await fetch('/api/settings/check-updates'); - const data = await response.json(); - - if (data.error) { - status.textContent = `Error: ${data.error}`; + await withLoadingButton(btn, 'Checking...', async () => { + try { + const data = await apiGet('/api/settings/check-updates'); + + if (data.error) { + status.textContent = `Error: ${data.error}`; + status.className = 'text-sm text-red-600'; + } else if (data.updates_available) { + status.textContent = 'Updates available! Click "Update Application" to install.'; + status.className = 'text-sm text-nlr-yellow font-medium'; + if (updateBtn) updateBtn.disabled = false; + } else if (data.up_to_date) { + status.textContent = 'Application is up to date.'; + status.className = 'text-sm text-green-600'; + } else { + status.textContent = 'Could not determine update status.'; + status.className = 'text-sm text-gray-600'; + } + } catch (error) { + status.textContent = /** @type {Error} */ (error).message; status.className = 'text-sm text-red-600'; - } else if (data.updates_available) { - status.textContent = 'Updates available! Click "Update Application" to install.'; - status.className = 'text-sm text-nrel-yellow font-medium'; - if (updateBtn) updateBtn.disabled = false; - } else if (data.up_to_date) { - status.textContent = 'Application is up to date.'; - status.className = 'text-sm text-green-600'; - } else { - status.textContent = 'Could not determine update status.'; - status.className = 'text-sm text-gray-600'; } - } catch (error) { - status.textContent = /** @type {Error} */ (error).message; - status.className = 'text-sm text-red-600'; - } - - btn.disabled = false; - btn.textContent = 'Check for Updates'; + }); } async function runUpdate() { const btn = /** @type {HTMLButtonElement | null} */ (document.getElementById('run-update-btn')); const status = document.getElementById('update-status'); - const logSection = document.getElementById('update-log'); - const logOutput = document.getElementById('update-output'); - if (!btn || !status || !logSection || !logOutput) return; + if (!btn || !status) return; - btn.disabled = true; - btn.textContent = 'Updating...'; status.textContent = 'Running update...'; + showEl('update-log'); + setText('update-output', 'Starting update...\n'); - logSection.classList.remove('hidden'); - logOutput.textContent = 'Starting update...\n'; + await withLoadingButton(btn, 'Updating...', async () => { + try { + const data = await apiPost('/api/settings/update'); - try { - const response = await fetch('/api/settings/update', { method: 'POST' }); - const data = await response.json(); + let output = ''; - let output = ''; + if (data.results.git_pull) { + output += '=== Git Pull ===\n'; + output += data.results.git_pull.success ? '[SUCCESS]\n' : '[FAILED]\n'; + output += `${data.results.git_pull.output}\n\n`; + } - if (data.results.git_pull) { - output += '=== Git Pull ===\n'; - output += data.results.git_pull.success ? '[SUCCESS]\n' : '[FAILED]\n'; - output += `${data.results.git_pull.output}\n\n`; - } + if (data.results.pip_install) { + output += '=== Pip Install ===\n'; + output += data.results.pip_install.success ? '[SUCCESS]\n' : '[FAILED]\n'; + output += `${data.results.pip_install.output}\n\n`; + } - if (data.results.pip_install) { - output += '=== Pip Install ===\n'; - output += data.results.pip_install.success ? '[SUCCESS]\n' : '[FAILED]\n'; - output += `${data.results.pip_install.output}\n\n`; - } + if (data.results.modaq_toolkit) { + output += '=== MODAQ Toolkit Update ===\n'; + output += data.results.modaq_toolkit.success ? '[SUCCESS]\n' : '[FAILED]\n'; + output += `${data.results.modaq_toolkit.output}\n`; + } - if (data.results.modaq_toolkit) { - output += '=== MODAQ Toolkit Update ===\n'; - output += data.results.modaq_toolkit.success ? '[SUCCESS]\n' : '[FAILED]\n'; - output += `${data.results.modaq_toolkit.output}\n`; - } + setText('update-output', output); - logOutput.textContent = output; + if (data.success) { + status.textContent = 'Update completed! Restart the application to apply changes.'; + status.className = 'text-sm text-green-600 font-medium'; + showNotification('Update completed! Please restart the application.', 'success'); + } else { + status.textContent = 'Update completed with some errors. Check the log below.'; + status.className = 'text-sm text-yellow-600'; + } - if (data.success) { - status.textContent = 'Update completed! Restart the application to apply changes.'; - status.className = 'text-sm text-green-600 font-medium'; - showNotification('Update completed! Please restart the application.', 'success'); - } else { - status.textContent = 'Update completed with some errors. Check the log below.'; - status.className = 'text-sm text-yellow-600'; + loadVersionInfo(); + } catch (error) { + status.textContent = /** @type {Error} */ (error).message; + status.className = 'text-sm text-red-600'; + appendText('update-output', `\nError: ${/** @type {Error} */ (error).message}`); } - - loadVersionInfo(); - } catch (error) { - status.textContent = /** @type {Error} */ (error).message; - status.className = 'text-sm text-red-600'; - logOutput.textContent += `\nError: ${/** @type {Error} */ (error).message}`; - } - - btn.disabled = false; - btn.textContent = 'Update Application'; + }); } async function resetSettings() { @@ -336,21 +288,13 @@ async function resetSettings() { } try { - const response = await fetch('/api/settings', { - method: 'PUT', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - aws_profile: 'default', - aws_region: 'us-west-2', - s3_bucket: '', - default_upload_folder: '', - }), + await apiPut('/api/settings', { + aws_profile: 'default', + aws_region: 'us-west-2', + s3_bucket: '', + default_upload_folder: '', }); - if (!response.ok) { - throw new Error('Failed to reset settings'); - } - loadCurrentSettings(); showNotification('Settings reset to defaults', 'success'); } catch (error) { @@ -377,29 +321,19 @@ function clearBrowserCache() { async function loadCacheStats() { try { - const response = await fetch('/api/settings/cache/stats'); - const data = await response.json(); + const data = await apiGet('/api/settings/cache/stats'); if (data.success && data.stats) { const stats = data.stats; - const cacheTotal = document.getElementById('cache-total'); - if (cacheTotal) cacheTotal.textContent = String(stats.total_entries || 0); - - const cacheExists = document.getElementById('cache-exists'); - if (cacheExists) cacheExists.textContent = String(stats.exists_count || 0); + setText('cache-total', stats.total_entries || 0); + setText('cache-exists', stats.exists_count || 0); + setText('cache-deleted', stats.not_exists_count || 0); - const cacheDeleted = document.getElementById('cache-deleted'); - if (cacheDeleted) cacheDeleted.textContent = String(stats.not_exists_count || 0); - - const cacheLastSync = document.getElementById('cache-last-sync'); - if (cacheLastSync) { - if (stats.last_full_sync) { - const syncDate = new Date(stats.last_full_sync); - cacheLastSync.textContent = syncDate.toLocaleString(); - } else { - cacheLastSync.textContent = 'Never'; - } + if (stats.last_full_sync) { + setText('cache-last-sync', new Date(stats.last_full_sync).toLocaleString()); + } else { + setText('cache-last-sync', 'Never'); } } } catch (error) { @@ -412,34 +346,30 @@ async function syncCacheWithAws() { const status = document.getElementById('sync-status'); if (!btn || !status) return; - btn.disabled = true; - btn.textContent = 'Syncing...'; status.classList.remove('hidden'); status.textContent = 'Fetching file list from S3...'; status.className = 'mt-2 text-sm text-gray-600'; - try { - const response = await fetch('/api/settings/cache/sync', { method: 'POST' }); - const data = await response.json(); - - if (data.success) { - status.textContent = data.message; - status.className = 'mt-2 text-sm text-green-600'; - showNotification(data.message, 'success'); - loadCacheStats(); - } else { - status.textContent = `Error: ${data.error}`; + await withLoadingButton(btn, 'Syncing...', async () => { + try { + const data = await apiPost('/api/settings/cache/sync'); + + if (data.success) { + status.textContent = data.message; + status.className = 'mt-2 text-sm text-green-600'; + showNotification(data.message, 'success'); + loadCacheStats(); + } else { + status.textContent = `Error: ${data.error}`; + status.className = 'mt-2 text-sm text-red-600'; + showNotification(data.error, 'error'); + } + } catch (error) { + status.textContent = `Error: ${/** @type {Error} */ (error).message}`; status.className = 'mt-2 text-sm text-red-600'; - showNotification(data.error, 'error'); + showNotification(/** @type {Error} */ (error).message, 'error'); } - } catch (error) { - status.textContent = `Error: ${/** @type {Error} */ (error).message}`; - status.className = 'mt-2 text-sm text-red-600'; - showNotification(/** @type {Error} */ (error).message, 'error'); - } - - btn.disabled = false; - btn.textContent = 'Sync Cache with AWS'; + }); } async function invalidateUploadCache() { @@ -454,25 +384,19 @@ async function invalidateUploadCache() { const btn = /** @type {HTMLButtonElement | null} */ ( document.getElementById('invalidate-cache-btn') ); - if (!btn) return; - - btn.disabled = true; - btn.textContent = 'Clearing...'; - try { - const response = await fetch('/api/settings/cache/invalidate', { method: 'POST' }); - const data = await response.json(); + await withLoadingButton(btn, 'Clearing...', async () => { + try { + const data = await apiPost('/api/settings/cache/invalidate'); - if (data.success) { - showNotification(data.message, 'success'); - loadCacheStats(); - } else { - showNotification(data.error || 'Failed to clear cache', 'error'); + if (data.success) { + showNotification(data.message, 'success'); + loadCacheStats(); + } else { + showNotification(data.error || 'Failed to clear cache', 'error'); + } + } catch (error) { + showNotification(/** @type {Error} */ (error).message, 'error'); } - } catch (error) { - showNotification(/** @type {Error} */ (error).message, 'error'); - } - - btn.disabled = false; - btn.textContent = 'Clear Upload Cache'; + }); } diff --git a/app/static/js/modules/sorting-helpers.js b/app/static/js/modules/sorting-helpers.js new file mode 100644 index 0000000..4ce1d70 --- /dev/null +++ b/app/static/js/modules/sorting-helpers.js @@ -0,0 +1,40 @@ +/** + * Table sorting helpers: toggle column direction and update header indicators. + */ + +/** + * Toggle sort direction on a column config object. + * @param {{ column: string, ascending: boolean }} config + * @param {string} column + */ +export function toggleSort(config, column) { + if (config.column === column) { + config.ascending = !config.ascending; + } else { + config.column = column; + config.ascending = true; + } +} + +/** + * Update sort indicator arrows in table headers. + * @param {string} headerSelector - e.g. `'[data-sort]'` + * @param {string} indicatorSelector - e.g. `'.sort-indicator'` + * @param {string} dataKey - dataset key, e.g. `'sort'` or `'fileSort'` + * @param {string} currentColumn + * @param {boolean} ascending + */ +export function updateSortIndicators( + headerSelector, + indicatorSelector, + dataKey, + currentColumn, + ascending, +) { + for (const th of document.querySelectorAll(headerSelector)) { + const indicator = th.querySelector(indicatorSelector); + if (!indicator) continue; + const col = /** @type {HTMLElement} */ (th).dataset[dataKey]; + indicator.textContent = col === currentColumn ? (ascending ? ' \u25B2' : ' \u25BC') : ''; + } +} diff --git a/app/static/js/modules/state.js b/app/static/js/modules/state.js index 888cc08..d2e9107 100644 --- a/app/static/js/modules/state.js +++ b/app/static/js/modules/state.js @@ -12,11 +12,11 @@ const state = { /** @type {string | null} */ selectedFolderPath: null, - /** Current upload step (1-5) */ + /** Current upload step (1-4) */ currentStep: 1, /** S3 file browser current prefix */ - currentPrefix: "", + currentPrefix: '', /** @type {{ version?: string, commit?: string, branch?: string } | null} */ appVersionData: null, @@ -27,8 +27,34 @@ const state = { /** @type {string[]} */ scanFilePaths: [], - /** @type {Array<{ path: string, filename: string, size: number, already_uploaded: boolean }>} */ + /** @type {Array<{ path: string, filename: string, size: number, mtime?: number, already_uploaded: boolean }>} */ scanFileStatuses: [], + + /** Total size of all scanned files in bytes */ + scanTotalSize: 0, + + /** @type {string | null} */ + scanFolderPath: null, + + /** Sort configuration for the review table */ + reviewSortConfig: { column: 'filename', ascending: true }, + + /** @type {Array<{ name: string, size: number, mtime: number }>} */ + browserFiles: [], + + /** Sort configuration for the folder browser file table */ + browserFileSortConfig: { column: 'name', ascending: true }, + + /** Log viewer filter state */ + logFilters: { + /** @type {string | null} */ date: null, + /** @type {string | null} */ level: null, + /** @type {string | null} */ category: null, + /** @type {string} */ search: '', + }, + + /** Log viewer pagination state */ + logPagination: { offset: 0, limit: 100 }, }; export default state; diff --git a/app/static/js/modules/stepper.js b/app/static/js/modules/stepper.js index 93b9b1b..bce8cdd 100644 --- a/app/static/js/modules/stepper.js +++ b/app/static/js/modules/stepper.js @@ -1,24 +1,20 @@ /** * Upload step indicator management. */ +import { setText } from './dom.js'; +import { showNotification } from './notify.js'; import state from './state.js'; -import { showNotification } from './utils.js'; export const UPLOAD_STEPS = { 1: { name: 'Select', description: 'Select files or a folder to upload' }, 2: { name: 'Review', - description: - 'Review files found - click Validate to continue, or Back to select different files', + description: 'Review files found - click Continue to upload, or Back to select different files', }, - 3: { name: 'Validate', description: 'Extracting timestamps and checking for duplicates...' }, - 4: { name: 'Upload', description: 'Uploading files to S3...' }, - 5: { name: 'Complete', description: 'Upload complete!' }, + 3: { name: 'Upload', description: 'Validating and uploading files...' }, + 4: { name: 'Complete', description: 'Upload complete!' }, }; -export const STEP_3_VALIDATING = 'Extracting timestamps and checking for duplicates...'; -export const STEP_3_VALIDATED = 'Validation complete - review results and click Upload Files'; - /** * Set the current step in the upload flow. * @param {number} step @@ -28,7 +24,7 @@ export function setUploadStep(step) { const stepsContainer = document.getElementById('upload-steps'); if (!stepsContainer) return; - for (let i = 1; i <= 5; i++) { + for (let i = 1; i <= 4; i++) { const stepEl = stepsContainer.querySelector(`[data-step="${i}"]`); if (!stepEl) continue; @@ -47,9 +43,8 @@ export function setUploadStep(step) { index < step - 1 ? '#5D9732' : '#D1D5DB'; }); - const descriptionEl = document.getElementById('step-description'); - if (descriptionEl && UPLOAD_STEPS[step]) { - descriptionEl.textContent = UPLOAD_STEPS[step].description; + if (UPLOAD_STEPS[step]) { + setText('step-description', UPLOAD_STEPS[step].description); } } @@ -76,20 +71,13 @@ export function hideUploadSteps() { export async function goToStep(targetStep) { if (targetStep >= state.currentStep) return; - if (targetStep === 1 && state.currentStep <= 3) { - const { resetUpload } = await import('./upload-control.js'); - resetUpload(); - return; - } - - if (targetStep === 2 && state.currentStep === 3) { + if (targetStep === 1 && state.currentStep <= 2) { const { resetUpload } = await import('./upload-control.js'); - showNotification('Going back to file selection', 'info'); resetUpload(); return; } - if (state.currentStep >= 4) { + if (state.currentStep >= 3) { showNotification('Cannot go back during or after upload', 'error'); } } diff --git a/app/static/js/modules/upload-control.js b/app/static/js/modules/upload-control.js index 42c3cef..62c107e 100644 --- a/app/static/js/modules/upload-control.js +++ b/app/static/js/modules/upload-control.js @@ -1,19 +1,18 @@ +import { resetAnalysisState } from './analysis.js'; /** * Cancel and reset operations for uploads. */ +import { apiPost } from './api.js'; +import { hideEl, showEl } from './dom.js'; +import { showNotification } from './notify.js'; import state from './state.js'; import { hideUploadSteps } from './stepper.js'; -import { showNotification } from './utils.js'; - -export function cancelAnalysis() { - resetUpload(); -} export async function cancelUpload() { if (!state.currentJobId) return; try { - await fetch(`/api/upload/cancel/${state.currentJobId}`, { method: 'POST' }); + await apiPost(`/api/upload/cancel/${state.currentJobId}`); if (state.eventSource) state.eventSource.close(); showNotification('Upload cancelled', 'info'); } catch (_error) { @@ -28,17 +27,18 @@ export function resetUpload() { state.eventSource = null; } + resetAnalysisState(); hideUploadSteps(); - document.getElementById('drop-zone')?.classList.remove('hidden'); - document.getElementById('analysis-section')?.classList.add('hidden'); - document.getElementById('progress-section')?.classList.add('hidden'); - document.getElementById('completion-section')?.classList.add('hidden'); - document.getElementById('scan-results-section')?.classList.add('hidden'); - - const fileInput = /** @type {HTMLInputElement | null} */ (document.getElementById('file-input')); - if (fileInput) fileInput.value = ''; + showEl('folder-browser-panel'); + hideEl('upload-section'); + hideEl('completion-section'); + hideEl('scan-results-section'); + hideEl('confirm-upload-modal'); state.selectedFolderPath = null; state.scanFilePaths = []; + state.scanFileStatuses = []; + state.scanTotalSize = 0; + state.scanFolderPath = null; } diff --git a/app/static/js/modules/upload-exec.js b/app/static/js/modules/upload-exec.js index dce318b..a164da9 100644 --- a/app/static/js/modules/upload-exec.js +++ b/app/static/js/modules/upload-exec.js @@ -1,198 +1,69 @@ /** - * Upload execution: start upload, SSE progress tracking, completion summary. + * Upload execution: progress tracking and completion summary. */ -import state from './state.js'; +import { hideEl, setText, showEl } from './dom.js'; +import { formatDuration, formatEta } from './formatters.js'; +import { fileIcon } from './icons.js'; +import { showNotification } from './notify.js'; import { setUploadStep } from './stepper.js'; -import { formatDuration, formatEta, showNotification } from './utils.js'; -export async function startUpload() { - if (!state.currentJobId) return; - - const skipDuplicatesEl = /** @type {HTMLInputElement | null} */ ( - document.getElementById('skip-duplicates') - ); - const skipDuplicates = skipDuplicatesEl?.checked ?? true; - - setUploadStep(4); - - document.getElementById('analysis-section')?.classList.add('hidden'); - document.getElementById('progress-section')?.classList.remove('hidden'); - - try { - const response = await fetch(`/api/upload/start/${state.currentJobId}`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ skip_duplicates: skipDuplicates }), - }); - - if (!response.ok) { - const data = await response.json(); - throw new Error(data.error || 'Failed to start upload'); - } - - connectProgressStream(); - } catch (error) { - showNotification(/** @type {Error} */ (error).message, 'error'); - } -} - -export function connectProgressStream() { - if (state.eventSource) { - state.eventSource.close(); - } - - state.eventSource = new EventSource(`/api/upload/progress/${state.currentJobId}`); - - state.eventSource.onmessage = (event) => { - const data = JSON.parse(event.data); - - if (data.error) { - showNotification(data.error, 'error'); - state.eventSource?.close(); - return; - } - - updateProgressUI(data); - - if (['completed', 'failed', 'cancelled'].includes(data.status)) { - state.eventSource?.close(); - showCompletionSummary(data); - } - }; - - state.eventSource.onerror = () => { - state.eventSource?.close(); - showNotification('Connection to server lost', 'error'); - }; -} +/** @type {any} */ +let lastCompletedJob = null; /** * @param {any} job */ export function updateProgressUI(job) { - const el = (/** @type {string} */ id) => document.getElementById(id); - - const progressPercent = el('progress-percent'); - if (progressPercent) progressPercent.textContent = job.progress_percent.toFixed(1); - - const progressBar = /** @type {HTMLElement | null} */ (el('progress-bar')); + setText('progress-percent', job.progress_percent.toFixed(1)); + setText('files-completed', job.files_completed); + setText('files-total', job.total_files); + setText('bytes-uploaded', job.uploaded_bytes_formatted); + setText('bytes-total', job.total_bytes_formatted); + setText('eta', formatEta(job.eta_seconds)); + + const progressBar = /** @type {HTMLElement | null} */ (document.getElementById('progress-bar')); if (progressBar) progressBar.style.width = `${job.progress_percent}%`; - - const filesCompleted = el('files-completed'); - if (filesCompleted) filesCompleted.textContent = job.files_completed; - - const filesTotal = el('files-total'); - if (filesTotal) filesTotal.textContent = job.total_files; - - const bytesUploaded = el('bytes-uploaded'); - if (bytesUploaded) bytesUploaded.textContent = job.uploaded_bytes_formatted; - - const bytesTotal = el('bytes-total'); - if (bytesTotal) bytesTotal.textContent = job.total_bytes_formatted; - - const eta = el('eta'); - if (eta) eta.textContent = formatEta(job.eta_seconds); - - const progressList = el('progress-list'); - if (progressList) { - progressList.innerHTML = job.files - .map((/** @type {any} */ file) => { - const statusIcon = - file.status === 'completed' - ? '' - : file.status === 'failed' - ? '' - : file.status === 'skipped' - ? '' - : file.status === 'uploading' - ? '' - : '
'; - - return ` -
-
- ${statusIcon} - ${file.filename} -
-
- ${file.file_size_formatted} - ${ - file.status === 'uploading' - ? ` -
-
-
- ${file.progress_percent.toFixed(0)}% - ` - : '' - } -
-
- `; - }) - .join(''); - } } /** * @param {any} job */ export function showCompletionSummary(job) { - setUploadStep(5); + lastCompletedJob = job; + setUploadStep(4); - document.getElementById('progress-section')?.classList.add('hidden'); - document.getElementById('completion-section')?.classList.remove('hidden'); + hideEl('upload-section'); + showEl('completion-section'); const completed = job.files.filter((/** @type {any} */ f) => f.status === 'completed').length; const skipped = job.files.filter((/** @type {any} */ f) => f.status === 'skipped').length; const failed = job.files.filter((/** @type {any} */ f) => f.status === 'failed').length; - const el = (/** @type {string} */ id) => document.getElementById(id); - - const completedCount = el('completed-count'); - if (completedCount) completedCount.textContent = String(completed); - - const skippedCount = el('skipped-count'); - if (skippedCount) skippedCount.textContent = String(skipped); - - const failedCount = el('failed-count'); - if (failedCount) failedCount.textContent = String(failed); - - const totalUploadedSize = el('total-uploaded-size'); - if (totalUploadedSize) { - totalUploadedSize.textContent = job.successfully_uploaded_bytes_formatted || '-'; - } - - const totalUploadTime = el('total-upload-time'); - if (totalUploadTime) { - totalUploadTime.textContent = job.total_upload_duration_formatted || '-'; - } - - const avgUploadSpeed = el('avg-upload-speed'); - if (avgUploadSpeed) { - avgUploadSpeed.textContent = job.average_upload_speed_mbps - ? `${job.average_upload_speed_mbps} Mbps` - : '-'; - } + setText('completed-count', completed); + setText('skipped-count', skipped); + setText('failed-count', failed); + setText('total-uploaded-size', job.successfully_uploaded_bytes_formatted || '-'); + setText('total-upload-time', job.total_upload_duration_formatted || '-'); + setText( + 'avg-upload-speed', + job.average_upload_speed_mbps ? `${job.average_upload_speed_mbps} Mbps` : '-', + ); const uploadedFiles = job.files.filter( (/** @type {any} */ f) => f.status === 'completed' && f.upload_duration_seconds, ); - const avgFileTime = el('avg-file-time'); - if (avgFileTime) { - if (uploadedFiles.length > 0) { - const avgSeconds = - uploadedFiles.reduce( - (/** @type {number} */ sum, /** @type {any} */ f) => sum + f.upload_duration_seconds, - 0, - ) / uploadedFiles.length; - avgFileTime.textContent = formatDuration(avgSeconds); - } else { - avgFileTime.textContent = '-'; - } + if (uploadedFiles.length > 0) { + const avgSeconds = + uploadedFiles.reduce( + (/** @type {number} */ sum, /** @type {any} */ f) => sum + f.upload_duration_seconds, + 0, + ) / uploadedFiles.length; + setText('avg-file-time', formatDuration(avgSeconds)); + } else { + setText('avg-file-time', '-'); } - const tbody = el('completion-file-list'); + const tbody = document.getElementById('completion-file-list'); if (tbody) { tbody.innerHTML = job.files .map((/** @type {any} */ file) => { @@ -221,13 +92,11 @@ export function showCompletionSummary(job) {
- - - + ${fileIcon('h-4 w-4 text-gray-400 mr-2')} ${file.filename}
+ ${file.s3_path || '-'} ${file.file_size_formatted} ${duration} ${speed} @@ -246,3 +115,46 @@ export function showCompletionSummary(job) { showNotification(`Upload completed with ${failed} failed files`, 'error'); } } + +/** + * Download the upload summary as a CSV file. + */ +export function downloadSummaryCSV() { + if (!lastCompletedJob || !lastCompletedJob.files) { + showNotification('No upload data available', 'error'); + return; + } + + const headers = ['Filename', 'Size', 'S3 Path', 'Status', 'Duration', 'Speed']; + const rows = lastCompletedJob.files.map((/** @type {any} */ file) => { + const duration = file.upload_duration_seconds + ? formatDuration(file.upload_duration_seconds) + : ''; + const speed = file.upload_speed_mbps ? `${file.upload_speed_mbps} Mbps` : ''; + return [ + file.filename, + file.file_size_formatted || '', + file.s3_path || '', + file.status, + duration, + speed, + ]; + }); + + const csvContent = [ + headers.join(','), + ...rows.map((/** @type {string[]} */ row) => + row.map((/** @type {string} */ cell) => `"${String(cell).replace(/"/g, '""')}"`).join(','), + ), + ].join('\n'); + + const blob = new Blob([csvContent], { type: 'text/csv;charset=utf-8;' }); + const url = URL.createObjectURL(blob); + const link = document.createElement('a'); + link.href = url; + const now = new Date(); + const pad = (/** @type {number} */ n) => String(n).padStart(2, '0'); + link.download = `upload-summary-${now.toISOString().slice(0, 10)}-${pad(now.getHours())}${pad(now.getMinutes())}.csv`; + link.click(); + URL.revokeObjectURL(url); +} diff --git a/app/static/js/modules/upload-init.js b/app/static/js/modules/upload-init.js index 1e09a22..f952c20 100644 --- a/app/static/js/modules/upload-init.js +++ b/app/static/js/modules/upload-init.js @@ -1,93 +1,34 @@ -import { applyUploadedFilter, checkForActiveJob, handleFiles } from './analysis.js'; -import { extractFilesFromDrop } from './file-handler.js'; -import { initFolderBrowser, openFolderBrowser, startDirectUpload } from './folder-browser.js'; +import { checkForActiveJob } from './analysis.js'; +import { hideEl, showEl } from './dom.js'; +import { initFolderBrowser, showConfirmModal, startCombinedUpload } from './folder-browser.js'; /** * Upload page initialization and event wiring. */ import state from './state.js'; import { setUploadStep } from './stepper.js'; -import { cancelAnalysis, cancelUpload, resetUpload } from './upload-control.js'; -import { startUpload } from './upload-exec.js'; -import { showNotification } from './utils.js'; +import { cancelUpload, resetUpload } from './upload-control.js'; +import { downloadSummaryCSV } from './upload-exec.js'; export function initUpload() { - const dropZone = document.getElementById('drop-zone'); - const fileInput = /** @type {HTMLInputElement | null} */ (document.getElementById('file-input')); - const folderBtn = document.getElementById('folder-btn'); - - if (!dropZone) return; + const panel = document.getElementById('folder-browser-panel'); + if (!panel) return; initFolderBrowser(); setUploadStep(1); checkForActiveJob(); - folderBtn?.addEventListener('click', (e) => { - e.preventDefault(); - e.stopPropagation(); - e.stopImmediatePropagation(); - openFolderBrowser(); - }); - - dropZone.addEventListener('click', (e) => { - if ( - /** @type {HTMLElement} */ (e.target).id === 'folder-btn' || - /** @type {HTMLElement} */ (e.target).closest('#folder-btn') - ) { - return; - } - fileInput?.click(); - }); - - fileInput?.addEventListener('change', () => { - if (fileInput.files && fileInput.files.length > 0) { - handleFiles(Array.from(fileInput.files)); - } - }); - - dropZone.addEventListener('dragover', (e) => { - e.preventDefault(); - dropZone.classList.add('drag-over'); - }); - - dropZone.addEventListener('dragleave', () => { - dropZone.classList.remove('drag-over'); - }); - - dropZone.addEventListener('drop', async (e) => { - e.preventDefault(); - dropZone.classList.remove('drag-over'); - - const items = e.dataTransfer?.items; - if (items && items.length > 0) { - const files = await extractFilesFromDrop(items); - const mcapFiles = files.filter((f) => f.name.endsWith('.mcap')); - if (mcapFiles.length > 0) { - handleFiles(mcapFiles); - } else { - showNotification('No MCAP files found in dropped items', 'error'); - } - } else { - const files = Array.from(e.dataTransfer?.files || []).filter((f) => f.name.endsWith('.mcap')); - if (files.length > 0) { - handleFiles(files); - } else { - showNotification('Please drop MCAP files only', 'error'); - } - } - }); - - document.getElementById('upload-btn')?.addEventListener('click', startUpload); - document.getElementById('cancel-analyze-btn')?.addEventListener('click', cancelAnalysis); document.getElementById('cancel-upload-btn')?.addEventListener('click', cancelUpload); document.getElementById('upload-more-btn')?.addEventListener('click', resetUpload); + document.getElementById('download-csv-btn')?.addEventListener('click', downloadSummaryCSV); - document.getElementById('hide-uploaded')?.addEventListener('change', applyUploadedFilter); + document.getElementById('continue-upload-btn')?.addEventListener('click', showConfirmModal); + document.getElementById('confirm-upload-btn')?.addEventListener('click', startCombinedUpload); - document.getElementById('start-upload-btn')?.addEventListener('click', startDirectUpload); document.getElementById('cancel-scan-btn')?.addEventListener('click', () => { - document.getElementById('scan-results-section')?.classList.add('hidden'); - document.getElementById('drop-zone')?.classList.remove('hidden'); + hideEl('scan-results-section'); + showEl('folder-browser-panel'); state.selectedFolderPath = null; + setUploadStep(1); }); } diff --git a/app/templates/base.html b/app/templates/base.html index 298c4f8..bbdf0aa 100644 --- a/app/templates/base.html +++ b/app/templates/base.html @@ -46,12 +46,12 @@ } /* Header styles matching NLR branding */ - .nrel-header { + .nlr-header { background: #fff; width: 100%; } - .nrel-header-top { + .nlr-header-top { display: flex; align-items: center; justify-content: space-between; @@ -61,25 +61,25 @@ padding: 0 15px; } - .nrel-header-title { + .nlr-header-title { font-weight: normal; color: #212224; font-size: 24px; letter-spacing: 0.5px; } - .nrel-logo-image { + .nlr-logo-image { width: 200px; } /* Menu bar styles */ - .nrel-menu-bar { + .nlr-menu-bar { background: #EDEDED; border-top: 1px solid #D6D4D4; height: 50px; } - .nrel-menu-container { + .nlr-menu-container { max-width: 1140px; margin: 0 auto; display: flex; @@ -88,7 +88,7 @@ padding: 0 15px; } - .nrel-menu-item { + .nlr-menu-item { color: #2F2F2F; padding: 14px 20px; font-size: 14px; @@ -96,40 +96,40 @@ transition: all 0.2s; } - .nrel-menu-item:hover, - .nrel-menu-item.active { + .nlr-menu-item:hover, + .nlr-menu-item.active { background: #5E6A71; color: #fff; } /* Footer styles matching NLR branding */ - .nrel-footer { + .nlr-footer { background-color: #E3E6E8; font-size: 14px; font-weight: 400; line-height: 1.2; } - .nrel-footer-top { + .nlr-footer-top { background-color: #D1D5D8; padding: 1.5em 0; } - .nrel-footer-bottom { + .nlr-footer-bottom { padding: 1.5em 0 3em 0; } - .nrel-footer a { + .nlr-footer a { color: #000; text-decoration: none; } - .nrel-footer a:hover { + .nlr-footer a:hover { color: #C60; text-decoration: underline; } - .nrel-attr { + .nlr-attr { font-size: 12px; } @@ -144,7 +144,12 @@ /* Progress bar animation */ .progress-bar { - transition: width 0.3s ease; + transition: width 0.4s ease-out; + } + + /* Phase label fade transition */ + #upload-phase-label { + transition: opacity 0.15s ease; } /* File item hover effect */ @@ -210,29 +215,29 @@ /* Responsive adjustments */ @media (max-width: 991px) { - .nrel-header-top { + .nlr-header-top { max-width: 720px; } - .nrel-menu-container { + .nlr-menu-container { max-width: 720px; } } @media (max-width: 767px) { - .nrel-header-top { + .nlr-header-top { flex-direction: column; height: auto; padding: 15px; text-align: center; } - .nrel-header-title { + .nlr-header-title { font-size: 20px; margin-top: 10px; } - .nrel-logo-image { + .nlr-logo-image { width: 180px; } - .nrel-menu-item { + .nlr-menu-item { padding: 12px 15px; font-size: 13px; } @@ -243,38 +248,39 @@ -
+
-
-

{{ display_name }}

+
+

{{ display_name }}

- National Laboratory of the Rockies + class="nlr-logo-image">
-
@@ -301,9 +307,9 @@

{{ display_name }}

-