Imagery - add OSI functionality

CHANGELOG.md

@@ -33,6 +33,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
     - Removed `solocam_img_meta`, and added `get_solocam_dt` to the `imagery` module for formatting the image timestamps
     - Added `utils.check_string_length`, for checking that all strings in a list (e.g., solocam image file names) are the same length, and returning useful warning logs if not
     - Changed `imagery.imagery_timeseries` to use `solocam_dt_from_meta` and `check_string_length`
+- Added function `generate_osi_manifest` for generating OSI image json file, for DAG processing. Also added assocaited script `generate_osi_json.py`
 
 ### Data corrections
 - Added several functions and a data file for checking and correcting ecopuck data:

esdglider/imagery.py

@@ -4,6 +4,7 @@
 import os
 from PIL import Image
 from PIL.ExifTags import TAGS
+import json
 
 import numpy as np
 import pandas as pd
@@ -412,3 +413,88 @@ def extract_image_metadata(image_path):
 
     except Exception:  # noqa: BLE001
         return {"n": image_path.name, "error": "failed"}
+
+
+def generate_osi_manifest(
+    jsonl_filepath: str,
+    output_filepath: str,
+    target_dirs: set[str],
+    deployment_name: str, 
+    log_interval: int = 10000
+) -> None:
+    """
+    Streams a JSONL file line-by-line, filters by directory, and writes a streaming
+    JSON manifest format without holding the full dataset in memory.
+
+    This function generates an OSI-compatible image manifest. 
+    """
+    _log.info(f"Starting processing for file: '{jsonl_filepath}'")
+
+    yr = utils.year_path(deployment_name)
+    base_uri = f"gs://swfscesd-glider-imagery-data-in/{yr}/{deployment_name}/images"
+    target_dirs_set = set(target_dirs)
+    _log.info(f"Target directories to filter: {target_dirs_set}")
+
+    total_lines = 0
+    matched_count = 0
+    malformed_count = 0
+
+    try:
+        with open(jsonl_filepath, "r", encoding="utf-8") as infile, \
+             open(output_filepath, "w", encoding="utf-8") as outfile:
+
+            outfile.write('{"instances": [{"input_images": [')
+
+            is_first_match = True
+
+            for line in infile:
+                total_lines += 1
+
+                # Log progress periodically for large files
+                if total_lines % log_interval == 0:
+                    _log.info(f"Processed {total_lines:,} lines... ({matched_count:,} matches found so far)")
+
+                line = line.strip()
+                if not line or not line.startswith('{'):
+                    continue
+
+                try:
+                    data = json.loads(line)
+                except json.JSONDecodeError:
+                    malformed_count += 1
+                    _log.warning(f"Line {total_lines:,}: Malformed JSON syntax. Skipping.")
+                    continue
+
+                # Check target filter
+                if data.get("p") in target_dirs_set:
+                    file_name = data.get("n")
+                    dir_name = data.get("p")
+
+                    if not file_name:
+                        _log.warning(f"Line {total_lines:,}: Missing filename key 'n'. Skipping.")
+                        continue
+
+                    uri = f"{base_uri}/{dir_name}/{file_name}"
+
+                    if not is_first_match:
+                        outfile.write(",\n")
+                    else:
+                        is_first_match = False
+
+                    outfile.write(f'"{uri}"')
+                    matched_count += 1
+
+            outfile.write(']}]}')
+
+        _log.info("Processing complete.")
+        _log.info(f"Total lines read: {total_lines:,}")
+        _log.info(f"Total matched records written to '{output_filepath}': {matched_count:,}")
+        if malformed_count > 0:
+            _log.warning(f"Encountered {malformed_count:,} malformed or skipped lines.")
+
+    except FileNotFoundError:
+        _log.error(f"Input file not found at: '{jsonl_filepath}'")
+        raise
+    except Exception as e:
+        _log.error(f"An unexpected error occurred on line {total_lines:,}: {str(e)}")
+        raise

scripts/extract-image-metadata.py

@@ -1,3 +1,5 @@
+# This script was written by Gemini, and adapted by Sam Woodman
+
 import json
 from pathlib import Path
 from concurrent.futures import ProcessPoolExecutor
@@ -28,8 +30,6 @@ def run_pipeline(files, deployment_name, depl_meta_file, img_meta_file, num_core
     Runs pipeline to generate the deployment-level and image-specific
     metadata files. 
 
-    This function was written by Gemini, and adapted by Sam Woodman
-
     Parameters
     ----------
     files : list

scripts/generate-osi-json.py

@@ -0,0 +1,57 @@
+import logging
+import json
+from esdglider import gcp, imagery, paths
+from pathlib import Path
+
+
+logger = logging.getLogger(__name__)
+
+# deployment_name = "amlr08-20220513"  #Dir0000
+# target_directories = {"Dir0000", "Dir0001"}
+
+deployment_name = "calanus-20260403"  #dir0000001
+target_directories = {f"dir{i:07d}" for i in range(0, 20)}
+output_filepath = f"/home/user/{deployment_name}-image-manifest1.json"
+
+home = Path.home()
+mnt_path = home / "mnt-gcs"
+
+# imagery_in_bucket_name = "swfscesd-glider-imagery-data-in"
+# imagery_in_path = mnt_path / imagery_in_bucket_name
+imagery_meta_bucket_name = "swfscesd-glider-imagery-metadata"
+imagery_meta_path = mnt_path / imagery_meta_bucket_name
+
+
+
+#------------------------------------------------------------------------------
+if __name__ == "__main__":
+    # gcp.gcs_mount_bucket(imagery_in_bucket_name, imagery_in_path, ro=True)
+    gcp.gcs_mount_bucket(imagery_meta_bucket_name, imagery_meta_path, ro=False)
+
+    logging.basicConfig(
+        # filename=logs_path / log_file_name,
+        # filemode="w",
+        format="%(name)s:%(asctime)s:%(levelname)s:%(message)s [line %(lineno)d]",
+        level=logging.INFO,
+        datefmt="%Y-%m-%d %H:%M:%S",
+    )
+    logging.captureWarnings(True)
+    # logger.info("Beginning scheduled processing for %s", file_info)
+    # print(f"Writing logs to {logs_path / log_file_name}")
+
+    img_paths = paths.get_path_imagery(
+        deployment_name = deployment_name, 
+        # imagery_in_path = imagery_in_path, 
+        imagery_meta_path = imagery_meta_path, 
+        # data_out_path = data_out_path, 
+    )
+
+
+    # Example Usage:
+    imagery.generate_osi_manifest(
+        jsonl_filepath=img_paths["imgmetapath"],
+        output_filepath=output_filepath,
+        target_dirs=target_directories,
+        deployment_name=deployment_name, 
+        log_interval=1000, 
+    )