From 47cf079356f0c604d50639f580a8fbb680a1328b Mon Sep 17 00:00:00 2001 From: Sam Woodman Date: Tue, 8 Sep 2026 21:22:33 +0000 Subject: [PATCH] First pass at function/script to generate image json --- CHANGELOG.md | 1 + esdglider/imagery.py | 86 +++++++++++++++++++++++++++++++ scripts/extract-image-metadata.py | 4 +- scripts/generate-osi-json.py | 57 ++++++++++++++++++++ 4 files changed, 146 insertions(+), 2 deletions(-) create mode 100644 scripts/generate-osi-json.py diff --git a/CHANGELOG.md b/CHANGELOG.md index c4a836d..90d602b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Removed `solocam_img_meta`, and added `get_solocam_dt` to the `imagery` module for formatting the image timestamps - Added `utils.check_string_length`, for checking that all strings in a list (e.g., solocam image file names) are the same length, and returning useful warning logs if not - Changed `imagery.imagery_timeseries` to use `solocam_dt_from_meta` and `check_string_length` +- Added function `generate_osi_manifest` for generating OSI image json file, for DAG processing. Also added assocaited script `generate_osi_json.py` ### Data corrections - Added several functions and a data file for checking and correcting ecopuck data: diff --git a/esdglider/imagery.py b/esdglider/imagery.py index 7cfe657..ddc2b6f 100644 --- a/esdglider/imagery.py +++ b/esdglider/imagery.py @@ -4,6 +4,7 @@ import os from PIL import Image from PIL.ExifTags import TAGS +import json import numpy as np import pandas as pd @@ -412,3 +413,88 @@ def extract_image_metadata(image_path): except Exception: # noqa: BLE001 return {"n": image_path.name, "error": "failed"} + + +def generate_osi_manifest( + jsonl_filepath: str, + output_filepath: str, + target_dirs: set[str], + deployment_name: str, + log_interval: int = 10000 +) -> None: + """ + Streams a JSONL file line-by-line, filters by directory, and writes a streaming + JSON manifest format without holding the full dataset in memory. + + This function generates an OSI-compatible image manifest. + """ + _log.info(f"Starting processing for file: '{jsonl_filepath}'") + + yr = utils.year_path(deployment_name) + base_uri = f"gs://swfscesd-glider-imagery-data-in/{yr}/{deployment_name}/images" + target_dirs_set = set(target_dirs) + _log.info(f"Target directories to filter: {target_dirs_set}") + + total_lines = 0 + matched_count = 0 + malformed_count = 0 + + try: + with open(jsonl_filepath, "r", encoding="utf-8") as infile, \ + open(output_filepath, "w", encoding="utf-8") as outfile: + + outfile.write('{"instances": [{"input_images": [') + + is_first_match = True + + for line in infile: + total_lines += 1 + + # Log progress periodically for large files + if total_lines % log_interval == 0: + _log.info(f"Processed {total_lines:,} lines... ({matched_count:,} matches found so far)") + + line = line.strip() + if not line or not line.startswith('{'): + continue + + try: + data = json.loads(line) + except json.JSONDecodeError: + malformed_count += 1 + _log.warning(f"Line {total_lines:,}: Malformed JSON syntax. Skipping.") + continue + + # Check target filter + if data.get("p") in target_dirs_set: + file_name = data.get("n") + dir_name = data.get("p") + + if not file_name: + _log.warning(f"Line {total_lines:,}: Missing filename key 'n'. Skipping.") + continue + + uri = f"{base_uri}/{dir_name}/{file_name}" + + if not is_first_match: + outfile.write(",\n") + else: + is_first_match = False + + outfile.write(f'"{uri}"') + matched_count += 1 + + outfile.write(']}]}') + + _log.info("Processing complete.") + _log.info(f"Total lines read: {total_lines:,}") + _log.info(f"Total matched records written to '{output_filepath}': {matched_count:,}") + if malformed_count > 0: + _log.warning(f"Encountered {malformed_count:,} malformed or skipped lines.") + + except FileNotFoundError: + _log.error(f"Input file not found at: '{jsonl_filepath}'") + raise + except Exception as e: + _log.error(f"An unexpected error occurred on line {total_lines:,}: {str(e)}") + raise \ No newline at end of file diff --git a/scripts/extract-image-metadata.py b/scripts/extract-image-metadata.py index 97d1286..fd1346d 100644 --- a/scripts/extract-image-metadata.py +++ b/scripts/extract-image-metadata.py @@ -1,3 +1,5 @@ +# This script was written by Gemini, and adapted by Sam Woodman + import json from pathlib import Path from concurrent.futures import ProcessPoolExecutor @@ -28,8 +30,6 @@ def run_pipeline(files, deployment_name, depl_meta_file, img_meta_file, num_core Runs pipeline to generate the deployment-level and image-specific metadata files. - This function was written by Gemini, and adapted by Sam Woodman - Parameters ---------- files : list diff --git a/scripts/generate-osi-json.py b/scripts/generate-osi-json.py new file mode 100644 index 0000000..1bd62c0 --- /dev/null +++ b/scripts/generate-osi-json.py @@ -0,0 +1,57 @@ +import logging +import json +from esdglider import gcp, imagery, paths +from pathlib import Path + + +logger = logging.getLogger(__name__) + +# deployment_name = "amlr08-20220513" #Dir0000 +# target_directories = {"Dir0000", "Dir0001"} + +deployment_name = "calanus-20260403" #dir0000001 +target_directories = {f"dir{i:07d}" for i in range(0, 20)} +output_filepath = f"/home/user/{deployment_name}-image-manifest1.json" + +home = Path.home() +mnt_path = home / "mnt-gcs" + +# imagery_in_bucket_name = "swfscesd-glider-imagery-data-in" +# imagery_in_path = mnt_path / imagery_in_bucket_name +imagery_meta_bucket_name = "swfscesd-glider-imagery-metadata" +imagery_meta_path = mnt_path / imagery_meta_bucket_name + + + +#------------------------------------------------------------------------------ +if __name__ == "__main__": + # gcp.gcs_mount_bucket(imagery_in_bucket_name, imagery_in_path, ro=True) + gcp.gcs_mount_bucket(imagery_meta_bucket_name, imagery_meta_path, ro=False) + + logging.basicConfig( + # filename=logs_path / log_file_name, + # filemode="w", + format="%(name)s:%(asctime)s:%(levelname)s:%(message)s [line %(lineno)d]", + level=logging.INFO, + datefmt="%Y-%m-%d %H:%M:%S", + ) + logging.captureWarnings(True) + # logger.info("Beginning scheduled processing for %s", file_info) + # print(f"Writing logs to {logs_path / log_file_name}") + + img_paths = paths.get_path_imagery( + deployment_name = deployment_name, + # imagery_in_path = imagery_in_path, + imagery_meta_path = imagery_meta_path, + # data_out_path = data_out_path, + ) + + + # Example Usage: + imagery.generate_osi_manifest( + jsonl_filepath=img_paths["imgmetapath"], + output_filepath=output_filepath, + target_dirs=target_directories, + deployment_name=deployment_name, + log_interval=1000, + ) \ No newline at end of file