Add qartod and config

CHANGELOG.md

@@ -13,6 +13,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Changed the name of the module `acoustics` to `aa`, to refect that this module is specific to active acoustic data
 - Added a `paths` module for all functions involved in generating file or directory paths. Moved all such path functions from other modules into `paths`, and updated these path functions to use the new ESD prod directory structure. 
 - Changed `gcp` module, using Gemini to follow a "Fail-Fast" module design. 
+- Added a qartod module, for generating qartod flags using the `ioos_qc` package for the science dataset.
+- Added dynamic QARTOD threshold calculation utilities to the `qartod` module and added create_qc_summary_table() for generating profile-level deployment QC summary tables.
+- Added combine_datasets() to the `utils` for combining profile NetCDF files into a deployment dataset.
+- Added deployment QC plotting utilities in `plots`, including stacked QC flag summary plots and variable-specific QC flag time-series plots.
 
 ### Imagery
 - Changed to extracting the date extracting EXIF metadata from the imagery files, rather than deriving image datetimes from the filenames. This included writing relevant metadata to a metadata-specific bucket, and reading the datetimes from the metadata files. Specifically:

environment.yml

@@ -27,6 +27,7 @@ dependencies:
   - folium
   - skyfield
   - timezonefinder
+  - ioos_qc
 
   - pip:
     - dbdreader

esdglider/data/qartod-config.yml

@@ -0,0 +1,207 @@
+contexts:
+  - streams:
+
+      # =====================================================
+      # POSITION
+      # =====================================================
+
+      latitude:
+        qartod:
+          gross_range_test:
+            suspect_span: [-90, 90]
+            fail_span: [-90, 90]
+
+      longitude:
+        qartod:
+          gross_range_test:
+            suspect_span: [-180, 180]
+            fail_span: [-180, 180]
+
+      lat:
+        qartod:
+          gross_range_test:
+            suspect_span: [-90, 90]
+            fail_span: [-90, 90]
+
+      lon:
+        qartod:
+          gross_range_test:
+            suspect_span: [-180, 180]
+            fail_span: [-180, 180]
+
+      profile_lat:
+        qartod:
+          gross_range_test:
+            suspect_span: [-90, 90]
+            fail_span: [-90, 90]
+
+      profile_lon:
+        qartod:
+          gross_range_test:
+            suspect_span: [-180, 180]
+            fail_span: [-180, 180]
+
+      lat_uv:
+        qartod:
+          gross_range_test:
+            suspect_span: [-90, 90]
+            fail_span: [-90, 90]
+
+      lon_uv:
+        qartod:
+          gross_range_test:
+            suspect_span: [-180, 180]
+            fail_span: [-180, 180]
+
+      # =====================================================
+      # PRESSURE / DEPTH
+      # =====================================================
+
+      pressure:
+        qartod:
+          gross_range_test:
+            suspect_span: [0, 1000]
+            fail_span: [0, 6000]
+
+          spike_test:
+            suspect_threshold: 0.0
+            fail_threshold: 0.0
+
+          rate_of_change_test:
+            threshold: 0.1
+
+          flat_line_test:
+            tolerance: 0.001
+            suspect_threshold: 3000
+            fail_threshold: 5000
+
+      depth:
+        qartod:
+          gross_range_test:
+            suspect_span: [0, 1200]
+            fail_span: [0, 2000]
+
+          spike_test:
+            suspect_threshold: 0.0
+            fail_threshold: 0.0
+
+          rate_of_change_test:
+            threshold: 0.1
+
+      # =====================================================
+      # TEMPERATURE
+      # =====================================================
+
+      temperature:
+        qartod:
+          gross_range_test:
+            suspect_span: [0, 35]
+            fail_span: [-2, 40]
+
+          spike_test:
+            suspect_threshold: 0.0
+            fail_threshold: 0.0
+
+          rate_of_change_test:
+            threshold: 0.1
+
+          flat_line_test:
+            tolerance: 0.001
+            suspect_threshold: 3000
+            fail_threshold: 5000
+
+      # =====================================================
+      # CONDUCTIVITY
+      # =====================================================
+
+      conductivity:
+        qartod:
+          gross_range_test:
+            suspect_span: [0, 6]
+            fail_span: [0, 9]
+
+          spike_test:
+            suspect_threshold: 0.0
+            fail_threshold: 0.0
+
+          rate_of_change_test:
+            threshold: 0.1
+
+          flat_line_test:
+            tolerance: 0.001
+            suspect_threshold: 3000
+            fail_threshold: 5000
+
+      # =====================================================
+      # SALINITY
+      # =====================================================
+
+      salinity:
+        qartod:
+            gross_range_test:
+              fail_span: [0, 42]
+              suspect_span: [10, 38]
+
+            spike_test:
+              suspect_threshold: 0.0
+              fail_threshold: 0.0
+
+            rate_of_change_test:
+              threshold: 0.1
+
+            flat_line_test:
+              tolerance: 0.001
+              suspect_threshold: 3000
+              fail_threshold: 5000
+
+      # =====================================================
+      # DENSITY
+      # =====================================================
+
+      density:
+        qartod:
+            gross_range_test:
+              fail_span: [1000, 1100]
+              suspect_span: null
+
+            spike_test:
+              suspect_threshold: 0.0
+              fail_threshold: 0.0
+
+            rate_of_change_test:
+              threshold: 0.1
+
+            flat_line_test:
+              tolerance: 0.001
+              suspect_threshold: 3000
+              fail_threshold: 5000
+
+      # =====================================================
+      # VELOCITY
+      # =====================================================
+
+      u:
+        qartod:
+          gross_range_test:
+            suspect_span: [-3, 3]
+            fail_span: [-10, 10]
+
+          spike_test:
+            suspect_threshold: 0.0
+            fail_threshold: 0.0
+
+          rate_of_change_test:
+            threshold: 0.1
+
+      v:
+        qartod:
+          gross_range_test:
+            suspect_span: [-3, 3]
+            fail_span: [-10, 10]
+
+          spike_test:
+            suspect_threshold: 0.0
+            fail_threshold: 0.0
+
+          rate_of_change_test:
+            threshold: 0.1 

esdglider/paths.py

@@ -55,6 +55,22 @@ def get_path_yaml_deployment_vars(yaml_type: str) -> str:
     with resources.as_file(ref) as path:
         return str(path)
 
+def get_path_qartod_config() -> str:
+    """
+    Get the path to the packaged QARTOD configuration file.
+    The configuration file is distributed with the package and
+    contains the default IOOS QARTOD test configuration used when
+    generating quality-control flags.
+
+    Returns
+    -------
+    str
+        Path to the packaged ``qartod-config.yml`` file.
+    """
+    ref = resources.files("esdglider.data") / "qartod-config.yml"
+
+    with resources.as_file(ref) as path:
+        return str(path)
 
 def get_path_glider_data_out(
     deployment_name: str,