Make ECS diagnostic compatible with CMIP7 data

changelog/671.fix.md

@@ -0,0 +1 @@
+Made ECS diagnostic work with CMIP7 data.

packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/ecs.py

@@ -22,7 +22,7 @@
     fillvalues_to_nan,
     get_cmip_source_type,
 )
-from climate_ref_esmvaltool.recipe import dataframe_to_recipe, get_child_and_parent_dataset
+from climate_ref_esmvaltool.recipe import get_child_and_parent_dataset
 from climate_ref_esmvaltool.types import MetricBundleArgs, OutputBundleArgs, Recipe
 
 
@@ -33,18 +33,14 @@ class EquilibriumClimateSensitivity(ESMValToolDiagnostic):
 
     name = "Equilibrium Climate Sensitivity"
     slug = "equilibrium-climate-sensitivity"
-    base_recipe = "recipe_ecs.yml"
+    base_recipe = "ref/recipe_ref_ecs.yml"
 
     variables = (
         "rlut",
         "rsdt",
         "rsut",
         "tas",
     )
-    experiments = (
-        "abrupt-4xCO2",
-        "piControl",
-    )
 
     data_requirements = (
         (
@@ -185,67 +181,28 @@ def update_recipe(
         input_files: dict[SourceDatasetType, pandas.DataFrame],
     ) -> None:
         """Update the recipe."""
-        # Only run the diagnostic that computes ECS for a single model.
-        recipe["diagnostics"] = {
-            "ecs": {
-                "description": "Calculate ECS.",
-                "variables": {
-                    "tas": {
-                        "preprocessor": "spatial_mean",
-                    },
-                    "rtnt": {
-                        "preprocessor": "spatial_mean",
-                        "derive": True,
-                    },
-                },
-                "scripts": {
-                    "calculate": {
-                        "script": "climate_metrics/ecs.py",
-                        "calculate_mmm": False,
-                    },
-                },
-            },
-        }
-
         # Prepare updated datasets section in recipe. It contains two
         # datasets, one for the "abrupt-4xCO2" and one for the "piControl"
         # experiment.
         cmip_source = get_cmip_source_type(input_files)
-        if cmip_source == SourceDatasetType.CMIP6:
-            df = input_files[SourceDatasetType.CMIP6]
-            recipe["datasets"] = get_child_and_parent_dataset(
-                df[df.variable_id == "tas"],
-                parent_experiment="piControl",
-                child_duration_in_years=150,
-                parent_offset_in_years=0,
-                parent_duration_in_years=150,
-            )
-        else:
-            # CMIP7: use per-variable additional_datasets to preserve correct branding_suffix
-            recipe_variables = dataframe_to_recipe(
-                input_files[cmip_source],
-                equalize_timerange=True,
-            )
-            recipe["datasets"] = []
-            for var_name, var_settings in recipe["diagnostics"]["ecs"]["variables"].items():
-                short_name = var_settings.get("short_name", var_name)
-                if short_name in recipe_variables:
-                    var_settings["additional_datasets"] = recipe_variables[short_name]["additional_datasets"]
-                elif var_name == "rtnt":
-                    # rtnt is derived from rlut, rsdt, rsut - use rlut's dataset
-                    var_settings["additional_datasets"] = recipe_variables["rlut"]["additional_datasets"]
+        df = input_files[cmip_source]
+        recipe["datasets"] = get_child_and_parent_dataset(
+            df[df.variable_id == "tas"],
+            parent_experiment="piControl",
+            child_duration_in_years=150,
+            parent_offset_in_years=0,
+            parent_duration_in_years=150,
+        )
 
-        # Remove keys from the recipe that are only used for YAML anchors
-        keys_to_remove = [
-            "CMIP5_RTMT",
-            "CMIP6_RTMT",
-            "CMIP5_RTNT",
-            "CMIP6_RTNT",
-            "ECS_SCRIPT",
-            "SCATTERPLOT",
-        ]
-        for key in keys_to_remove:
-            recipe.pop(key, None)
+        # Delete branding suffixes from dataset entries because they are
+        # variable-specific
+        for dataset in recipe["datasets"]:
+            dataset.pop("branding_suffix", None)
+
+        # For CMIP6, delete all appearances of branding suffixes
+        if cmip_source == SourceDatasetType.CMIP6:
+            for variable in recipe["diagnostics"]["ecs"]["variables"].values():
+                variable.pop("branding_suffix", None)
 
     @staticmethod
     def format_result(

packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/recipe.py

@@ -296,7 +296,7 @@ def get_child_and_parent_dataset(
     return [child_dataset, parent_dataset]
 
 
-_ESMVALTOOL_COMMIT = "dead90ca848d3892ed90785b3940909d58d27780"
+_ESMVALTOOL_COMMIT = "0a9a986bd598f6d9334d8f480e03d57e456c3bb4"
 _ESMVALTOOL_VERSION = f"2.14.0.dev65+g{_ESMVALTOOL_COMMIT[:9]}"
 _ESMVALTOOL_URL = f"git+https://github.com/ESMValGroup/ESMValTool.git@{_ESMVALTOOL_COMMIT}"
 

packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/recipes.txt

@@ -1,6 +1,5 @@
 examples/recipe_python.yml                  ab3f06d269bb2c1368f4dc39da9bcb232fb2adb1fa556ba769e6c16294ffb4a3
 recipe_calculate_gwl_exceedance_stats.yml   9f28304b85f7043fb76eb29ee84da46e084941c9db80eaa2993c151dbf022472
-recipe_ecs.yml                              0cc57034fcb64e32015b4ff949ece5df8cdb8c6f493618b50ceded119fb37918
 recipe_seaice_sensitivity.yml               07270818ebf04a58713dc989720cc9ace77f28d42b1ea189c3bdaf22f5d64577
 recipe_tcr.yml                              35f9ef035a4e71aff5cac5dd26c49da2162fc00291bf3b0bd16b661b7b2f606b
 recipe_tcre.yml                             48fc9e3baf541bbcef7491853ea3a774053771dca33352b41466425faeaa38af
@@ -9,8 +8,9 @@ ref/recipe_enso_basicclimatology.yml        9ea7deb7ee668e39ac44618b96496d898bd8
 ref/recipe_enso_characteristics.yml         34c2518b138068ac96d212910b979d54a8fcedee2c0089b5acd56a42c41dc3e4
 ref/recipe_ref_annual_cycle_region.yml      88e0c2f07a76c4de4276547badcc74091516e03840eddb1f0e62469c67a0a86d
 ref/recipe_ref_cre.yml                      4375f262479c3b3e1b348b71080a6d758e195bda76516a591182045a3a29aa32
+ref/recipe_ref_ecs.yml                      3121df2d8c27f42b11e56f918ecbce49b0c8dc831e6d6886493f64bc8cce3032
 ref/recipe_ref_fire.yml                     2ad82effaca4e742d8abe6a0aa07bb46e1e92ef0d2d240760f7623b0ba045926
-ref/recipe_ref_ozone.yml                    388aaf01721c0856ab7143b47b9d2f1efeaaae4ef57e11572c1e133b49aad0a6
+ref/recipe_ref_ozone.yml                    09ded4946ecd762ba000916c9f8618120a2e6269930427ae5129da466165f3e5
 ref/recipe_ref_sea_ice_area_basic.yml       7d01a8527880663ca28284772f83a8356d9972fb4f022a4000e50a56ce044b09
 ref/recipe_ref_scatterplot.yml              b99d1736e16256d161847b025811d7088ad9f892d4887fb009fa99c4079135a0
 ref/recipe_ref_timeseries_region.yml        deee4403ea25867401e163f6c9d7fcc4fee09d3088009007fcad6a29d01247a8

packages/climate-ref-esmvaltool/tests/unit/diagnostics/recipes/recipe_equilibrium_climate_sensitivity_cmip6.yml

@@ -12,25 +12,6 @@ documentation:
   - schlund_manuel
   references:
   - gregory04grl
-  projects:
-  - crescendo
-preprocessors:
-  spatial_mean:
-    area_statistics:
-      operator: mean
-diagnostics:
-  ecs:
-    description: Calculate ECS.
-    variables:
-      tas:
-        preprocessor: spatial_mean
-      rtnt:
-        preprocessor: spatial_mean
-        derive: true
-    scripts:
-      calculate:
-        script: climate_metrics/ecs.py
-        calculate_mmm: false
 datasets:
 - project: CMIP6
   activity: CMIP
@@ -50,3 +31,20 @@ datasets:
   grid: gn
   mip: Amon
   timerange: 0101/0250
+preprocessors:
+  spatial_mean:
+    area_statistics:
+      operator: mean
+diagnostics:
+  ecs:
+    description: Calculate ECS.
+    variables:
+      tas:
+        preprocessor: spatial_mean
+      rtnt:
+        derive: true
+        preprocessor: spatial_mean
+    scripts:
+      calculate:
+        script: climate_metrics/ecs.py
+        calculate_mmm: false

packages/climate-ref-esmvaltool/tests/unit/diagnostics/recipes/recipe_equilibrium_climate_sensitivity_cmip7.yml

@@ -12,8 +12,29 @@ documentation:
   - schlund_manuel
   references:
   - gregory04grl
-  projects:
-  - crescendo
+datasets:
+- project: CMIP7
+  activity: CMIP
+  dataset: ACCESS-ESM1-5
+  ensemble: r1i1p1f1
+  exp: abrupt-4xCO2
+  frequency: mon
+  grid: gn
+  institute: CSIRO
+  mip: atmos
+  region: glb
+  timerange: 0101/0250
+- project: CMIP7
+  activity: CMIP
+  dataset: ACCESS-ESM1-5
+  ensemble: r1i1p1f1
+  exp: piControl
+  frequency: mon
+  grid: gn
+  institute: CSIRO
+  mip: atmos
+  region: glb
+  timerange: 0101/0250
 preprocessors:
   spatial_mean:
     area_statistics:
@@ -23,62 +44,13 @@ diagnostics:
     description: Calculate ECS.
     variables:
       tas:
+        branding_suffix: tavg-h2m-hxy-u
         preprocessor: spatial_mean
-        additional_datasets:
-        - project: CMIP7
-          activity: CMIP
-          branding_suffix: tavg-h2m-hxy-u
-          dataset: ACCESS-ESM1-5
-          ensemble: r1i1p1f1
-          exp: abrupt-4xCO2
-          frequency: mon
-          grid: gn
-          institute: CSIRO
-          mip: atmos
-          region: glb
-          timerange: 01010116T120000/02501216T120000
-        - project: CMIP7
-          activity: CMIP
-          branding_suffix: tavg-h2m-hxy-u
-          dataset: ACCESS-ESM1-5
-          ensemble: r1i1p1f1
-          exp: piControl
-          frequency: mon
-          grid: gn
-          institute: CSIRO
-          mip: atmos
-          region: glb
-          timerange: 01010116T120000/02501216T120000
       rtnt:
-        preprocessor: spatial_mean
+        branding_suffix: tavg-u-hxy-u
         derive: true
-        additional_datasets:
-        - project: CMIP7
-          activity: CMIP
-          branding_suffix: tavg-u-hxy-u
-          dataset: ACCESS-ESM1-5
-          ensemble: r1i1p1f1
-          exp: abrupt-4xCO2
-          frequency: mon
-          grid: gn
-          institute: CSIRO
-          mip: atmos
-          region: glb
-          timerange: 01010116T120000/02501216T120000
-        - project: CMIP7
-          activity: CMIP
-          branding_suffix: tavg-u-hxy-u
-          dataset: ACCESS-ESM1-5
-          ensemble: r1i1p1f1
-          exp: piControl
-          frequency: mon
-          grid: gn
-          institute: CSIRO
-          mip: atmos
-          region: glb
-          timerange: 01010116T120000/02501216T120000
+        preprocessor: spatial_mean
     scripts:
       calculate:
         script: climate_metrics/ecs.py
         calculate_mmm: false
-datasets: []