From 75bcd8fece8c62a63467424480b6287599e61ba7 Mon Sep 17 00:00:00 2001
From: Max Kipp <jkipp@dewberry.com>
Date: Wed, 23 Sep 2026 16:18:10 -0400
Subject: [PATCH] Fix NWM CONUS performance: Leverage cache, slice once after
 merging variables,  load variables sequentially for optimal memory footprint.

---
 .../historical_forcing.py                     | 28 +++++++++----------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/historical_forcing.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/historical_forcing.py
index 374154bb..fe1f3083 100644
--- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/historical_forcing.py
+++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/historical_forcing.py
@@ -319,7 +319,7 @@ def compute_ds(self) -> xr.Dataset:
                     os.replace(tmp_file, self.nc_path)
                     LOG.info(f"Renamed: {tmp_file} -> {self.nc_path}")
                 except Exception as e:
-                    pass
+                    LOG.warning(f"Unable to write cache file {self.nc_path}: {e}")
         return ds
 
     @cached_property
@@ -653,19 +653,19 @@ def sliced_ds(self) -> xr.Dataset:
         :return: xarray Dataset
         :raises Exception: If zarr open fails
         """
-        datasets = []
-        for var in self.vars:
-            try:
-                with self.timing_block(f"lazy loading {self.dataset_name} data"):
-                    datasets.append(self.slice_ds(self.s3_lazy_ds[var]))
-            except Exception as e:
-                LOG.critical(
-                    f"Error opening {self.dataset_name} data from {self.url(var)}: {e}\n"
-                )
-                raise e
-        return xr.merge(datasets, compat="override").rename(
-            {self.x_label: "x", self.y_label: "y"}
-        )
+        cached_data = self.load_cache()
+        if cached_data is not None:
+            return cached_data
+        try:
+            with self.timing_block(f"Loading {self.dataset_name} data"):
+                ds = xr.merge(self.s3_lazy_ds.values(), compat="override")
+                ds = self.slice_ds(ds).rename({self.x_label: "x", self.y_label: "y"})
+                for variable in ds.data_vars.values():
+                    variable.load()
+                return ds
+        except Exception as e:
+            LOG.critical(f"Error opening {self.dataset_name} data: {e}\n")
+            raise e
 
     @cached_property
     def s3_lazy_ds(self) -> dict[str, xr.Dataset]: