From 5daa828995e823173e6a44b8e6e5411e587678ec Mon Sep 17 00:00:00 2001 From: Lauren Schambach <157057315+laurscham@users.noreply.github.com> Date: Thu, 4 Jun 2026 12:51:11 -0400 Subject: [PATCH 1/7] updates to run run_coastal.sh via the RTE --- .../NextGen_Forcings_Engine/bmi_model.py | 5 +++-- .../NextGen_Forcings_Engine/core/geoMod.py | 8 ++++---- .../NextGen_Forcings_Engine/core/parallel.py | 3 +++ 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py index 3c27eacb..94cca1f0 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py @@ -178,6 +178,7 @@ def __init__(self): self._job_meta = None self._mpi_meta = None self.geo_meta = None + self.GeoMeta = HydrofabricGeoMeta self._grid_type = None self._grids = None self._grid_map = None @@ -290,7 +291,7 @@ def initialize(self, config_file: str, output_path: str | None = None) -> None: # Initialize MPI communication self._mpi_meta = MpiConfig(self._job_meta) - self.geo_meta = HydrofabricGeoMeta(self._job_meta, self._mpi_meta) + self.geo_meta = self.GeoMeta(self._job_meta, self._mpi_meta) try: comm = MPI.Comm.f2py(self._comm) if self._comm is not None else None @@ -1665,7 +1666,7 @@ def set_var_names(self) -> None: # will support a BMI field for liquid fraction of precipitation self._output_var_names = BMI_MODEL["_output_var_names"] self._var_name_units_map = BMI_MODEL["_var_name_units_map"] - if self.config_options.include_lqfrac == 1: + if self._job_meta.include_lqfrac == 1: self._output_var_names += ["LQFRAC_ELEMENT"] self._var_name_units_map |= { "LQFRAC_ELEMENT": ["Liquid Fraction of Precipitation", "%"] diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/geoMod.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/geoMod.py index 90a225cf..db3a95e7 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/geoMod.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/geoMod.py @@ -387,9 +387,9 @@ def latitude_grid(self) -> np.ndarray: # Scatter global XLAT_M grid to processors.. if self.mpi_config.rank == 0: if self.ndim_lat == 3: - var_tmp = self.lat_var[0, :, :] + var_tmp = np.asarray(self.lat_var[0, :, :]) elif self.ndim_lat == 2: - var_tmp = self.lat_var[:, :] + var_tmp = np.asarray(self.lat_var[:, :]) elif self.ndim_lat == 1: lat = self.lat_var[:] lon = self.lon_var[:] @@ -421,9 +421,9 @@ def longitude_grid(self) -> np.ndarray: if ( self.ndim_lat == 3 ): # NOTE The original code has lat here... should it maybe be lon instead? - var_tmp = self.lon_var[0, :, :] + var_tmp = np.asarray(self.lon_var[0, :, :]) elif self.ndim_lon == 2: - var_tmp = self.lon_var[:, :] + var_tmp = np.asarray(self.lon_var[:, :]) elif self.ndim_lon == 1: lat = self.lat_var[:] lon = self.lon_var[:] diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py index e0e48891..d63ed9a0 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/parallel.py @@ -239,6 +239,9 @@ def _cleanup_geogrid(self) -> None: if self.config_options is None: self.log_debug("Cleanup: config_options is not set") return + if getattr(self.config_options, "grid_type", None) == "gridded": + self.log_debug("Cleanup: skipping geogrid cleanup for gridded mode") + return geogrid = getattr(self.config_options, "geogrid", None) if geogrid is not None: self.try_delete_file_no_reraise(geogrid) From 5681f153bde332143c4c956c8c84eae1123a80e9 Mon Sep 17 00:00:00 2001 From: Lauren Schambach <157057315+laurscham@users.noreply.github.com> Date: Thu, 4 Jun 2026 15:14:51 -0400 Subject: [PATCH 2/7] adding debug statements to geoMod.py and regrid.py to debug no data values coming out of forcing engine on regular grids --- .../NextGen_Forcings_Engine/core/geoMod.py | 18 ++++++-- .../NextGen_Forcings_Engine/core/regrid.py | 41 +++++++++++++++++++ 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/geoMod.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/geoMod.py index db3a95e7..64fbb8b4 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/geoMod.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/geoMod.py @@ -274,12 +274,15 @@ def nx_global(self) -> int: if self.mpi_config.rank == 0: try: if self.ndim_lat == 3: - return self.lat_var.shape[2] + nx = self.lat_var.shape[2] elif self.ndim_lat == 2: - return self.lat_var.shape[1] + nx = self.lat_var.shape[1] else: # NOTE Is this correct? using lon_var - return self.lon_var.shape[0] + nx = self.lon_var.shape[0] + print(f"[DEBUG GeoMeta] geogrid={self.config_options.geogrid}") + print(f"[DEBUG GeoMeta] ndim_lat={self.ndim_lat}, ny_global={self.ny_global}, nx_global={nx}") + return nx except Exception as e: self.config_options.errMsg = f"Unable to extract X dimension size from {self.config_options.lon_var} in: {self.config_options.geogrid}" log_critical(self.config_options, self.mpi_config) @@ -398,6 +401,11 @@ def latitude_grid(self) -> np.ndarray: # Flag to grab entire array for AWS slicing if self.config_options.aws: self.lat_bounds = var_tmp + print(f"[DEBUG GeoMeta] lat_var name: {self.config_options.lat_var}") + print(f"[DEBUG GeoMeta] latitude_grid shape: {var_tmp.shape}") + print(f"[DEBUG GeoMeta] latitude_grid min/max: {var_tmp.min():.4f} / {var_tmp.max():.4f}") + print(f"[DEBUG GeoMeta] latitude_grid corners: [0,0]={var_tmp[0,0]:.4f}, [-1,-1]={var_tmp[-1,-1]:.4f}") + print(f"[DEBUG GeoMeta] latitude_grid first row mean: {var_tmp[0,:].mean():.4f}, last row mean: {var_tmp[-1,:].mean():.4f}") else: var_tmp = None return var_tmp, "latitude_grid", self.config_options, False @@ -432,6 +440,10 @@ def longitude_grid(self) -> np.ndarray: # Flag to grab entire array for AWS slicing if self.config_options.aws: self.lon_bounds = var_tmp + print(f"[DEBUG GeoMeta] lon_var name: {self.config_options.lon_var}") + print(f"[DEBUG GeoMeta] longitude_grid shape: {var_tmp.shape}") + print(f"[DEBUG GeoMeta] longitude_grid min/max: {var_tmp.min():.4f} / {var_tmp.max():.4f}") + print(f"[DEBUG GeoMeta] longitude_grid corners: [0,0]={var_tmp[0,0]:.4f}, [-1,-1]={var_tmp[-1,-1]:.4f}") else: var_tmp = None diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/regrid.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/regrid.py index 5dd3a109..2b3b6336 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/regrid.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/regrid.py @@ -1928,6 +1928,14 @@ def regrid_conus_hrrr(input_forcings, config_options, wrf_hydro_geo_meta, mpi_co ) err_handler.check_program_status(config_options, mpi_config) + if mpi_config.rank == 0: + out_data = input_forcings.esmf_field_out.data + n_nodata = int(np.sum(out_data == config_options.globalNdv)) + n_total = out_data.size + print(f"[DEBUG regrid out] var={input_forcings.netcdf_var_names[force_count]}, esmf_field_out shape: {out_data.shape}") + print(f"[DEBUG regrid out] min/max: {out_data.min():.4f} / {out_data.max():.4f}") + print(f"[DEBUG regrid out] no-data count: {n_nodata}/{n_total} ({100*n_nodata/n_total:.1f}%)") + try: input_forcings.regridded_forcings2[ input_forcings.input_map_output[force_count], :, : @@ -11276,6 +11284,21 @@ def check_regrid_status( """ pt = Partials(mpi_config, config_options) + # For gridded domains, esmf_lat/esmf_lon on GriddedGeoMeta are shadowed to None by + # GeoMeta.__init__ (via GEOMOD["GeoMeta"]), so the @cached_property never runs and the + # destination ESMF Grid's coordinate arrays are never populated. Populate them once here. + if config_options.grid_type == "gridded" and wrf_hydro_geo_meta.esmf_lat is None: + esmf_lat = wrf_hydro_geo_meta.esmf_grid.get_coords(1) + esmf_lat[:, :] = wrf_hydro_geo_meta.latitude_grid + wrf_hydro_geo_meta.esmf_lat = esmf_lat + esmf_lon = wrf_hydro_geo_meta.esmf_grid.get_coords(0) + esmf_lon[:, :] = wrf_hydro_geo_meta.longitude_grid + wrf_hydro_geo_meta.esmf_lon = esmf_lon + if mpi_config.rank == 0: + print(f"[DEBUG dest grid] populated ESMF destination grid coords") + print(f"[DEBUG dest grid] esmf_lat min/max: {esmf_lat.min():.4f} / {esmf_lat.max():.4f}") + print(f"[DEBUG dest grid] esmf_lon min/max: {esmf_lon.min():.4f} / {esmf_lon.max():.4f}") + # If the destination ESMF field hasn't been created, create it here. if not input_forcings.esmf_field_out: if config_options.grid_type == "gridded": @@ -12017,6 +12040,19 @@ def calculate_weights( err_handler.check_program_status(config_options, mpi_config) + if mpi_config.rank == 0 and lat_tmp is not None and lon_tmp is not None: + print(f"[DEBUG calc_weights] product={input_forcings.product_name}, lat_var='{lat_var}', lon_var='{lon_var}'") + print(f"[DEBUG calc_weights] raw lat_tmp shape: {lat_tmp.shape}, lon_tmp shape: {lon_tmp.shape}") + print(f"[DEBUG calc_weights] lat_tmp min/max: {lat_tmp.min():.4f} / {lat_tmp.max():.4f}") + print(f"[DEBUG calc_weights] lon_tmp min/max: {lon_tmp.min():.4f} / {lon_tmp.max():.4f}") + print(f"[DEBUG calc_weights] lat_tmp corners: [0,0]={lat_tmp[0,0]:.4f}, [-1,-1]={lat_tmp[-1,-1]:.4f}") + print(f"[DEBUG calc_weights] lon_tmp corners: [0,0]={lon_tmp[0,0]:.4f}, [-1,-1]={lon_tmp[-1,-1]:.4f}") + print(f"[DEBUG calc_weights] input_forcings.ny_global={input_forcings.ny_global}, nx_global={input_forcings.nx_global}") + # Normalize source longitudes from 0-360 to -180/+180 to match geo_em geogrid convention + if lon_tmp.max() > 180: + lon_tmp = np.where(lon_tmp > 180, lon_tmp - 360, lon_tmp) + print(f"[DEBUG calc_weights] lon_tmp normalized to -180/+180: min/max={lon_tmp.min():.4f} / {lon_tmp.max():.4f}") + # Scatter global GFS latitude grid to processors.. if mpi_config.rank == 0: var_tmp = lat_tmp @@ -12050,6 +12086,11 @@ def calculate_weights( input_forcings.esmf_lats[:, :] = var_sub_lat_tmp input_forcings.esmf_lons[:, :] = var_sub_lon_tmp + if mpi_config.rank == 0: + print(f"[DEBUG ESMF src] esmf_lats (get_coords(1)) shape={input_forcings.esmf_lats.shape}") + print(f"[DEBUG ESMF src] esmf_lats min/max: {input_forcings.esmf_lats.min():.4f} / {input_forcings.esmf_lats.max():.4f}") + print(f"[DEBUG ESMF src] esmf_lons (get_coords(0)) shape={input_forcings.esmf_lons.shape}") + print(f"[DEBUG ESMF src] esmf_lons min/max: {input_forcings.esmf_lons.min():.4f} / {input_forcings.esmf_lons.max():.4f}") del var_sub_lat_tmp del var_sub_lon_tmp del lat_tmp From 33d7170196bf49762f565bdbd67bf500d8262ed5 Mon Sep 17 00:00:00 2001 From: Lauren Schambach <157057315+laurscham@users.noreply.github.com> Date: Thu, 4 Jun 2026 23:47:32 -0400 Subject: [PATCH 3/7] Attempt to reduce memory usage - only read in fields needed from the geogrids --- .../NextGen_Forcings_Engine/core/geoMod.py | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/geoMod.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/geoMod.py index 64fbb8b4..0ccfed49 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/geoMod.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/geoMod.py @@ -131,12 +131,33 @@ def spatial_metadata_exists(self) -> bool: else: return True + # All config_options attributes that hold geogrid variable names. + # Used to select only the needed variables when opening the geogrid file. + _GEOGRID_VAR_ATTRS = ( + "lat_var", "lon_var", "hgt_var", "cosalpha_var", "sinalpha_var", + "slope_var", "slope_azimuth_var", "slope_var_elem", "slope_azimuth_var_elem", + "nodecoords_var", "elemcoords_var", "elemconn_var", "numelemconn_var", + "element_id_var", "hgt_elem_var", + ) + @cached_property def geogrid_ds(self) -> xr.Dataset: - """Open the geogrid file and return the xarray dataset object.""" + """Open the geogrid file and load only the variables needed by this run. + + geo_em_CONUS.nc is ~8.9 GB; loading the whole file with ds.load() exhausts + RAM on standard instances. We open lazily, select only the variables + referenced by config_options, then load that subset into memory. + Global attributes (DX, DY, etc.) are preserved on the subset dataset. + """ try: with xr.open_dataset(self.config_options.geogrid) as ds: - return ds.load() + needed = [ + getattr(self.config_options, attr) + for attr in self._GEOGRID_VAR_ATTRS + if getattr(self.config_options, attr, None) is not None + and getattr(self.config_options, attr) in ds + ] + return ds[needed].load() except Exception as e: self.config_options.errMsg = "Unable to open geogrid file with xarray" log_critical(self.config_options, self.mpi_config) From 6d52e36311864b57c0733e60f295bf20f4505bf1 Mon Sep 17 00:00:00 2001 From: Lauren Schambach Date: Wed, 5 Aug 2026 12:14:53 -0400 Subject: [PATCH 4/7] remove debug statements --- .../NextGen_Forcings_Engine/core/geoMod.py | 11 -------- .../NextGen_Forcings_Engine/core/regrid.py | 25 ------------------- 2 files changed, 36 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/geoMod.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/geoMod.py index 0ccfed49..5db72a1a 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/geoMod.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/geoMod.py @@ -301,8 +301,6 @@ def nx_global(self) -> int: else: # NOTE Is this correct? using lon_var nx = self.lon_var.shape[0] - print(f"[DEBUG GeoMeta] geogrid={self.config_options.geogrid}") - print(f"[DEBUG GeoMeta] ndim_lat={self.ndim_lat}, ny_global={self.ny_global}, nx_global={nx}") return nx except Exception as e: self.config_options.errMsg = f"Unable to extract X dimension size from {self.config_options.lon_var} in: {self.config_options.geogrid}" @@ -422,11 +420,6 @@ def latitude_grid(self) -> np.ndarray: # Flag to grab entire array for AWS slicing if self.config_options.aws: self.lat_bounds = var_tmp - print(f"[DEBUG GeoMeta] lat_var name: {self.config_options.lat_var}") - print(f"[DEBUG GeoMeta] latitude_grid shape: {var_tmp.shape}") - print(f"[DEBUG GeoMeta] latitude_grid min/max: {var_tmp.min():.4f} / {var_tmp.max():.4f}") - print(f"[DEBUG GeoMeta] latitude_grid corners: [0,0]={var_tmp[0,0]:.4f}, [-1,-1]={var_tmp[-1,-1]:.4f}") - print(f"[DEBUG GeoMeta] latitude_grid first row mean: {var_tmp[0,:].mean():.4f}, last row mean: {var_tmp[-1,:].mean():.4f}") else: var_tmp = None return var_tmp, "latitude_grid", self.config_options, False @@ -461,10 +454,6 @@ def longitude_grid(self) -> np.ndarray: # Flag to grab entire array for AWS slicing if self.config_options.aws: self.lon_bounds = var_tmp - print(f"[DEBUG GeoMeta] lon_var name: {self.config_options.lon_var}") - print(f"[DEBUG GeoMeta] longitude_grid shape: {var_tmp.shape}") - print(f"[DEBUG GeoMeta] longitude_grid min/max: {var_tmp.min():.4f} / {var_tmp.max():.4f}") - print(f"[DEBUG GeoMeta] longitude_grid corners: [0,0]={var_tmp[0,0]:.4f}, [-1,-1]={var_tmp[-1,-1]:.4f}") else: var_tmp = None diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/regrid.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/regrid.py index 2b3b6336..a3cf22fa 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/regrid.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/regrid.py @@ -1928,14 +1928,6 @@ def regrid_conus_hrrr(input_forcings, config_options, wrf_hydro_geo_meta, mpi_co ) err_handler.check_program_status(config_options, mpi_config) - if mpi_config.rank == 0: - out_data = input_forcings.esmf_field_out.data - n_nodata = int(np.sum(out_data == config_options.globalNdv)) - n_total = out_data.size - print(f"[DEBUG regrid out] var={input_forcings.netcdf_var_names[force_count]}, esmf_field_out shape: {out_data.shape}") - print(f"[DEBUG regrid out] min/max: {out_data.min():.4f} / {out_data.max():.4f}") - print(f"[DEBUG regrid out] no-data count: {n_nodata}/{n_total} ({100*n_nodata/n_total:.1f}%)") - try: input_forcings.regridded_forcings2[ input_forcings.input_map_output[force_count], :, : @@ -11294,10 +11286,6 @@ def check_regrid_status( esmf_lon = wrf_hydro_geo_meta.esmf_grid.get_coords(0) esmf_lon[:, :] = wrf_hydro_geo_meta.longitude_grid wrf_hydro_geo_meta.esmf_lon = esmf_lon - if mpi_config.rank == 0: - print(f"[DEBUG dest grid] populated ESMF destination grid coords") - print(f"[DEBUG dest grid] esmf_lat min/max: {esmf_lat.min():.4f} / {esmf_lat.max():.4f}") - print(f"[DEBUG dest grid] esmf_lon min/max: {esmf_lon.min():.4f} / {esmf_lon.max():.4f}") # If the destination ESMF field hasn't been created, create it here. if not input_forcings.esmf_field_out: @@ -12041,17 +12029,9 @@ def calculate_weights( err_handler.check_program_status(config_options, mpi_config) if mpi_config.rank == 0 and lat_tmp is not None and lon_tmp is not None: - print(f"[DEBUG calc_weights] product={input_forcings.product_name}, lat_var='{lat_var}', lon_var='{lon_var}'") - print(f"[DEBUG calc_weights] raw lat_tmp shape: {lat_tmp.shape}, lon_tmp shape: {lon_tmp.shape}") - print(f"[DEBUG calc_weights] lat_tmp min/max: {lat_tmp.min():.4f} / {lat_tmp.max():.4f}") - print(f"[DEBUG calc_weights] lon_tmp min/max: {lon_tmp.min():.4f} / {lon_tmp.max():.4f}") - print(f"[DEBUG calc_weights] lat_tmp corners: [0,0]={lat_tmp[0,0]:.4f}, [-1,-1]={lat_tmp[-1,-1]:.4f}") - print(f"[DEBUG calc_weights] lon_tmp corners: [0,0]={lon_tmp[0,0]:.4f}, [-1,-1]={lon_tmp[-1,-1]:.4f}") - print(f"[DEBUG calc_weights] input_forcings.ny_global={input_forcings.ny_global}, nx_global={input_forcings.nx_global}") # Normalize source longitudes from 0-360 to -180/+180 to match geo_em geogrid convention if lon_tmp.max() > 180: lon_tmp = np.where(lon_tmp > 180, lon_tmp - 360, lon_tmp) - print(f"[DEBUG calc_weights] lon_tmp normalized to -180/+180: min/max={lon_tmp.min():.4f} / {lon_tmp.max():.4f}") # Scatter global GFS latitude grid to processors.. if mpi_config.rank == 0: @@ -12086,11 +12066,6 @@ def calculate_weights( input_forcings.esmf_lats[:, :] = var_sub_lat_tmp input_forcings.esmf_lons[:, :] = var_sub_lon_tmp - if mpi_config.rank == 0: - print(f"[DEBUG ESMF src] esmf_lats (get_coords(1)) shape={input_forcings.esmf_lats.shape}") - print(f"[DEBUG ESMF src] esmf_lats min/max: {input_forcings.esmf_lats.min():.4f} / {input_forcings.esmf_lats.max():.4f}") - print(f"[DEBUG ESMF src] esmf_lons (get_coords(0)) shape={input_forcings.esmf_lons.shape}") - print(f"[DEBUG ESMF src] esmf_lons min/max: {input_forcings.esmf_lons.min():.4f} / {input_forcings.esmf_lons.max():.4f}") del var_sub_lat_tmp del var_sub_lon_tmp del lat_tmp From 9687736ff9cb877e8179f9e9de353991d0d00c57 Mon Sep 17 00:00:00 2001 From: Lauren Schambach Date: Wed, 5 Aug 2026 17:05:02 -0400 Subject: [PATCH 5/7] when type is gridded, does not delete the esmf mesh --- .../NextGen_Forcings_Engine/bmi_model.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py index 94cca1f0..7e00de78 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py @@ -306,6 +306,15 @@ def initialize(self, config_file: str, output_path: str | None = None) -> None: # LOG.debug(f"self._job_meta type: {type(self._job_meta)}") # Call ESMF mesh creation process if self._mpi_meta.rank == 0: + cat_ids = esmf_creation.create_mesh(self._job_meta) + if self._job_meta.grid_type == "gridded": + # Gridded output regrids directly onto the target grid and has no + # catchments/hydrofabric divides -- ESMF mesh creation (which + # requires a hydrofabric geopackage) does not apply here. See + # consts.py's COASTAL_CONFIG_OVERRIDES in nwm-rte, which + # deliberately leaves "Geopackage" empty for gridded runs. + cat_ids = np.array([], dtype=np.int64) + else: cat_ids = esmf_creation.create_mesh(self._job_meta) cat_count = np.array([ len(cat_ids) if self._mpi_meta.rank == 0 else 0 From 56b4c2c2420f24bdab7ffd44681de4559c1af102 Mon Sep 17 00:00:00 2001 From: Lauren Schambach Date: Wed, 5 Aug 2026 21:39:28 +0000 Subject: [PATCH 6/7] messed up syntax in last commit --- .../NextGen_Forcings_Engine/bmi_model.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py index 7e00de78..c4351bcd 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py @@ -306,16 +306,15 @@ def initialize(self, config_file: str, output_path: str | None = None) -> None: # LOG.debug(f"self._job_meta type: {type(self._job_meta)}") # Call ESMF mesh creation process if self._mpi_meta.rank == 0: - cat_ids = esmf_creation.create_mesh(self._job_meta) - if self._job_meta.grid_type == "gridded": - # Gridded output regrids directly onto the target grid and has no - # catchments/hydrofabric divides -- ESMF mesh creation (which - # requires a hydrofabric geopackage) does not apply here. See - # consts.py's COASTAL_CONFIG_OVERRIDES in nwm-rte, which - # deliberately leaves "Geopackage" empty for gridded runs. - cat_ids = np.array([], dtype=np.int64) - else: - cat_ids = esmf_creation.create_mesh(self._job_meta) + if self._job_meta.grid_type == "gridded": + # Gridded output regrids directly onto the target grid and has no + # catchments/hydrofabric divides -- ESMF mesh creation (which + # requires a hydrofabric geopackage) does not apply here. See + # consts.py's COASTAL_CONFIG_OVERRIDES in nwm-rte, which + # deliberately leaves "Geopackage" empty for gridded runs. + cat_ids = np.array([], dtype=np.int64) + else: + cat_ids = esmf_creation.create_mesh(self._job_meta) cat_count = np.array([ len(cat_ids) if self._mpi_meta.rank == 0 else 0 ], dtype=np.intc) From d70d6ee300447c2df6450f725e94266e286cb524 Mon Sep 17 00:00:00 2001 From: Lauren Schambach Date: Tue, 11 Aug 2026 19:50:52 +0000 Subject: [PATCH 7/7] updates for forcing type gridded so that esmf weight files can be saved and re-used between coastal forecasts --- .../NextGen_Forcings_Engine/core/geoMod.py | 7 +++- .../NextGen_Forcings_Engine/core/regrid.py | 33 +++++++++++++++---- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/geoMod.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/geoMod.py index 5db72a1a..880b852b 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/geoMod.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/geoMod.py @@ -92,7 +92,12 @@ def wrapper(self) -> Any: assert isinstance(post_slice, bool) assert isinstance(name, str) assert isinstance(config_options, ConfigOptions) - assert isinstance(var, np.ndarray) + # var is only populated on rank 0 -- every other rank gets its + # share via scatter_array() below, which is None-safe on + # non-zero ranks (confirmed in parallel.py). Asserting here + # unconditionally rejected every rank but 0. + if self.mpi_config.rank == 0: + assert isinstance(var, np.ndarray) var = self.mpi_config.scatter_array(self, var, config_options) if post_slice: diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/regrid.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/regrid.py index a3cf22fa..354af976 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/regrid.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/regrid.py @@ -12412,13 +12412,32 @@ def calculate_supp_pcp_weights( supplemental_precip.esmf_field_in.data[:] = var_sub_tmp # mpi_config.comm.barrier() - supplemental_precip.regridObj = pt.esmf_regrid_retry_partial( - supplemental_precip.esmf_field_in, - supplemental_precip.esmf_field_out, - src_mask_values=np.array([0]), - regrid_method=ESMF.RegridMethod.BILINEAR, - unmapped_action=ESMF.UnmappedAction.IGNORE, - ) + # Reuse a cached weight file if one exists (see get_weight_file_names / + # load_weight_file, also used by calculate_weights() for InputForcings) -- + # this code path previously always recomputed weights in memory, with no + # on-disk caching, unlike the InputForcings path. + weight_file, _ = get_weight_file_names(mpi_config, config_options, supplemental_precip) + if config_options.weightsDir is not None: + if not os.path.exists(weight_file): + supplemental_precip.regridObj = pt.esmf_regrid_retry_partial( + supplemental_precip.esmf_field_in, + supplemental_precip.esmf_field_out, + src_mask_values=np.array([0]), + regrid_method=ESMF.RegridMethod.BILINEAR, + unmapped_action=ESMF.UnmappedAction.IGNORE, + filename=weight_file, + ) + load_weight_file( + mpi_config, config_options, supplemental_precip, weight_file, element_mode=False + ) + else: + supplemental_precip.regridObj = pt.esmf_regrid_retry_partial( + supplemental_precip.esmf_field_in, + supplemental_precip.esmf_field_out, + src_mask_values=np.array([0]), + regrid_method=ESMF.RegridMethod.BILINEAR, + unmapped_action=ESMF.UnmappedAction.IGNORE, + ) # Run the regridding object on this test dataset. Check the output grid for # any 0 values.