From bbce87e069e09dc33540fb9b027cd23f58005d3a Mon Sep 17 00:00:00 2001 From: apcraig Date: Thu, 30 Apr 2026 12:44:49 -0600 Subject: [PATCH 1/3] Update Derecho inteloneapi port. Update the inteloneapi compiler version on derecho to 2025.2.1. Update the intel and inteloneapi compiler debug options on derecho to better trap and warn. These changes are based on the UFS compiler settings and to help meet UFS requirements. The new settings are FFLAGS += -O0 -check all -check noarg_temp_created -warn -warn noerrors -fp-stack-check -fstack-protector-all -fpe0 -debug -ftrapuv -init=snan,arrays -link_mpi=dbg Fix several issues with uninitialized variables trapped with the new debug flags. For the most part, standalone Icepack results are bit-for-bit with the code changes except for Icepack bgc configurations because Sil and Nit were not initialized properly. Moved the init_forcing_bgc call earlier in the initialization calling sequence because sil_data and nit_data are needed in get_forcing_bgc which is called by init_bgc which is called by init_restart early-ish in the initialization sequencing. get_forcing_bgc is called again later in the initialization sequence and this seems redundant. It might be worth reviewing the bgc initialization. This was tested in standalone Icepack and standalone CICE on derecho with intel, inteloneapi, and gnu with the following results - Icepack derecho inteloneapi results are NOT bit-for-bit due to the updated compiler version on derecho with inteloneapi - Icepack derecho intel results are bit-for-bit for debug tests but are failing regression testing because fill values have changed on restart comparison. The log files are bit-for-bit. - Icepack bgc results are NOT bit-for-bit for all machines and compilers due to fixes in the bgc initialization in the Icepack driver. - These Icepack changes applied to CICE are all bit-for-bit in CICE, any bit-for-bit changes in standalone Icepack were implemented in the Icepack driver, not in columnphysics code. --- columnphysics/icepack_algae.F90 | 3 ++ columnphysics/icepack_therm_vertical.F90 | 3 +- columnphysics/icepack_zbgc.F90 | 2 + configuration/driver/icedrv_InitMod.F90 | 7 +-- configuration/driver/icedrv_flux.F90 | 43 +++++++++++-------- configuration/driver/icedrv_init.F90 | 13 ++++++ configuration/driver/icedrv_init_column.F90 | 8 ++++ .../scripts/machines/Macros.derecho_intel | 3 +- .../machines/Macros.derecho_inteloneapi | 3 +- .../scripts/machines/env.derecho_inteloneapi | 16 +++---- 10 files changed, 69 insertions(+), 32 deletions(-) diff --git a/columnphysics/icepack_algae.F90 b/columnphysics/icepack_algae.F90 index a39b4a647..191dbc425 100644 --- a/columnphysics/icepack_algae.F90 +++ b/columnphysics/icepack_algae.F90 @@ -238,6 +238,9 @@ subroutine zbio (dt, & character(len=*),parameter :: subname='(zbio)' + upNOn(:,:) = c0 + upNHn(:,:) = c0 + grow_alg(:,:) = c0 zbgc_snown(:) = c0 zbgc_atmn (:) = c0 flux_bion (:) = c0 diff --git a/columnphysics/icepack_therm_vertical.F90 b/columnphysics/icepack_therm_vertical.F90 index 00b3756d6..1a2be844a 100644 --- a/columnphysics/icepack_therm_vertical.F90 +++ b/columnphysics/icepack_therm_vertical.F90 @@ -1623,7 +1623,7 @@ subroutine thickness_changes (dt, yday, & ! Convert snow to ice if snow lies below freeboard. !------------------------------------------------------------------- - if (ktherm /= 2) & + if (ktherm /= 2) then call freeboard (snoice, & hin, hsn, & zqin, zqsn, & @@ -1631,6 +1631,7 @@ subroutine thickness_changes (dt, yday, & dsnow, & massice, massliq) if (icepack_warnings_aborted(subname)) return + endif !------------------------------------------------------------------- ! Update snow mass tracers for uneven layers diff --git a/columnphysics/icepack_zbgc.F90 b/columnphysics/icepack_zbgc.F90 index 7ee662763..6852b0dd3 100644 --- a/columnphysics/icepack_zbgc.F90 +++ b/columnphysics/icepack_zbgc.F90 @@ -770,6 +770,8 @@ subroutine icepack_biogeochemistry(dt, & kavg = c0 bphi_o = c0 sloss = c0 + upNO = c0 + upNH = c0 !----------------------------------------------------------------- ! brine dynamics diff --git a/configuration/driver/icedrv_InitMod.F90 b/configuration/driver/icedrv_InitMod.F90 index 0785ab7c3..114192da7 100644 --- a/configuration/driver/icedrv_InitMod.F90 +++ b/configuration/driver/icedrv_InitMod.F90 @@ -105,6 +105,10 @@ subroutine icedrv_initialize endif call init_fsd + call icepack_query_parameters(skl_bgc_out=skl_bgc) + call icepack_query_parameters(z_tracers_out=z_tracers) + if (skl_bgc .or. z_tracers) call init_forcing_bgc !cn + call calendar(time) ! determine the initial date call init_state ! initialize the ice state @@ -130,8 +134,6 @@ subroutine icedrv_initialize !-------------------------------------------------------------------- ! coupler communication or forcing data initialization !-------------------------------------------------------------------- - call icepack_query_parameters(skl_bgc_out=skl_bgc) - call icepack_query_parameters(z_tracers_out=z_tracers) call icepack_query_parameters(wave_spec_out=wave_spec) call icepack_query_tracer_flags(tr_snow_out=tr_snow) call icepack_query_tracer_flags(tr_aero_out=tr_aero) @@ -142,7 +144,6 @@ subroutine icedrv_initialize file=__FILE__,line= __LINE__) call init_forcing ! initialize forcing (standalone) - if (skl_bgc .or. z_tracers) call init_forcing_bgc !cn if (tr_fsd .and. wave_spec) call get_wave_spec ! wave spectrum in ice if (precalc_forc) then call get_forcing(istep) ! precalculated arrays are indexed by istep diff --git a/configuration/driver/icedrv_flux.F90 b/configuration/driver/icedrv_flux.F90 index b05092869..bd5b50558 100644 --- a/configuration/driver/icedrv_flux.F90 +++ b/configuration/driver/icedrv_flux.F90 @@ -595,6 +595,8 @@ subroutine init_coupler_flux wind (:) = sqrt(uatm(:)**2 + vatm(:)**2) ! wind speed, (m/s) Cdn_atm(:) = (vonkar/log(zref/iceruf)) & * (vonkar/log(zref/iceruf)) ! atmo drag for RASM + rdg_conv(:) = c0 + rdg_shear(:) = c0 end subroutine init_coupler_flux @@ -722,8 +724,15 @@ subroutine init_history_therm dpnd_freebd (:) = c0 dpnd_initial(:) = c0 dpnd_dlid (:) = c0 + dpnd_flushn (:,:) = c0 + dpnd_exponn (:,:) = c0 + dpnd_freebdn (:,:) = c0 + dpnd_initialn(:,:) = c0 + dpnd_dlidn (:,:) = c0 dpnd_melt (:) = c0 dpnd_ridge (:) = c0 + frz_onset (:) = c0 + mlt_onset (:) = c0 ! drag coefficients are computed prior to the atmo_boundary call, ! during the thermodynamics section @@ -731,24 +740,22 @@ subroutine init_history_therm Cdn_atm(:) = (vonkar/log(zref/iceruf)) & * (vonkar/log(zref/iceruf)) ! atmo drag for RASM - if (formdrag) then - Cdn_atm_rdg (:) = c0 - Cdn_atm_ratio(:)= c0 - Cdn_atm_floe(:) = c0 - Cdn_atm_pond(:) = c0 - Cdn_atm_skin(:) = c0 - Cdn_ocn_skin(:) = c0 - Cdn_ocn_keel(:) = c0 - Cdn_ocn_floe(:) = c0 - hfreebd (:) = c0 - hdraft (:) = c0 - hridge (:) = c0 - distrdg (:) = c0 - hkeel (:) = c0 - dkeel (:) = c0 - lfloe (:) = c0 - dfloe (:) = c0 - endif + Cdn_atm_rdg (:) = c0 + Cdn_atm_ratio(:)= c0 + Cdn_atm_floe(:) = c0 + Cdn_atm_pond(:) = c0 + Cdn_atm_skin(:) = c0 + Cdn_ocn_skin(:) = c0 + Cdn_ocn_keel(:) = c0 + Cdn_ocn_floe(:) = c0 + hfreebd (:) = c0 + hdraft (:) = c0 + hridge (:) = c0 + distrdg (:) = c0 + hkeel (:) = c0 + dkeel (:) = c0 + lfloe (:) = c0 + dfloe (:) = c0 end subroutine init_history_therm diff --git a/configuration/driver/icedrv_init.F90 b/configuration/driver/icedrv_init.F90 index 133683dd6..2566f292e 100644 --- a/configuration/driver/icedrv_init.F90 +++ b/configuration/driver/icedrv_init.F90 @@ -1158,6 +1158,8 @@ subroutine init_state use icedrv_state, only: trcr_depend, aicen, trcrn, vicen, vsnon use icedrv_state, only: aice0, aice, vice, vsno, trcr, aice_init use icedrv_state, only: n_trcr_strata, nt_strata, trcr_base + use icedrv_state, only: uvel,vvel,divu,shear,strength + use icedrv_arrays_column, only: dhsn, ffracn integer (kind=int_kind) :: & i , & ! horizontal indes @@ -1270,6 +1272,9 @@ subroutine init_state do it = 1, ntrcr ! mask for base quantity on which tracers are carried + trcr_base(it,1) = c0 + trcr_base(it,2) = c0 + trcr_base(it,3) = c0 if (trcr_depend(it) == 0) then ! area trcr_base(it,1) = c1 elseif (trcr_depend(it) == 1) then ! ice volume @@ -1317,6 +1322,14 @@ subroutine init_state aicen (:,:), trcrn(:,:,:), & vicen (:,:), vsnon(:,:)) + uvel(:) = c0 + vvel(:) = c0 + divu(:) = c0 + shear(:) = c0 + strength(:) = c0 + dhsn(:,:) = c0 + ffracn(:,:) = c0 + !----------------------------------------------------------------- ! compute aggregate ice state and open water area !----------------------------------------------------------------- diff --git a/configuration/driver/icedrv_init_column.F90 b/configuration/driver/icedrv_init_column.F90 index 87d0b9da5..81082c39e 100644 --- a/configuration/driver/icedrv_init_column.F90 +++ b/configuration/driver/icedrv_init_column.F90 @@ -199,6 +199,11 @@ subroutine init_shortwave alidrn(i,n) = c0 alvdfn(i,n) = c0 alidfn(i,n) = c0 + albicen(i,n) = c0 + albsnon(i,n) = c0 + albpndn(i,n) = c0 + snowfracn(i,n) = c0 + apeffn(i,n) = c0 fswsfcn(i,n) = c0 fswintn(i,n) = c0 fswthrun(i,n) = c0 @@ -366,6 +371,7 @@ subroutine init_bgc() use icedrv_arrays_column, only: zfswin, trcrn_sw use icedrv_arrays_column, only: ocean_bio_all, ice_bio_net, snow_bio_net use icedrv_arrays_column, only: bphi, iDi, bTiz, iki + use icedrv_arrays_column, only: Zoo use icedrv_calendar, only: istep1 use icedrv_system, only: icedrv_system_abort use icedrv_flux, only: sss, nit, amm, sil, dmsp, dms, algalN, & @@ -426,6 +432,8 @@ subroutine init_bgc() ! biogeochemistry initialization !----------------------------------------------------------------- + Zoo(:,:,:) = c0 + !----------------------------------------------------------------- ! Initial Ocean Values if not coupled to the ocean bgc !----------------------------------------------------------------- diff --git a/configuration/scripts/machines/Macros.derecho_intel b/configuration/scripts/machines/Macros.derecho_intel index d8d5efc78..113258fea 100644 --- a/configuration/scripts/machines/Macros.derecho_intel +++ b/configuration/scripts/machines/Macros.derecho_intel @@ -12,7 +12,8 @@ FFLAGS := -fp-model precise -convert big_endian -assume byterecl -ftz -trace FFLAGS_NOOPT:= -O0 ifeq ($(ICE_BLDDEBUG), true) - FFLAGS += -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created -link_mpi=dbg +# FFLAGS += -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created -link_mpi=dbg + FFLAGS += -O0 -check all -check noarg_temp_created -warn -warn noerrors -fp-stack-check -fstack-protector-all -fpe0 -debug -ftrapuv -init=snan,arrays -link_mpi=dbg # FFLAGS += -O0 -g -check all -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -link_mpi=dbg -stand f08 # FFLAGS += -O0 -g -check all -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -init=snan,arrays -link_mpi=dbg else diff --git a/configuration/scripts/machines/Macros.derecho_inteloneapi b/configuration/scripts/machines/Macros.derecho_inteloneapi index e073f8c21..78d9a8a46 100644 --- a/configuration/scripts/machines/Macros.derecho_inteloneapi +++ b/configuration/scripts/machines/Macros.derecho_inteloneapi @@ -13,7 +13,8 @@ FFLAGS_NOOPT:= -O0 ifeq ($(ICE_BLDDEBUG), true) # -check uninit is needed on the ld step but it still throws errors in 2023.* and 2024.0.*, likely compiler bug - FFLAGS += -O0 -g -check bounds -check pointers -fpe0 -check noarg_temp_created -link_mpi=dbg +# FFLAGS += -O0 -g -check bounds -check pointers -fpe0 -check noarg_temp_created -link_mpi=dbg + FFLAGS += -O0 -check all -check noarg_temp_created -warn -warn noerrors -fp-stack-check -fstack-protector-all -fpe0 -debug -ftrapuv -init=snan,arrays -link_mpi=dbg # FFLAGS += -O0 -g -check all -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -link_mpi=dbg -stand f08 # FFLAGS += -O0 -g -check all -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -init=snan,arrays -link_mpi=dbg else diff --git a/configuration/scripts/machines/env.derecho_inteloneapi b/configuration/scripts/machines/env.derecho_inteloneapi index 70c98c68b..ebbe6ebb5 100644 --- a/configuration/scripts/machines/env.derecho_inteloneapi +++ b/configuration/scripts/machines/env.derecho_inteloneapi @@ -10,14 +10,14 @@ if ("$inp" != "-nomodules") then source ${MODULESHOME}/init/csh module --force purge -module load ncarenv/23.09 +module load ncarenv/25.10 module load craype -module load intel-oneapi/2023.2.1 -module load ncarcompilers -#module load cray-mpich/8.1.25 -#module load hdf5/1.12.2 -module load netcdf/4.9.2 -module load cray-libsci/23.09.1.1 +module load intel/2025.2.1 +#module load mkl/2023.3.0 +module load ncarcompilers/1.2.0 +#module load cray-mpich/8.1.32 +module load netcdf/4.9.3 +module load cray-libsci/25.03.0 # For perftools with mpiexec # module load perftools-base @@ -38,7 +38,7 @@ setenv OMP_STACKSIZE 64M setenv ICE_MACHINE_MACHNAME derecho setenv ICE_MACHINE_MACHINFO "HPE Cray EX Milan Slingshot 11" setenv ICE_MACHINE_ENVNAME inteloneapi -setenv ICE_MACHINE_ENVINFO "ifx 2023.2.0 20230721, netcdf4.9.2" +setenv ICE_MACHINE_ENVINFO "ifx 2025.2.1 20250806, netcdf4.9.3" setenv ICE_MACHINE_MAKE gmake setenv ICE_MACHINE_WKDIR /glade/derecho/scratch/$user/ICEPACK_RUNS setenv ICE_MACHINE_INPUTDATA /glade/campaign/cesm/development/pcwg From bf8d705d883e4fdeec64ca4dd6b429924f553325 Mon Sep 17 00:00:00 2001 From: apcraig Date: Thu, 30 Apr 2026 14:55:36 -0600 Subject: [PATCH 2/3] Update time_forc initial value for better backward compatibility with regression testing. --- configuration/driver/icedrv_calendar.F90 | 1 + 1 file changed, 1 insertion(+) diff --git a/configuration/driver/icedrv_calendar.F90 b/configuration/driver/icedrv_calendar.F90 index e5e02678e..0b338e124 100644 --- a/configuration/driver/icedrv_calendar.F90 +++ b/configuration/driver/icedrv_calendar.F90 @@ -120,6 +120,7 @@ subroutine init_calendar istep = 0 ! local timestep number time0=istep0*dt ! start time time=istep0*dt ! s + time_forc=c0 ! forcing time, not used anymore yday=c0 ! absolute day number mday=0 ! day of the month month=0 ! month From 0f60d6b614cd14707ab13b69f893a7ed2a4308c4 Mon Sep 17 00:00:00 2001 From: apcraig Date: Fri, 1 May 2026 12:39:53 -0600 Subject: [PATCH 3/3] Update documentation --- configuration/driver/icedrv_calendar.F90 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/configuration/driver/icedrv_calendar.F90 b/configuration/driver/icedrv_calendar.F90 index 0b338e124..aef174d21 100644 --- a/configuration/driver/icedrv_calendar.F90 +++ b/configuration/driver/icedrv_calendar.F90 @@ -120,7 +120,8 @@ subroutine init_calendar istep = 0 ! local timestep number time0=istep0*dt ! start time time=istep0*dt ! s - time_forc=c0 ! forcing time, not used anymore + time_forc=c0 ! forcing time, not used anymore, + ! maintained because it's written to the restart file yday=c0 ! absolute day number mday=0 ! day of the month month=0 ! month