From 677ab5d798d74c6b5628dfe10b26a08627570f41 Mon Sep 17 00:00:00 2001 From: Chris Wood Date: Wed, 14 Jul 2021 16:55:15 +0100 Subject: [PATCH 01/17] Fixing removal of directory... --- Singularity.nemo | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Singularity.nemo b/Singularity.nemo index 3698269..3bc534d 100644 --- a/Singularity.nemo +++ b/Singularity.nemo @@ -140,7 +140,7 @@ From: ubuntu:20.04 fi cd /opt/mpi - rm -r openmpi # removes the 303 mb directory + rm -r mpi # removes the 303 mb directory ## # compile HDF5 libraries From 726791e5d2c41b66d2ffb4f822209c77a11a8727 Mon Sep 17 00:00:00 2001 From: Chris Wood Date: Mon, 9 Aug 2021 20:06:20 +0100 Subject: [PATCH 02/17] MPICH --- input_files/NEMO_in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/input_files/NEMO_in b/input_files/NEMO_in index ad883d3..b1630d6 100644 --- a/input_files/NEMO_in +++ b/input_files/NEMO_in @@ -3,4 +3,4 @@ NEMO_VERSION=4.0.4 # Check that VERSION is 4.0.[2-6], 4.0_HEAD or tr XIOS_REVISION= # Use default value if empty NEMO_COMPONENTS='OCE' CPP_KEYS= -MPI= # Which MPI implementation to use; valid options are currently MPICH (which installs v) or OMPI (which installs v4.1.1) +MPI=MPICH # Which MPI implementation to use; valid options are currently MPICH (which installs v) or OMPI (which installs v4.1.1) From 9833235647935a200bb777963f3b5197d98fd9cd Mon Sep 17 00:00:00 2001 From: Chris Wood Date: Wed, 8 Sep 2021 18:16:18 +0100 Subject: [PATCH 03/17] apt update needs to be earlier than any apt install --- Singularity.nemo | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Singularity.nemo b/Singularity.nemo index 962e719..893d192 100644 --- a/Singularity.nemo +++ b/Singularity.nemo @@ -55,9 +55,9 @@ From: ubuntu:20.04 apt install -y locales #locales-all locale-gen en_GB en_GB.UTF-8 # en_US en_US.UTF-8 + apt update apt install -y software-properties-common add-apt-repository universe - apt update # n.b. # - libcurl4-openssl-dev also installs libcurl4 (as does curl, if not already installed) From 43918199eb13475efeb2c5d920ce3806d69a1289 Mon Sep 17 00:00:00 2001 From: Chris Wood Date: Wed, 8 Sep 2021 18:17:36 +0100 Subject: [PATCH 04/17] No longer using crudini --- Singularity.nemo | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Singularity.nemo b/Singularity.nemo index 893d192..485ab56 100644 --- a/Singularity.nemo +++ b/Singularity.nemo @@ -77,8 +77,7 @@ From: ubuntu:20.04 zlib1g-dev \ libibverbs-dev \ libpmix-dev \ - libslurm-dev \ - crudini + libslurm-dev update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 --slave /usr/bin/g++ g++ /usr/bin/g++-10 update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-10 100 From 14ec954ad83ae34ff43440ec0b3f99e7f55088e6 Mon Sep 17 00:00:00 2001 From: Chris Wood Date: Wed, 8 Sep 2021 18:16:18 +0100 Subject: [PATCH 05/17] apt update needs to be earlier than any apt install --- Singularity.nemo | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Singularity.nemo b/Singularity.nemo index 962e719..893d192 100644 --- a/Singularity.nemo +++ b/Singularity.nemo @@ -55,9 +55,9 @@ From: ubuntu:20.04 apt install -y locales #locales-all locale-gen en_GB en_GB.UTF-8 # en_US en_US.UTF-8 + apt update apt install -y software-properties-common add-apt-repository universe - apt update # n.b. # - libcurl4-openssl-dev also installs libcurl4 (as does curl, if not already installed) From 4c21f4cb4c3503e8e31efbd6ee9ab006e378d6f4 Mon Sep 17 00:00:00 2001 From: Chris Wood Date: Wed, 8 Sep 2021 18:17:36 +0100 Subject: [PATCH 06/17] No longer using crudini --- Singularity.nemo | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Singularity.nemo b/Singularity.nemo index 893d192..845ca7d 100644 --- a/Singularity.nemo +++ b/Singularity.nemo @@ -77,8 +77,7 @@ From: ubuntu:20.04 zlib1g-dev \ libibverbs-dev \ libpmix-dev \ - libslurm-dev \ - crudini + libslurm-dev update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 --slave /usr/bin/g++ g++ /usr/bin/g++-10 update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-10 100 @@ -102,8 +101,6 @@ From: ubuntu:20.04 ## # compiling openmpi ## - - # MPI=`crudini --get /input_files/NEMO_in DEFAULT MPI` # For some reason this yanks the whole line including comments . /input_files/NEMO_in From 18fa1e851937e76297bc996f55abb4bd3e814539 Mon Sep 17 00:00:00 2001 From: Chris Wood Date: Thu, 9 Sep 2021 10:54:23 +0100 Subject: [PATCH 07/17] Minor typos --- Singularity.nemo | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Singularity.nemo b/Singularity.nemo index 845ca7d..47fa828 100644 --- a/Singularity.nemo +++ b/Singularity.nemo @@ -86,6 +86,13 @@ From: ubuntu:20.04 # this decreases the size of the container apt clean + ## + # Read in the environment variables that can be user-defined + ## + + . /input_files/NEMO_in + + ## # make user `nemo` - mpiexec in hdf5 `make check` complains if run as root (although we're not running it at the moment) ## @@ -229,8 +236,6 @@ From: ubuntu:20.04 LD_LIBRARY_PATH=/opt/hdf5/install/lib:$LD_LIBRARY_PATH cd $WORK_DIR - - . /input_files/NEMO_in chmod u+x /input_files/setup_nemo /input_files/setup_nemo -x /nemo -w /nemo -s /nemo/AMM7_lite -m singularity -v $NEMO_VERSION -c gnu From e662d1108922d305e749954c1fae55ce40806a97 Mon Sep 17 00:00:00 2001 From: Chris Wood Date: Tue, 9 Nov 2021 16:36:38 +0000 Subject: [PATCH 08/17] Adding ARCHER2-recommended build settings for MPICH --- Singularity.nemo | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/Singularity.nemo b/Singularity.nemo index 47fa828..123de22 100644 --- a/Singularity.nemo +++ b/Singularity.nemo @@ -56,13 +56,14 @@ From: ubuntu:20.04 locale-gen en_GB en_GB.UTF-8 # en_US en_US.UTF-8 apt update - apt install -y software-properties-common + apt install -y software-properties-common build-essential add-apt-repository universe # n.b. # - libcurl4-openssl-dev also installs libcurl4 (as does curl, if not already installed) # - zlib already installed - apt install -y python \ + DEBIAN_FRONTEND=noninteractive && apt install -y \ + python \ subversion \ wget \ git \ @@ -134,7 +135,7 @@ From: ubuntu:20.04 rm mpich-3.4.2.tar.gz cd mpi - ./configure CC=gcc CXX=g++ FC=gfortran --prefix=/opt/mpi/install FFLAGS=-fallow-argument-mismatch + ./configure CC=gcc CXX=g++ FC=gfortran --prefix=/opt/mpi FFLAGS=-fallow-argument-mismatch --with-device=ch3:nemesis:ofi make make install @@ -146,7 +147,7 @@ From: ubuntu:20.04 rm openmpi-4.1.1.tar.bz2 cd mpi - ./configure CC=gcc CXX=g++ FC=gfortran --enable-mpi1-compatibility --prefix=/opt/mpi/install --with-verbs --with-slurm --with-pmix + ./configure CC=gcc CXX=g++ FC=gfortran --enable-mpi1-compatibility --prefix=/opt/mpi --with-verbs --with-slurm --with-pmix make make install else @@ -173,7 +174,7 @@ From: ubuntu:20.04 cd hdf # n.b. --enable-fortran specifically not needed (https://www.unidata.ucar.edu/software/netcdf/docs/getting_and_building_netcdf.html#build_default) - CC=/opt/mpi/install/bin/mpicc ./configure --prefix=${H5DIR} --enable-hl --enable-parallel --with-default-api-version=v18 + CC=/opt/mpi/bin/mpicc ./configure --prefix=${H5DIR} --enable-hl --enable-parallel --with-default-api-version=v18 make # maybe we shouldn't bother with make check - it complains about a lot and we're not actually building on a parallel file system # so some of the checks are totally irrelevant @@ -199,7 +200,7 @@ From: ubuntu:20.04 cd netcdf NCDIR=/opt/netcdf/install - CC=/opt/mpi/install/bin/mpicc FC=/opt/mpi/install/bin/mpifort CPPFLAGS="-I${H5DIR}/include" LDFLAGS="-L${H5DIR}/lib" ./configure --disable-shared --prefix=$NCDIR + CC=/opt/mpi/bin/mpicc FC=/opt/mpi/bin/mpifort CPPFLAGS="-I${H5DIR}/include" LDFLAGS="-L${H5DIR}/lib" ./configure --disable-shared --prefix=$NCDIR make # make check fails; maybe we should avoid running it (only 1 response from a question on the netcdf mailing list, saying it is really finicky) # make check @@ -215,7 +216,7 @@ From: ubuntu:20.04 cd netcdf NFDIR=/opt/netcdf/install - CC=/opt/mpi/install/bin/mpicc FC=/opt/mpi/install/bin/mpifort CPPFLAGS="-I${NCDIR}/include -I${H5DIR}/include" LDFLAGS="-L${NCDIR}/lib -L${H5DIR}/lib" LIBS="-lhdf5 -lhdf5_hl -lcurl" ./configure --prefix=$NFDIR --disable-shared + CC=/opt/mpi/bin/mpicc FC=/opt/mpi/bin/mpifort CPPFLAGS="-I${NCDIR}/include -I${H5DIR}/include" LDFLAGS="-L${NCDIR}/lib -L${H5DIR}/lib" LIBS="-lhdf5 -lhdf5_hl -lcurl" ./configure --prefix=$NFDIR --disable-shared make make install cd .. @@ -225,14 +226,14 @@ From: ubuntu:20.04 # Now we can install NEMO ## - ln -s /opt/mpi/install/bin/mpif90 /usr/bin/mpif90 + ln -s /opt/mpi/bin/mpif90 /usr/bin/mpif90 WORK_DIR=/nemo >> $SINGULARITY_ENVIRONMENT mkdir $WORK_DIR cd $WORK_DIR chown nemo:nemo -R $WORK_DIR - PATH=$PATH:/opt/mpi/install/bin:/opt/hdf5/install/bin + PATH=$PATH:/opt/mpi/bin:/opt/hdf5/install/bin LD_LIBRARY_PATH=/opt/hdf5/install/lib:$LD_LIBRARY_PATH cd $WORK_DIR From a05e6f8bb869244c1f5b690ca817b40682a0b5d4 Mon Sep 17 00:00:00 2001 From: Chris Wood Date: Wed, 10 Nov 2021 12:52:11 +0000 Subject: [PATCH 09/17] Fixing file permissions --- Singularity.nemo | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Singularity.nemo b/Singularity.nemo index 47fa828..7e48226 100644 --- a/Singularity.nemo +++ b/Singularity.nemo @@ -265,7 +265,9 @@ From: ubuntu:20.04 # minimal possible permissions: directories executable; binaries read + executable (need to be readable to that library headers can be read!); everything else only readable for g+o chmod -Rv 644 /opt find /opt -type d -print0 | xargs -0 chmod -v 511 - find /opt -type f -exec file -i {} + | grep ":[^:]*binary[^:]*$" | sed 's/^\(.*\):[^:]*$/\1/' | xargs chmod -v 555 + # is the sed command a bit overkill, rather than other option? It change pdfs and zip files to executable which isn't idea... (https://stackoverflow.com/questions/36947061/see-if-binary-files-exist-in-subfolders) + #find /opt -type f -exec file -i {} + | grep ":[^:]*binary[^:]*$" | sed 's/^\(.*\):[^:]*$/\1/' | xargs chmod -v 555 + find /opt -type f -exec file -i {} + | grep "text/x-c\|text/x-shellscript\|binary" | sed 's/^\(.*\):[^:]*$/\1/' | xargs chmod -v 555 rm -rf /var/lib/apt/lists/* /var/lib/dpkg/info/* From ee2f6e171eda86ef5de106510ddb1fcdde2e393b Mon Sep 17 00:00:00 2001 From: Chris Wood Date: Wed, 10 Nov 2021 12:52:11 +0000 Subject: [PATCH 10/17] Fixing file permissions --- Singularity.nemo | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Singularity.nemo b/Singularity.nemo index 123de22..2dca528 100644 --- a/Singularity.nemo +++ b/Singularity.nemo @@ -56,7 +56,7 @@ From: ubuntu:20.04 locale-gen en_GB en_GB.UTF-8 # en_US en_US.UTF-8 apt update - apt install -y software-properties-common build-essential + apt install -y software-properties-common #build-essential add-apt-repository universe # n.b. @@ -266,7 +266,9 @@ From: ubuntu:20.04 # minimal possible permissions: directories executable; binaries read + executable (need to be readable to that library headers can be read!); everything else only readable for g+o chmod -Rv 644 /opt find /opt -type d -print0 | xargs -0 chmod -v 511 - find /opt -type f -exec file -i {} + | grep ":[^:]*binary[^:]*$" | sed 's/^\(.*\):[^:]*$/\1/' | xargs chmod -v 555 + # is the sed command a bit overkill, rather than other option? It change pdfs and zip files to executable which isn't idea... (https://stackoverflow.com/questions/36947061/see-if-binary-files-exist-in-subfolders) + #find /opt -type f -exec file -i {} + | grep ":[^:]*binary[^:]*$" | sed 's/^\(.*\):[^:]*$/\1/' | xargs chmod -v 555 + find /opt -type f -exec file -i {} + | grep "text/x-c\|text/x-shellscript\|binary" | sed 's/^\(.*\):[^:]*$/\1/' | xargs chmod -v 555 rm -rf /var/lib/apt/lists/* /var/lib/dpkg/info/* From 99d471527fd874b74559e8127a018204371660ce Mon Sep 17 00:00:00 2001 From: Chris Wood Date: Tue, 11 Jan 2022 10:24:53 +0000 Subject: [PATCH 11/17] adding runscripts --- runscripts/rs_mpich_bind | 169 +++++++++++++++++++++++++++++++++ runscripts/rs_mpich_bind_clean | 169 +++++++++++++++++++++++++++++++++ runscripts/rs_mpich_hybrid | 163 +++++++++++++++++++++++++++++++ 3 files changed, 501 insertions(+) create mode 100644 runscripts/rs_mpich_bind create mode 100644 runscripts/rs_mpich_bind_clean create mode 100644 runscripts/rs_mpich_hybrid diff --git a/runscripts/rs_mpich_bind b/runscripts/rs_mpich_bind new file mode 100644 index 0000000..ae29486 --- /dev/null +++ b/runscripts/rs_mpich_bind @@ -0,0 +1,169 @@ +#!/bin/bash +#SBATCH --job-name=1Xg_95N +#SBATCH --time=00:20:00 +#SBATCH --account=ecseab13 +#SBATCH --partition=standard +#SBATCH --qos=short +#SBATCH --reservation=shortqos + +#SBATCH --partition=standard +#SBATCH --nodes=1 +#SBATCH --ntasks=57 +#SBATCH --ntasks-per-node=57 +#SBATCH --ntasks-per-core=1 + +#SBATCH hetjob +#SBATCH --partition=standard +#SBATCH --nodes=1 +#SBATCH --ntasks=39 +#SBATCH --ntasks-per-node=39 +#SBATCH --ntasks-per-core=1 + +#module load cpe/21.03 +#module load cray-hdf5-parallel +#module load cray-netcdf-hdf5parallel +#export LD_LIBRARY_PATH=/work/ecseab13/shared/install/lib:${LD_LIBRARY_PATH} +#export LD_LIBRARY_PATH=/opt/cray/pe/libsci/20.10.1.2/GNU/9.1/x86_64/lib:${LD_LIBRARY_PATH} + +#module -s restore /work/n01/shared/acc/n01_modules/ucx_env + +export OMP_NUM_THREADS=1 +#export MPI_DIR=/opt/cray/pe/mpich/8.1.3/ofi/gnu/9.1/lib-abi-mpich +# Set the LD_LIBRARY_PATH environment variable within the Singularity container +# to ensure that it used the correct MPI libraries +#export SINGULARITYENV_LD_LIBRARY_PATH=/usr/lib:/lib/x86_64-linux-gnu:/opt/hdf5/install/lib:/opt/cray/pe/mpich/8.1.3/ofi/gnu/9.1/lib-abi-mpich:/opt/cray/pe/pmi/6.0.7/lib:/opt/cray/libfabric/1.11.0.0.233/lib64:/usr/lib64/host: + +export SINGULARITYENV_PREPEND_PATH=/usr/bin/host + +#export SINGULARITYENV_LD_LIBRARY_PATH=\ +#/opt/hdf5/install/lib:\ +#/opt/cray/pe/mpich/8.1.3/ofi/gnu/9.1/lib-abi-mpich:\ +#/usr/lib/x86_64-linux-gnu/libibverbs:\ +#/opt/cray/pe/pmi/6.0.7/lib:\ +#/opt/cray/libfabric/1.11.0.0.233/lib64 + +#SINGULARITYENV_LD_LIBRARY_PATH=\ +#/opt/hdf5/install/lib:\ +#/opt/cray/pe/mpich/8.1.3/ofi/gnu/9.1/lib-abi-mpich:\ +#/usr/lib/x86_64-linux-gnu/libibverbs:\ +#/opt/cray/pe/pmi/6.0.7/lib:\ +#/opt/cray/libfabric/1.11.0.0.233/lib64:\ +#/usr/lib64/host:\ +#/lib/x86_64-linux-gnu:\ +#/.singularity.d/libs + +SINGULARITYENV_LD_LIBRARY_PATH=\ +/opt/cray/pe/hdf5-parallel/1.12.0.2/crayclang/9.1/lib:\ +/opt/cray/pe/mpich/8.1.3/ucx/cray/9.1/lib-abi-mpich:\ +/opt/cray/pe/pmi/6.0.7/lib:\ +/opt/cray/libfabric/1.11.0.0.233/lib64:\ +/usr/lib64/host:\ + +#export SINGULARITYENV_LD_LIBRARY_PATH=\ +#/opt/libs:\ +#/usr/lib:\ +#/opt/mpi/install/lib:\ +#/lib/x86_64-linux-gnu:\ +#/opt/hdf5/install/lib:\ +#/opt/cray/pe/mpich/8.1.3/ofi/gnu/9.1/lib-abi-mpich:\ +#/opt/cray/pe/pmi/6.0.7/lib:\ +#/opt/cray/libfabric/1.11.0.0.233/lib64:\ +#/lib/x86_64-linux-gnu:\ +#/usr/lib64/host + +#export SINGULARITYENV_LD_LIBRARY_PATH=/opt/libs:/usr/lib:/opt/mpi/install/lib:/lib/x86_64-linux-gnu:/opt/hdf5/install/lib:/lib/x86_64-linux-gnu: + + +# Set the options for the Singularity executable +# This makes sure the locations with Cray Slingshot interconnect libraries are available +#singopts="-B /etc/libibverbs.d,/opt/cray,/usr/lib64:/usr/lib64/host,/usr/lib64/tcl,/var/spool/slurmd/mpi_cray_shasta" +singopts="-B /etc/libibverbs.d,/opt/cray,/usr/lib64:/usr/lib64/host,/usr/lib64/tcl,/var/spool/slurmd/mpi_cray_shasta" + +#singopts="-B /etc/libibverbs.d,/opt/cray,/usr/lib64:/usr/lib64/host,/usr/lib64/tcl" + + +# Set the LD_LIBRARY_PATH environment variable within the Singularity container +# to ensure that it used the correct MPI libraries +#singopts="-B /opt/cray,/usr/lib64:/usr/lib64/host,/usr/lib64/tcl,/var/spool/slurmd/mpi_cray_shasta" + +#=============================================================== +# LAUNCH JOB +#=============================================================== +# 5x6 25 61x65 +# 7x8 47 45x49 +# 11x11 95 29x36 +# 14x16 184 22x26 +# 22x23 370 16x19 +#=============================================================== +echo `date` : Launch Job +numactl --hardware + +sed "s/XXX_JPNI_XXX/11/g" EXP00/namelist_cfg_template > tmp_out +sed "s/XXX_JPNJ_XXX/11/g" tmp_out > EXP00/namelist_cfg + +#rm nemo.sif +#ln -s nemo-mpich-O0.sif nemo.sif + +cat > myscript_wrapper.sh << EOFB +#!/bin/ksh +# +#set -A map '--mca orte_base_help_aggregate 0 singularity run /work/ecseab13/ecseab13/jhcones/testing/sing3/nemo.sif xios EXP00' '--mca orte_base_help_aggregate 0 singularity run -B /etc/libibverbs.d /work/ecseab13/ecseab13/jhcones/testing/sing3/nemo.sif nemo EXP00' +map[0]='singularity run /work/ecseab13/ecseab13/ccwcones/sing3/nemo.sif xios EXP00' +map[1]='singularity run /work/ecseab13/ecseab13/ccwcones/sing3/nemo.sif nemo EXP00' + +exec_map=( 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 ) +# +echo \${map[\${exec_map[\$SLURM_PROCID]}]} +exec \${map[\${exec_map[\$SLURM_PROCID]}]} +## +EOFB +chmod u+x ./myscript_wrapper.sh + +rm EXP00/file_def_nemo-oce.xml +cd EXP00 +ln -s file_def_nemo-oce-M.xml file_def_nemo-oce.xml +cd ../ + +srun hostname -s > hostfile +#./build_rankfile -S 1 -s 16 -m 1 -C 6 -c 1 -H 1 -v > rankfile +./build_rankfile -S 1 -s 16 -m 1 -C 95 -c 15 -H 1 -v > rankfile + +#export LD_LIBRARY_PATH=$MPI_DIR:/opt/hdf5/install/lib:/.singularity.d/libs:$LD_LIBRARY_PATH +#export SINGULARITYENV_LD_LIBRARY_PATH=$LD_LIBRARY_PATH + +#singularity exec --cleanenv nemo.sif env + +echo '******************************' +#singularity exec nemo.sif env + +srun --mpi=pmi2 --hint=nomultithread --mem-bind=local \ +--pack-group=0 --cpu-bind=v,mask_cpu:0x1,0x10000,0x40000,0x100000,0x400000,0x1000000,0x4000000,0x10000000,0x40000000,0x100000000,0x400000000,0x1000000000,0x4000000000,0x10000000000,0x40000000000,0x100000000000,0x400000000000,0x1000000000000,0x4000000000000,0x10000000000000,0x40000000000000,0x100000000000000,0x400000000000000,0x1000000000000000,0x4000000000000000,0x10000000000000000,0x40000000000000000,0x100000000000000000,0x400000000000000000,0x1000000000000000000,0x4000000000000000000,0x10000000000000000000,0x40000000000000000000,0x100000000000000000000,0x400000000000000000000,0x1000000000000000000000,0x4000000000000000000000,0x10000000000000000000000,0x40000000000000000000000,0x100000000000000000000000,0x400000000000000000000000,0x1000000000000000000000000,0x4000000000000000000000000,0x10000000000000000000000000,0x40000000000000000000000000,0x100000000000000000000000000,0x400000000000000000000000000,0x1000000000000000000000000000,0x4000000000000000000000000000,0x10000000000000000000000000000,0x40000000000000000000000000000,0x100000000000000000000000000000,0x400000000000000000000000000000,0x1000000000000000000000000000000,0x4000000000000000000000000000000,0x10000000000000000000000000000000,0x40000000000000000000000000000000 ./myscript_wrapper.sh \ +: --pack-group=1 --cpu-bind=v,mask_cpu:0x1,0x4,0x10,0x40,0x100,0x400,0x1000,0x4000,0x10000,0x40000,0x100000,0x400000,0x1000000,0x4000000,0x10000000,0x40000000,0x100000000,0x400000000,0x1000000000,0x4000000000,0x10000000000,0x40000000000,0x100000000000,0x400000000000,0x1000000000000,0x4000000000000,0x10000000000000,0x40000000000000,0x100000000000000,0x400000000000000,0x1000000000000000,0x4000000000000000,0x10000000000000000,0x40000000000000000,0x100000000000000000,0x400000000000000000,0x1000000000000000000,0x4000000000000000000,0x10000000000000000000 ./myscript_wrapper.sh + + +#mpirun --oversubscribe -rf rankfile --report-bindings -v -np 1 --bind-to core singularity run /work/ecseab13/ecseab13/jhcones/testing/sing/nemo.sif xios EXP00 : -np 95 --bind-to core --mca btl_vader_single_copy_mechanism none --mca btl ^sm --mca btl_openib_allow_ib true --bind-to core singularity run /work/ecseab13/ecseab13/jhcones/testing/sing/nemo.sif nemo EXP00 + + +# Tidy up + +#if [ ! -d TIMINGS ]; then +# mkdir TIMINGS +#fi +#mv EXP00/timing.output TIMINGS/timing.output.$SLURM_JOB_NAME +#mv EXP00/ocean.output meta_out/ocean.output.$SLURM_JOB_NAME +#if [ ! -d meta_out/"$SLURM_JOB_NAME" ]; then +# mkdir meta_out/$SLURM_JOB_NAME +#fi +#mv EXP00/slurm-* meta_out/$SLURM_JOB_NAME +#if [ ! -d RESTARTS/"$SLURM_JOB_NAME" ]; then +# mkdir RESTARTS/$SLURM_JOB_NAME +#fi +#mv EXP00/*restart* RESTARTS/$SLURM_JOB_NAME +#if [ ! -d OUTPUTS/"$SLURM_JOB_NAME" ]; then +# mkdir OUTPUTS/$SLURM_JOB_NAME +#fi +#mv EXP00/*_grid_* OUTPUTS/$SLURM_JOB_NAME + +# Daisy-chain? + +#sbatch runscript_2Xg_184N.slurm diff --git a/runscripts/rs_mpich_bind_clean b/runscripts/rs_mpich_bind_clean new file mode 100644 index 0000000..2c9c8ea --- /dev/null +++ b/runscripts/rs_mpich_bind_clean @@ -0,0 +1,169 @@ +#!/bin/bash +#SBATCH --job-name=1Xg_95N +#SBATCH --time=00:20:00 +#SBATCH --account=ecseab13 +#SBATCH --partition=standard +#SBATCH --qos=short +#SBATCH --reservation=shortqos + +#SBATCH --partition=standard +#SBATCH --nodes=1 +#SBATCH --ntasks=57 +#SBATCH --ntasks-per-node=57 +#SBATCH --ntasks-per-core=1 + +#SBATCH hetjob +#SBATCH --partition=standard +#SBATCH --nodes=1 +#SBATCH --ntasks=39 +#SBATCH --ntasks-per-node=39 +#SBATCH --ntasks-per-core=1 + +#module load cpe/21.03 +#module load cray-hdf5-parallel +#module load cray-netcdf-hdf5parallel +#export LD_LIBRARY_PATH=/work/ecseab13/shared/install/lib:${LD_LIBRARY_PATH} +#export LD_LIBRARY_PATH=/opt/cray/pe/libsci/20.10.1.2/GNU/9.1/x86_64/lib:${LD_LIBRARY_PATH} + +#module -s restore /work/n01/shared/acc/n01_modules/ucx_env + +export OMP_NUM_THREADS=1 +#export MPI_DIR=/opt/cray/pe/mpich/8.1.3/ofi/gnu/9.1/lib-abi-mpich +# Set the LD_LIBRARY_PATH environment variable within the Singularity container +# to ensure that it used the correct MPI libraries +export SINGULARITYENV_LD_LIBRARY_PATH=/usr/lib:/lib/x86_64-linux-gnu:/opt/hdf5/install/lib:/opt/cray/pe/mpich/8.1.3/ofi/gnu/9.1/lib-abi-mpich:/opt/cray/pe/pmi/6.0.7/lib:/opt/cray/libfabric/1.11.0.0.233/lib64:/usr/lib64/host: + +export SINGULARITYENV_PREPEND_PATH=/usr/bin/host + +#export SINGULARITYENV_LD_LIBRARY_PATH=\ +#/opt/hdf5/install/lib:\ +#/opt/cray/pe/mpich/8.1.3/ofi/gnu/9.1/lib-abi-mpich:\ +#/usr/lib/x86_64-linux-gnu/libibverbs:\ +#/opt/cray/pe/pmi/6.0.7/lib:\ +#/opt/cray/libfabric/1.11.0.0.233/lib64 + +#SINGULARITYENV_LD_LIBRARY_PATH=\ +#/opt/hdf5/install/lib:\ +#/opt/cray/pe/mpich/8.1.3/ofi/gnu/9.1/lib-abi-mpich:\ +#/usr/lib/x86_64-linux-gnu/libibverbs:\ +#/opt/cray/pe/pmi/6.0.7/lib:\ +#/opt/cray/libfabric/1.11.0.0.233/lib64:\ +#/usr/lib64/host:\ +#/lib/x86_64-linux-gnu:\ +#/.singularity.d/libs + +SINGULARITYENV_LD_LIBRARY_PATH=\ +/opt/cray/pe/hdf5-parallel/1.12.0.2/crayclang/9.1/lib:\ +/opt/cray/pe/mpich/8.1.3/ucx/cray/9.1/lib-abi-mpich:\ +/opt/cray/pe/pmi/6.0.7/lib:\ +/opt/cray/libfabric/1.11.0.0.233/lib64:\ +/usr/lib64/host:\ + +#export SINGULARITYENV_LD_LIBRARY_PATH=\ +#/opt/libs:\ +#/usr/lib:\ +#/opt/mpi/install/lib:\ +#/lib/x86_64-linux-gnu:\ +#/opt/hdf5/install/lib:\ +#/opt/cray/pe/mpich/8.1.3/ofi/gnu/9.1/lib-abi-mpich:\ +#/opt/cray/pe/pmi/6.0.7/lib:\ +#/opt/cray/libfabric/1.11.0.0.233/lib64:\ +#/lib/x86_64-linux-gnu:\ +#/usr/lib64/host + +#export SINGULARITYENV_LD_LIBRARY_PATH=/opt/libs:/usr/lib:/opt/mpi/install/lib:/lib/x86_64-linux-gnu:/opt/hdf5/install/lib:/lib/x86_64-linux-gnu: + + +# Set the options for the Singularity executable +# This makes sure the locations with Cray Slingshot interconnect libraries are available +#singopts="-B /etc/libibverbs.d,/opt/cray,/usr/lib64:/usr/lib64/host,/usr/lib64/tcl,/var/spool/slurmd/mpi_cray_shasta" +singopts="-B /etc/libibverbs.d,/opt/cray,/usr/lib64:/usr/lib64/host,/usr/lib64/tcl,/var/spool/slurmd/mpi_cray_shasta" + +#singopts="-B /etc/libibverbs.d,/opt/cray,/usr/lib64:/usr/lib64/host,/usr/lib64/tcl" + + +# Set the LD_LIBRARY_PATH environment variable within the Singularity container +# to ensure that it used the correct MPI libraries +#singopts="-B /opt/cray,/usr/lib64:/usr/lib64/host,/usr/lib64/tcl,/var/spool/slurmd/mpi_cray_shasta" + +#=============================================================== +# LAUNCH JOB +#=============================================================== +# 5x6 25 61x65 +# 7x8 47 45x49 +# 11x11 95 29x36 +# 14x16 184 22x26 +# 22x23 370 16x19 +#=============================================================== +echo `date` : Launch Job +numactl --hardware + +sed "s/XXX_JPNI_XXX/11/g" EXP00/namelist_cfg_template > tmp_out +sed "s/XXX_JPNJ_XXX/11/g" tmp_out > EXP00/namelist_cfg + +#rm nemo.sif +#ln -s nemo-mpich-O0.sif nemo.sif + +cat > myscript_wrapper.sh << EOFB +#!/bin/ksh +# +#set -A map '--mca orte_base_help_aggregate 0 singularity run /work/ecseab13/ecseab13/jhcones/testing/sing3/nemo.sif xios EXP00' '--mca orte_base_help_aggregate 0 singularity run -B /etc/libibverbs.d /work/ecseab13/ecseab13/jhcones/testing/sing3/nemo.sif nemo EXP00' +map[0]='singularity run /work/ecseab13/ecseab13/ccwcones/sing3/nemo.sif xios EXP00' +map[1]='singularity run /work/ecseab13/ecseab13/ccwcones/sing3/nemo.sif nemo EXP00' + +exec_map=( 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 ) +# +echo \${map[\${exec_map[\$SLURM_PROCID]}]} +exec \${map[\${exec_map[\$SLURM_PROCID]}]} +## +EOFB +chmod u+x ./myscript_wrapper.sh + +rm EXP00/file_def_nemo-oce.xml +cd EXP00 +ln -s file_def_nemo-oce-M.xml file_def_nemo-oce.xml +cd ../ + +srun hostname -s > hostfile +#./build_rankfile -S 1 -s 16 -m 1 -C 6 -c 1 -H 1 -v > rankfile +./build_rankfile -S 1 -s 16 -m 1 -C 95 -c 15 -H 1 -v > rankfile + +#export LD_LIBRARY_PATH=$MPI_DIR:/opt/hdf5/install/lib:/.singularity.d/libs:$LD_LIBRARY_PATH +#export SINGULARITYENV_LD_LIBRARY_PATH=$LD_LIBRARY_PATH + +#singularity exec --cleanenv nemo.sif env + +echo '******************************' +#singularity exec nemo.sif env + +srun --mpi=pmi2 --hint=nomultithread --mem-bind=local \ +--pack-group=0 --cpu-bind=v,mask_cpu:0x1,0x10000,0x40000,0x100000,0x400000,0x1000000,0x4000000,0x10000000,0x40000000,0x100000000,0x400000000,0x1000000000,0x4000000000,0x10000000000,0x40000000000,0x100000000000,0x400000000000,0x1000000000000,0x4000000000000,0x10000000000000,0x40000000000000,0x100000000000000,0x400000000000000,0x1000000000000000,0x4000000000000000,0x10000000000000000,0x40000000000000000,0x100000000000000000,0x400000000000000000,0x1000000000000000000,0x4000000000000000000,0x10000000000000000000,0x40000000000000000000,0x100000000000000000000,0x400000000000000000000,0x1000000000000000000000,0x4000000000000000000000,0x10000000000000000000000,0x40000000000000000000000,0x100000000000000000000000,0x400000000000000000000000,0x1000000000000000000000000,0x4000000000000000000000000,0x10000000000000000000000000,0x40000000000000000000000000,0x100000000000000000000000000,0x400000000000000000000000000,0x1000000000000000000000000000,0x4000000000000000000000000000,0x10000000000000000000000000000,0x40000000000000000000000000000,0x100000000000000000000000000000,0x400000000000000000000000000000,0x1000000000000000000000000000000,0x4000000000000000000000000000000,0x10000000000000000000000000000000,0x40000000000000000000000000000000 ./myscript_wrapper.sh \ +: --pack-group=1 --cpu-bind=v,mask_cpu:0x1,0x4,0x10,0x40,0x100,0x400,0x1000,0x4000,0x10000,0x40000,0x100000,0x400000,0x1000000,0x4000000,0x10000000,0x40000000,0x100000000,0x400000000,0x1000000000,0x4000000000,0x10000000000,0x40000000000,0x100000000000,0x400000000000,0x1000000000000,0x4000000000000,0x10000000000000,0x40000000000000,0x100000000000000,0x400000000000000,0x1000000000000000,0x4000000000000000,0x10000000000000000,0x40000000000000000,0x100000000000000000,0x400000000000000000,0x1000000000000000000,0x4000000000000000000,0x10000000000000000000 ./myscript_wrapper.sh + + +#mpirun --oversubscribe -rf rankfile --report-bindings -v -np 1 --bind-to core singularity run /work/ecseab13/ecseab13/jhcones/testing/sing/nemo.sif xios EXP00 : -np 95 --bind-to core --mca btl_vader_single_copy_mechanism none --mca btl ^sm --mca btl_openib_allow_ib true --bind-to core singularity run /work/ecseab13/ecseab13/jhcones/testing/sing/nemo.sif nemo EXP00 + + +# Tidy up + +#if [ ! -d TIMINGS ]; then +# mkdir TIMINGS +#fi +#mv EXP00/timing.output TIMINGS/timing.output.$SLURM_JOB_NAME +#mv EXP00/ocean.output meta_out/ocean.output.$SLURM_JOB_NAME +#if [ ! -d meta_out/"$SLURM_JOB_NAME" ]; then +# mkdir meta_out/$SLURM_JOB_NAME +#fi +#mv EXP00/slurm-* meta_out/$SLURM_JOB_NAME +#if [ ! -d RESTARTS/"$SLURM_JOB_NAME" ]; then +# mkdir RESTARTS/$SLURM_JOB_NAME +#fi +#mv EXP00/*restart* RESTARTS/$SLURM_JOB_NAME +#if [ ! -d OUTPUTS/"$SLURM_JOB_NAME" ]; then +# mkdir OUTPUTS/$SLURM_JOB_NAME +#fi +#mv EXP00/*_grid_* OUTPUTS/$SLURM_JOB_NAME + +# Daisy-chain? + +#sbatch runscript_2Xg_184N.slurm diff --git a/runscripts/rs_mpich_hybrid b/runscripts/rs_mpich_hybrid new file mode 100644 index 0000000..f7a08ba --- /dev/null +++ b/runscripts/rs_mpich_hybrid @@ -0,0 +1,163 @@ +#!/bin/bash +#SBATCH --job-name=1Xg_95N +#SBATCH --time=00:20:00 +#SBATCH --account=ecseab13 +#SBATCH --partition=standard +#SBATCH --qos=short +#SBATCH --reservation=shortqos + +#SBATCH --partition=standard +#SBATCH --nodes=1 +#SBATCH --ntasks=57 +#SBATCH --ntasks-per-node=57 +#SBATCH --ntasks-per-core=1 + +#SBATCH hetjob +#SBATCH --partition=standard +#SBATCH --nodes=1 +#SBATCH --ntasks=39 +#SBATCH --ntasks-per-node=39 +#SBATCH --ntasks-per-core=1 + +#module load cpe/21.03 +#module load cray-hdf5-parallel +#module load cray-netcdf-hdf5parallel +#export LD_LIBRARY_PATH=/work/ecseab13/shared/install/lib:${LD_LIBRARY_PATH} +#export LD_LIBRARY_PATH=/opt/cray/pe/libsci/20.10.1.2/GNU/9.1/x86_64/lib:${LD_LIBRARY_PATH} + +module-s restore /work/n01/shared/acc/n01_modules/ucx_env + +export OMP_NUM_THREADS=1 +#export MPI_DIR=/opt/cray/pe/mpich/8.1.3/ofi/gnu/9.1/lib-abi-mpich +# Set the LD_LIBRARY_PATH environment variable within the Singularity container +# to ensure that it used the correct MPI libraries +#export SINGULARITYENV_LD_LIBRARY_PATH=/usr/lib:/lib/x86_64-linux-gnu:/opt/hdf5/install/lib:/opt/cray/pe/mpich/8.1.3/ofi/gnu/9.1/lib-abi-mpich:/opt/cray/pe/pmi/6.0.7/lib:/opt/cray/libfabric/1.11.0.0.233/lib64:/usr/lib64/host: + +#export SINGULARITYENV_PREPEND_PATH=/usr/bin/host + +#export SINGULARITYENV_LD_LIBRARY_PATH=\ +#/opt/hdf5/install/lib:\ +#/opt/cray/pe/mpich/8.1.3/ofi/gnu/9.1/lib-abi-mpich:\ +#/usr/lib/x86_64-linux-gnu/libibverbs:\ +#/opt/cray/pe/pmi/6.0.7/lib:\ +#/opt/cray/libfabric/1.11.0.0.233/lib64 + +#SINGULARITYENV_LD_LIBRARY_PATH=\ +#/opt/hdf5/install/lib:\ +#/opt/cray/pe/mpich/8.1.3/ofi/gnu/9.1/lib-abi-mpich:\ +#/usr/lib/x86_64-linux-gnu/libibverbs:\ +#/opt/cray/pe/pmi/6.0.7/lib:\ +#/opt/cray/libfabric/1.11.0.0.233/lib64:\ +#/usr/lib64/host:\ +#/lib/x86_64-linux-gnu:\ +#/.singularity.d/libs + + +#export SINGULARITYENV_LD_LIBRARY_PATH=\ +#/opt/libs:\ +#/usr/lib:\ +#/opt/mpi/install/lib:\ +#/lib/x86_64-linux-gnu:\ +#/opt/hdf5/install/lib:\ +#/opt/cray/pe/mpich/8.1.3/ofi/gnu/9.1/lib-abi-mpich:\ +#/opt/cray/pe/pmi/6.0.7/lib:\ +#/opt/cray/libfabric/1.11.0.0.233/lib64:\ +#/lib/x86_64-linux-gnu:\ +#/usr/lib64/host + +#export SINGULARITYENV_LD_LIBRARY_PATH=/opt/libs:/usr/lib:/opt/mpi/install/lib:/lib/x86_64-linux-gnu:/opt/hdf5/install/lib:/lib/x86_64-linux-gnu: + + +# Set the options for the Singularity executable +# This makes sure the locations with Cray Slingshot interconnect libraries are available +#singopts="-B /etc/libibverbs.d,/opt/cray,/usr/lib64:/usr/lib64/host,/usr/lib64/tcl,/var/spool/slurmd/mpi_cray_shasta" +#singopts="-B /etc/libibverbs.d,/opt/cray,/usr/lib64:/usr/lib64/host,/usr/lib64/tcl,/var/spool/slurmd/mpi_cray_shasta" + +#singopts="-B /etc/libibverbs.d,/opt/cray,/usr/lib64:/usr/lib64/host,/usr/lib64/tcl" + + +# Set the LD_LIBRARY_PATH environment variable within the Singularity container +# to ensure that it used the correct MPI libraries +#singopts="-B /opt/cray,/usr/lib64:/usr/lib64/host,/usr/lib64/tcl,/var/spool/slurmd/mpi_cray_shasta" + +#=============================================================== +# LAUNCH JOB +#=============================================================== +# 5x6 25 61x65 +# 7x8 47 45x49 +# 11x11 95 29x36 +# 14x16 184 22x26 +# 22x23 370 16x19 +#=============================================================== +echo `date` : Launch Job +numactl --hardware + +sed "s/XXX_JPNI_XXX/11/g" EXP00/namelist_cfg_template > tmp_out +sed "s/XXX_JPNJ_XXX/11/g" tmp_out > EXP00/namelist_cfg + +#rm nemo.sif +#ln -s nemo-mpich-O0.sif nemo.sif + +cat > myscript_wrapper.sh << EOFB +#!/bin/ksh +# +#set -A map '--mca orte_base_help_aggregate 0 singularity run /work/ecseab13/ecseab13/jhcones/testing/sing3/nemo.sif xios EXP00' '--mca orte_base_help_aggregate 0 singularity run -B /etc/libibverbs.d /work/ecseab13/ecseab13/jhcones/testing/sing3/nemo.sif nemo EXP00' +map[0]='singularity run /work/ecseab13/ecseab13/ccwcones/sing3/nemo.sif xios EXP00' +map[1]='singularity run /work/ecseab13/ecseab13/ccwcones/sing3/nemo.sif nemo EXP00' + +exec_map=( 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 ) +# +echo \${map[\${exec_map[\$SLURM_PROCID]}]} +exec \${map[\${exec_map[\$SLURM_PROCID]}]} +## +EOFB +chmod u+x ./myscript_wrapper.sh + +rm EXP00/file_def_nemo-oce.xml +cd EXP00 +ln -s file_def_nemo-oce-M.xml file_def_nemo-oce.xml +cd ../ + +srun hostname -s > hostfile +#./build_rankfile -S 1 -s 16 -m 1 -C 6 -c 1 -H 1 -v > rankfile +./build_rankfile -S 1 -s 16 -m 1 -C 95 -c 15 -H 1 -v > rankfile + +#export LD_LIBRARY_PATH=$MPI_DIR:/opt/hdf5/install/lib:/.singularity.d/libs:$LD_LIBRARY_PATH +#export SINGULARITYENV_LD_LIBRARY_PATH=$LD_LIBRARY_PATH + +#singularity exec --cleanenv nemo.sif env + +echo '******************************' +#singularity exec nemo.sif env + +srun --hint=nomultithread --mem-bind=local \ +--pack-group=0 --cpu-bind=v,mask_cpu:0x1,0x10000,0x40000,0x100000,0x400000,0x1000000,0x4000000,0x10000000,0x40000000,0x100000000,0x400000000,0x1000000000,0x4000000000,0x10000000000,0x40000000000,0x100000000000,0x400000000000,0x1000000000000,0x4000000000000,0x10000000000000,0x40000000000000,0x100000000000000,0x400000000000000,0x1000000000000000,0x4000000000000000,0x10000000000000000,0x40000000000000000,0x100000000000000000,0x400000000000000000,0x1000000000000000000,0x4000000000000000000,0x10000000000000000000,0x40000000000000000000,0x100000000000000000000,0x400000000000000000000,0x1000000000000000000000,0x4000000000000000000000,0x10000000000000000000000,0x40000000000000000000000,0x100000000000000000000000,0x400000000000000000000000,0x1000000000000000000000000,0x4000000000000000000000000,0x10000000000000000000000000,0x40000000000000000000000000,0x100000000000000000000000000,0x400000000000000000000000000,0x1000000000000000000000000000,0x4000000000000000000000000000,0x10000000000000000000000000000,0x40000000000000000000000000000,0x100000000000000000000000000000,0x400000000000000000000000000000,0x1000000000000000000000000000000,0x4000000000000000000000000000000,0x10000000000000000000000000000000,0x40000000000000000000000000000000 ./myscript_wrapper.sh \ +: --pack-group=1 --cpu-bind=v,mask_cpu:0x1,0x4,0x10,0x40,0x100,0x400,0x1000,0x4000,0x10000,0x40000,0x100000,0x400000,0x1000000,0x4000000,0x10000000,0x40000000,0x100000000,0x400000000,0x1000000000,0x4000000000,0x10000000000,0x40000000000,0x100000000000,0x400000000000,0x1000000000000,0x4000000000000,0x10000000000000,0x40000000000000,0x100000000000000,0x400000000000000,0x1000000000000000,0x4000000000000000,0x10000000000000000,0x40000000000000000,0x100000000000000000,0x400000000000000000,0x1000000000000000000,0x4000000000000000000,0x10000000000000000000 ./myscript_wrapper.sh + + +#mpirun --oversubscribe -rf rankfile --report-bindings -v -np 1 --bind-to core singularity run /work/ecseab13/ecseab13/jhcones/testing/sing/nemo.sif xios EXP00 : -np 95 --bind-to core --mca btl_vader_single_copy_mechanism none --mca btl ^sm --mca btl_openib_allow_ib true --bind-to core singularity run /work/ecseab13/ecseab13/jhcones/testing/sing/nemo.sif nemo EXP00 + + +# Tidy up + +#if [ ! -d TIMINGS ]; then +# mkdir TIMINGS +#fi +#mv EXP00/timing.output TIMINGS/timing.output.$SLURM_JOB_NAME +#mv EXP00/ocean.output meta_out/ocean.output.$SLURM_JOB_NAME +#if [ ! -d meta_out/"$SLURM_JOB_NAME" ]; then +# mkdir meta_out/$SLURM_JOB_NAME +#fi +#mv EXP00/slurm-* meta_out/$SLURM_JOB_NAME +#if [ ! -d RESTARTS/"$SLURM_JOB_NAME" ]; then +# mkdir RESTARTS/$SLURM_JOB_NAME +#fi +#mv EXP00/*restart* RESTARTS/$SLURM_JOB_NAME +#if [ ! -d OUTPUTS/"$SLURM_JOB_NAME" ]; then +# mkdir OUTPUTS/$SLURM_JOB_NAME +#fi +#mv EXP00/*_grid_* OUTPUTS/$SLURM_JOB_NAME + +# Daisy-chain? + +#sbatch runscript_2Xg_184N.slurm From b0d7bbaa45bec9472c43763a44055ca07f335c16 Mon Sep 17 00:00:00 2001 From: Chris Wood Date: Mon, 4 Apr 2022 15:42:32 +0100 Subject: [PATCH 12/17] Compiling mpi with a UCX built from source --- Singularity.nemo | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/Singularity.nemo b/Singularity.nemo index 2dca528..911aa19 100644 --- a/Singularity.nemo +++ b/Singularity.nemo @@ -135,7 +135,17 @@ From: ubuntu:20.04 rm mpich-3.4.2.tar.gz cd mpi - ./configure CC=gcc CXX=g++ FC=gfortran --prefix=/opt/mpi FFLAGS=-fallow-argument-mismatch --with-device=ch3:nemesis:ofi + cd /opt + mkdir ucx && cd ucx + wget https://github.com/openucx/ucx/releases/download/v1.11.2/ucx-1.11.2.tar.gz + rm v1.11.2.tar.gz + tar xzf ucx-1.11.2.tar.gz + cd ucx-1.11.2 + ./contrib/configure-release --prefix=$PWD/install + make && make install + + ./configure CC=gcc CXX=g++ FC=gfortran --prefix=/opt/mpi2/install FFLAGS=-fallow-argument-mismatch --with-device=ch4:ucx --with-ucx=/opt/ucx/ucx-1.11.2/install + make make install @@ -235,7 +245,7 @@ From: ubuntu:20.04 PATH=$PATH:/opt/mpi/bin:/opt/hdf5/install/bin LD_LIBRARY_PATH=/opt/hdf5/install/lib:$LD_LIBRARY_PATH - + cd $WORK_DIR chmod u+x /input_files/setup_nemo @@ -255,7 +265,7 @@ From: ubuntu:20.04 cd $WORK_DIR mkdir /opt/nemo - mv /nemo/nemo/cfgs/NEMO/EXP00/nemo /opt/nemo/ + c mkdir /opt/xios mv /nemo/nemo/cfgs/NEMO/EXP00/xios_server.exe /opt/xios/xios From e7f49db512ba85d08e303bbb4c92d55d77e71613 Mon Sep 17 00:00:00 2001 From: Chris Wood Date: Tue, 9 Nov 2021 16:36:38 +0000 Subject: [PATCH 13/17] Adding ARCHER2-recommended build settings for MPICH --- Singularity.nemo | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/Singularity.nemo b/Singularity.nemo index 7e48226..323c16c 100644 --- a/Singularity.nemo +++ b/Singularity.nemo @@ -56,13 +56,14 @@ From: ubuntu:20.04 locale-gen en_GB en_GB.UTF-8 # en_US en_US.UTF-8 apt update - apt install -y software-properties-common + apt install -y software-properties-common build-essential add-apt-repository universe # n.b. # - libcurl4-openssl-dev also installs libcurl4 (as does curl, if not already installed) # - zlib already installed - apt install -y python \ + DEBIAN_FRONTEND=noninteractive && apt install -y \ + python \ subversion \ wget \ git \ @@ -134,7 +135,7 @@ From: ubuntu:20.04 rm mpich-3.4.2.tar.gz cd mpi - ./configure CC=gcc CXX=g++ FC=gfortran --prefix=/opt/mpi/install FFLAGS=-fallow-argument-mismatch + ./configure CC=gcc CXX=g++ FC=gfortran --prefix=/opt/mpi FFLAGS=-fallow-argument-mismatch --with-device=ch3:nemesis:ofi make make install @@ -146,7 +147,7 @@ From: ubuntu:20.04 rm openmpi-4.1.1.tar.bz2 cd mpi - ./configure CC=gcc CXX=g++ FC=gfortran --enable-mpi1-compatibility --prefix=/opt/mpi/install --with-verbs --with-slurm --with-pmix + ./configure CC=gcc CXX=g++ FC=gfortran --enable-mpi1-compatibility --prefix=/opt/mpi --with-verbs --with-slurm --with-pmix make make install else @@ -173,7 +174,7 @@ From: ubuntu:20.04 cd hdf # n.b. --enable-fortran specifically not needed (https://www.unidata.ucar.edu/software/netcdf/docs/getting_and_building_netcdf.html#build_default) - CC=/opt/mpi/install/bin/mpicc ./configure --prefix=${H5DIR} --enable-hl --enable-parallel --with-default-api-version=v18 + CC=/opt/mpi/bin/mpicc ./configure --prefix=${H5DIR} --enable-hl --enable-parallel --with-default-api-version=v18 make # maybe we shouldn't bother with make check - it complains about a lot and we're not actually building on a parallel file system # so some of the checks are totally irrelevant @@ -199,7 +200,7 @@ From: ubuntu:20.04 cd netcdf NCDIR=/opt/netcdf/install - CC=/opt/mpi/install/bin/mpicc FC=/opt/mpi/install/bin/mpifort CPPFLAGS="-I${H5DIR}/include" LDFLAGS="-L${H5DIR}/lib" ./configure --disable-shared --prefix=$NCDIR + CC=/opt/mpi/bin/mpicc FC=/opt/mpi/bin/mpifort CPPFLAGS="-I${H5DIR}/include" LDFLAGS="-L${H5DIR}/lib" ./configure --disable-shared --prefix=$NCDIR make # make check fails; maybe we should avoid running it (only 1 response from a question on the netcdf mailing list, saying it is really finicky) # make check @@ -215,7 +216,7 @@ From: ubuntu:20.04 cd netcdf NFDIR=/opt/netcdf/install - CC=/opt/mpi/install/bin/mpicc FC=/opt/mpi/install/bin/mpifort CPPFLAGS="-I${NCDIR}/include -I${H5DIR}/include" LDFLAGS="-L${NCDIR}/lib -L${H5DIR}/lib" LIBS="-lhdf5 -lhdf5_hl -lcurl" ./configure --prefix=$NFDIR --disable-shared + CC=/opt/mpi/bin/mpicc FC=/opt/mpi/bin/mpifort CPPFLAGS="-I${NCDIR}/include -I${H5DIR}/include" LDFLAGS="-L${NCDIR}/lib -L${H5DIR}/lib" LIBS="-lhdf5 -lhdf5_hl -lcurl" ./configure --prefix=$NFDIR --disable-shared make make install cd .. @@ -225,14 +226,14 @@ From: ubuntu:20.04 # Now we can install NEMO ## - ln -s /opt/mpi/install/bin/mpif90 /usr/bin/mpif90 + ln -s /opt/mpi/bin/mpif90 /usr/bin/mpif90 WORK_DIR=/nemo >> $SINGULARITY_ENVIRONMENT mkdir $WORK_DIR cd $WORK_DIR chown nemo:nemo -R $WORK_DIR - PATH=$PATH:/opt/mpi/install/bin:/opt/hdf5/install/bin + PATH=$PATH:/opt/mpi/bin:/opt/hdf5/install/bin LD_LIBRARY_PATH=/opt/hdf5/install/lib:$LD_LIBRARY_PATH cd $WORK_DIR From 9bd8a73a7fcd8a43f50a1abc4a6cc83f1394c47d Mon Sep 17 00:00:00 2001 From: Chris Wood Date: Tue, 5 Apr 2022 16:02:57 +0100 Subject: [PATCH 14/17] fixing apparent copy/paste error --- Singularity.nemo | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Singularity.nemo b/Singularity.nemo index f377501..336d157 100644 --- a/Singularity.nemo +++ b/Singularity.nemo @@ -29,10 +29,12 @@ From: nemo_baseOS.sif cd $WORK_DIR chown nemo:nemo -R $WORK_DIR - PATH=$PATH:/opt/mpi/bin:/opt/hdf5/install/bin + PATH=$PATH:/opt/mpi/install/bin:/opt/hdf5/install/bin LD_LIBRARY_PATH=/opt/hdf5/install/lib:$LD_LIBRARY_PATH - + cd $WORK_DIR + + . /input_files/NEMO_in chmod u+x /input_files/setup_nemo /input_files/setup_nemo -x /nemo -w /nemo -s /nemo/AMM7_lite -m singularity -v $NEMO_VERSION -c gnu @@ -51,7 +53,7 @@ From: nemo_baseOS.sif cd $WORK_DIR mkdir /opt/nemo - c + mv /nemo/nemo/cfgs/NEMO/EXP00/nemo /opt/nemo/ mkdir /opt/xios mv /nemo/nemo/cfgs/NEMO/EXP00/xios_server.exe /opt/xios/xios From 810c6e3b22892f444a32a45a42332b5d0a99dcf6 Mon Sep 17 00:00:00 2001 From: Chris Wood Date: Thu, 19 May 2022 12:04:45 +0100 Subject: [PATCH 15/17] Reverting MPI namelist var to be null for correct github action to be carried out --- input_files/NEMO_in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/input_files/NEMO_in b/input_files/NEMO_in index b1630d6..ad883d3 100644 --- a/input_files/NEMO_in +++ b/input_files/NEMO_in @@ -3,4 +3,4 @@ NEMO_VERSION=4.0.4 # Check that VERSION is 4.0.[2-6], 4.0_HEAD or tr XIOS_REVISION= # Use default value if empty NEMO_COMPONENTS='OCE' CPP_KEYS= -MPI=MPICH # Which MPI implementation to use; valid options are currently MPICH (which installs v) or OMPI (which installs v4.1.1) +MPI= # Which MPI implementation to use; valid options are currently MPICH (which installs v) or OMPI (which installs v4.1.1) From badf341d23c04861ee4c48e3c040e29d7bbb06ac Mon Sep 17 00:00:00 2001 From: Chris Wood Date: Thu, 19 May 2022 12:08:38 +0100 Subject: [PATCH 16/17] Moving apt update as suggested by James --- Singularity.nemo | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Singularity.nemo b/Singularity.nemo index 7e48226..cf2f4f9 100644 --- a/Singularity.nemo +++ b/Singularity.nemo @@ -52,10 +52,11 @@ From: ubuntu:20.04 # install basic stuff ## + apt -y update + apt install -y locales #locales-all locale-gen en_GB en_GB.UTF-8 # en_US en_US.UTF-8 - apt update apt install -y software-properties-common add-apt-repository universe From 5e95ebcb8a4a58063cd37f05c722eef32d17503c Mon Sep 17 00:00:00 2001 From: Chris Wood Date: Tue, 3 Aug 2021 17:29:00 +0100 Subject: [PATCH 17/17] Initial commit of nemo on alpine --- nemo_alpine.def | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 nemo_alpine.def diff --git a/nemo_alpine.def b/nemo_alpine.def new file mode 100644 index 0000000..81e2c29 --- /dev/null +++ b/nemo_alpine.def @@ -0,0 +1,3 @@ +Bootstrap: library +From: alpine:3.9.2 +