diff --git a/.github/workflows/ci-flux-artifacts.yml b/.github/workflows/ci-flux-artifacts.yml new file mode 100644 index 00000000..d6bd5167 --- /dev/null +++ b/.github/workflows/ci-flux-artifacts.yml @@ -0,0 +1,105 @@ +name: ci-flux-artifacts + +on: + workflow_dispatch: + +permissions: + contents: read + packages: read + +jobs: + spindle-flux-ubuntu: + name: Testsuite (Flux, Ubuntu) + environment: Spindle CI + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Check out Spindle + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + + - name: Setup Docker Compose + uses: docker/setup-compose-action@16feee727cbdc83b6a014e6cc26fec4a79bcf30c + with: + version: latest + + - name: Build spindle-flux-ubuntu image + id: flux-ubuntu-build + run: | + cd containers/spindle-flux-ubuntu + docker compose --progress=plain build + + - name: Bring spindle-flux-ubuntu up + id: flux-ubuntu-up + run: | + cd containers/spindle-flux-ubuntu + docker compose up -d --wait --wait-timeout 60 + + - name: Verify munge works in spindle-flux-ubuntu + id: flux-ubuntu-munge + run: | + docker exec node-1 bash -c 'munge -n | unmunge' + + - name: Run spindle-flux-ubuntu testsuite + id: flux-ubuntu-testsuite + run: | + docker exec node-1 bash -c 'cd Spindle-build/testsuite && SPINDLE_DEBUG=3 flux alloc --nodes=${workers} ./runTests --nodes=${workers} --tasks-per-node=3' + + - name: Collect Spindle logs from containers + if: ${{ always() }} + continue-on-error: true + run: | + failed=0 + for node in node-1 node-2 node-3 node-4; do + echo "::group::Collecting logs from $node" + + if ! workdir=$(docker inspect -f '{{.Config.WorkingDir}}' "$node" 2>/dev/null); then + echo "Container $node not found; skipping." + failed=1 + echo "::endgroup::" + continue + fi + + if [ -z "$workdir" ]; then + echo "WorkingDir empty for $node; cannot resolve log path." + failed=1 + echo "::endgroup::" + continue + fi + + src="$workdir/Spindle-build/testsuite" + mkdir -p "spindle-logs/$node" + + if docker cp "$node:$src/." "spindle-logs/$node/"; then + ls -la "spindle-logs/$node/" + else + echo "Copy failed for $node; listing container path:" + docker exec "$node" ls -la "$src" || docker exec "$node" ls -la "$workdir" || true + failed=1 + continue + fi + find "spindle-logs/$node/" -type f ! -name 'spindle_output.*' -delete + + echo "::endgroup::" + done + + echo "Collected tree:" + ls -R spindle-logs/ || true + exit "$failed" + + + - name: Upload Spindle logs + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: spindle-logs-flux + path: spindle-logs/ + retention-days: 7 + if-no-files-found: warn + + - name: Bring spindle-flux-ubuntu down + id: flux-ubuntu-down + if: ${{ always() }} + continue-on-error: true + run: | + cd containers/spindle-flux-ubuntu + docker compose down diff --git a/.github/workflows/ci-serial-arm64-artifacts.yml b/.github/workflows/ci-serial-arm64-artifacts.yml new file mode 100644 index 00000000..0146a7be --- /dev/null +++ b/.github/workflows/ci-serial-arm64-artifacts.yml @@ -0,0 +1,77 @@ +name: ci-serial-arm64-artifacts + +on: + workflow_dispatch: + +permissions: + contents: read + packages: read + +jobs: + spindle-serial-ubuntu_arm64: + name: Testsuite (Serial, Ubuntu, arm64) + environment: Spindle CI + runs-on: ubuntu-24.04-arm + timeout-minutes: 30 + steps: + - name: Check out Spindle + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + + - name: Setup Docker Compose + uses: docker/setup-compose-action@16feee727cbdc83b6a014e6cc26fec4a79bcf30c + with: + version: latest + + - name: Build spindle-serial-ubuntu image + id: serial-ubuntu-build + run: | + cd containers/spindle-serial-ubuntu + docker compose --progress=plain build + + - name: Bring spindle-serial-ubuntu up + id: serial-ubuntu-up + run: | + cd containers/spindle-serial-ubuntu + docker compose up -d + + - name: Verify munge works in spindle-serial-ubuntu + id: serial-ubuntu-munge + run: | + docker exec spindlenode bash -c 'munge -n | unmunge' + + - name: Run spindle-serial-ubuntu testsuite + id: serial-ubuntu-testsuite + run: | + docker exec spindlenode bash -c 'cd Spindle-build/testsuite && SPINDLE_DEBUG=3 ./runTests' + + - name: Collect Spindle logs from container + if: ${{ always() }} + continue-on-error: true + run: | + mkdir -p spindle-logs + workdir=$(docker inspect -f '{{.Config.WorkingDir}}' spindlenode) + echo "Container WorkingDir: ${workdir:-}" + src="${workdir:?WorkingDir was empty - cannot resolve log path}/Spindle-build/testsuite" + docker cp "spindlenode:$src/." spindle-logs/ || { + echo "Copy failed; listing container path:" + docker exec spindlenode ls -la "$src" || docker exec spindlenode ls -la "$workdir" + exit 1 + } + ls -la spindle-logs/ + + - name: Upload Spindle logs + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: spindle-logs-serial-arm64 + path: spindle-logs/spindle_output.* + retention-days: 7 + if-no-files-found: warn + + - name: Bring spindle-serial-ubuntu down + id: serial-ubuntu-down + if: ${{ always() }} + continue-on-error: true + run: | + cd containers/spindle-serial-ubuntu + docker compose down diff --git a/.github/workflows/ci-serial-x86-artifacts.yml b/.github/workflows/ci-serial-x86-artifacts.yml new file mode 100644 index 00000000..9f8db79e --- /dev/null +++ b/.github/workflows/ci-serial-x86-artifacts.yml @@ -0,0 +1,77 @@ +name: ci-serial-x86-artifacts + +on: + workflow_dispatch: + +permissions: + contents: read + packages: read + +jobs: + spindle-serial-ubuntu_x86: + name: Testsuite (Serial, Ubuntu, x86) + environment: Spindle CI + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Check out Spindle + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + + - name: Setup Docker Compose + uses: docker/setup-compose-action@16feee727cbdc83b6a014e6cc26fec4a79bcf30c + with: + version: latest + + - name: Build spindle-serial-ubuntu image + id: serial-ubuntu-build + run: | + cd containers/spindle-serial-ubuntu + docker compose --progress=plain build + + - name: Bring spindle-serial-ubuntu up + id: serial-ubuntu-up + run: | + cd containers/spindle-serial-ubuntu + docker compose up -d + + - name: Verify munge works in spindle-serial-ubuntu + id: serial-ubuntu-munge + run: | + docker exec spindlenode bash -c 'munge -n | unmunge' + + - name: Run spindle-serial-ubuntu testsuite + id: serial-ubuntu-testsuite + run: | + docker exec spindlenode bash -c 'cd Spindle-build/testsuite && SPINDLE_DEBUG=3 ./runTests' + + - name: Collect Spindle logs from container + if: ${{ always() }} + continue-on-error: true + run: | + mkdir -p spindle-logs + workdir=$(docker inspect -f '{{.Config.WorkingDir}}' spindlenode) + echo "Container WorkingDir: ${workdir:-}" + src="${workdir:?WorkingDir was empty - cannot resolve log path}/Spindle-build/testsuite" + docker cp "spindlenode:$src/." spindle-logs/ || { + echo "Copy failed; listing container path:" + docker exec spindlenode ls -la "$src" || docker exec spindlenode ls -la "$workdir" + exit 1 + } + ls -la spindle-logs/ + + - name: Upload Spindle logs + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: spindle-logs-serial-x86 + path: spindle-logs/spindle_output.* + retention-days: 7 + if-no-files-found: warn + + - name: Bring spindle-serial-ubuntu down + id: serial-ubuntu-down + if: ${{ always() }} + continue-on-error: true + run: | + cd containers/spindle-serial-ubuntu + docker compose down diff --git a/.github/workflows/ci-slurm-artifacts.yml b/.github/workflows/ci-slurm-artifacts.yml new file mode 100644 index 00000000..38ee5b21 --- /dev/null +++ b/.github/workflows/ci-slurm-artifacts.yml @@ -0,0 +1,121 @@ +name: ci-slurm-artifacts + +on: + workflow_dispatch: + +permissions: + contents: read + packages: read + +jobs: + spindle-slurm-ubuntu: + name: Testsuite (Slurm rshlaunch, Ubuntu) + environment: Spindle CI + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Check out Spindle + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + + - name: Setup Docker Compose + uses: docker/setup-compose-action@16feee727cbdc83b6a014e6cc26fec4a79bcf30c + with: + version: latest + + - name: Login to GitHub Container Registry + if: ${{ !env.ACT }} + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Generate MariaDB configuration + id: slurm-ubuntu-mariadb + run: | + cd containers/spindle-slurm-ubuntu/testing + ./generate_config.sh + + - name: Build spindle-slurm-ubuntu image + id: slurm-ubuntu-build + run: | + cd containers/spindle-slurm-ubuntu/testing + docker compose --progress=plain build + + - name: Bring spindle-slurm-ubuntu up + id: slurm-ubuntu-up + run: | + cd containers/spindle-slurm-ubuntu/testing + docker compose up -d --wait --wait-timeout 120 + + - name: Verify munge works in spindle-slurm-ubuntu + id: slurm-ubuntu-munge + run: | + docker exec slurm-head bash -c 'munge -n | unmunge' + + - name: Run spindle-slurm-ubuntu testsuite + id: slurm-ubuntu-testsuite + run: | + docker exec slurm-head bash -c 'cd Spindle-build/testsuite && SPINDLE_DEBUG=3 salloc -n${workers} -N${workers} ./runTests ${workers}' + + + - name: Collect Spindle logs from containers + if: ${{ always() }} + continue-on-error: true + run: | + failed=0 + for node in slurm-head slurm-node-1 slurm-node-2 slurm-node-3 slurm-node-4; do + echo "::group::Collecting logs from $node" + + if ! workdir=$(docker inspect -f '{{.Config.WorkingDir}}' "$node" 2>/dev/null); then + echo "Container $node not found; skipping." + failed=1 + echo "::endgroup::" + continue + fi + + if [ -z "$workdir" ]; then + echo "WorkingDir empty for $node; cannot resolve log path." + failed=1 + echo "::endgroup::" + continue + fi + + src="$workdir/Spindle-build/testsuite" + mkdir -p "spindle-logs/$node" + + if docker cp "$node:$src/." "spindle-logs/$node/"; then + ls -la "spindle-logs/$node/" + else + echo "Copy failed for $node; listing container path:" + docker exec "$node" ls -la "$src" || docker exec "$node" ls -la "$workdir" || true + failed=1 + continue + fi + find "spindle-logs/$node/" -type f ! -name 'spindle_output.*' -delete + + echo "::endgroup::" + done + + echo "Collected tree:" + ls -R spindle-logs/ || true + exit "$failed" + + + - name: Upload Spindle logs + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: spindle-logs-slurm + path: spindle-logs/ + retention-days: 7 + if-no-files-found: warn + + + - name: Bring spindle-slurm-ubuntu down + id: slurm-ubuntu-down + if: ${{ always() }} + continue-on-error: true + run: | + cd containers/spindle-slurm-ubuntu/testing + docker compose down diff --git a/.github/workflows/ci-slurm-plugin-artifacts.yml b/.github/workflows/ci-slurm-plugin-artifacts.yml new file mode 100644 index 00000000..3a1ee0c0 --- /dev/null +++ b/.github/workflows/ci-slurm-plugin-artifacts.yml @@ -0,0 +1,123 @@ +name: ci-slurm-plugin-artifacts + +on: + workflow_dispatch: + +permissions: + contents: read + packages: read + +jobs: + spindle-slurm-plugin-ubuntu: + name: Testsuite (Slurm Plugin, Ubuntu) + environment: Spindle CI + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Check out Spindle + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + + - name: Setup Docker Compose + uses: docker/setup-compose-action@16feee727cbdc83b6a014e6cc26fec4a79bcf30c + with: + version: latest + + - name: Login to GitHub Container Registry + if: ${{ !env.ACT }} + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Generate MariaDB configuration + id: slurm-ubuntu-mariadb + run: | + cd containers/spindle-slurm-ubuntu/testing-plugin + ./generate_config.sh + + - name: Build spindle-slurm-plugin-ubuntu image + id: slurm-ubuntu-build + run: | + cd containers/spindle-slurm-ubuntu/testing-plugin + docker compose --progress=plain build + + - name: Bring spindle-slurm-plugin-ubuntu up + id: slurm-ubuntu-up + run: | + cd containers/spindle-slurm-ubuntu/testing-plugin + docker compose up -d --wait --wait-timeout 120 + + - name: Verify munge works in spindle-slurm-plugin-ubuntu + id: slurm-ubuntu-munge + run: | + docker exec slurm-plugin-head bash -c 'munge -n | unmunge' + + - name: Run spindle-slurm-plugin-ubuntu testsuite + id: slurm-ubuntu-testsuite + run: | + docker exec slurm-plugin-head bash -c 'cd Spindle-build/testsuite && SPINDLE_DEBUG=3 salloc -n${workers} -N${workers} ./runTests ${workers}' + + - name: Run spindle-slurm-plugin-ubuntu session testsuite + id: slurm-ubuntu-testsuite-sessions + run: | + docker exec slurm-plugin-head bash -c 'cd Spindle-build/testsuite && SPINDLE_DEBUG=3 salloc -n${workers} -N${workers} --spindle-session ./runTests ${workers}' + + - name: Collect Spindle logs from containers + if: ${{ always() }} + continue-on-error: true + run: | + failed=0 + for node in slurm-plugin-head slurm-plugin-node-1 slurm-plugin-node-2 slurm-plugin-node-3 slurm-plugin-node-4; do + echo "::group::Collecting logs from $node" + + if ! workdir=$(docker inspect -f '{{.Config.WorkingDir}}' "$node" 2>/dev/null); then + echo "Container $node not found; skipping." + failed=1 + echo "::endgroup::" + continue + fi + + if [ -z "$workdir" ]; then + echo "WorkingDir empty for $node; cannot resolve log path." + failed=1 + echo "::endgroup::" + continue + fi + + src="$workdir/Spindle-build/testsuite" + mkdir -p "spindle-logs/$node" + + if docker cp "$node:$src/." "spindle-logs/$node/"; then + ls -la "spindle-logs/$node/" + else + echo "Copy failed for $node; listing container path:" + docker exec "$node" ls -la "$src" || docker exec "$node" ls -la "$workdir" || true + failed=1 + continue + fi + find "spindle-logs/$node/" -type f ! -name 'spindle_output.*' -delete + + echo "::endgroup::" + done + + echo "Collected tree:" + ls -R spindle-logs/ || true + exit "$failed" + + - name: Upload Spindle logs + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: spindle-logs-slurm-plugin + path: spindle-logs/ + retention-days: 7 + if-no-files-found: warn + + - name: Bring spindle-slurm-plugin-ubuntu down + id: slurm-ubuntu-down + if: ${{ always() }} + continue-on-error: true + run: | + cd containers/spindle-slurm-ubuntu/testing-plugin + docker compose down diff --git a/.github/workflows/ci-slurm-srun-artifacts.yml b/.github/workflows/ci-slurm-srun-artifacts.yml new file mode 100644 index 00000000..e12077c9 --- /dev/null +++ b/.github/workflows/ci-slurm-srun-artifacts.yml @@ -0,0 +1,120 @@ +name: ci-slurm-srun-artifacts + +on: + workflow_dispatch: + +permissions: + contents: read + packages: read + +jobs: + spindle-slurm-srun-ubuntu: + name: Testsuite (Slurm srun, Ubuntu) + environment: Spindle CI + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Check out Spindle + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + + - name: Setup Docker Compose + uses: docker/setup-compose-action@16feee727cbdc83b6a014e6cc26fec4a79bcf30c + with: + version: latest + + - name: Login to GitHub Container Registry + if: ${{ !env.ACT }} + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Generate MariaDB configuration + id: slurm-ubuntu-mariadb + run: | + cd containers/spindle-slurm-ubuntu/testing-srun + ./generate_config.sh + + - name: Build spindle-slurm-ubuntu image + id: slurm-ubuntu-build + run: | + cd containers/spindle-slurm-ubuntu/testing-srun + docker compose --progress=plain build + + - name: Bring spindle-slurm-ubuntu up + id: slurm-ubuntu-up + run: | + cd containers/spindle-slurm-ubuntu/testing-srun + docker compose up -d --wait --wait-timeout 120 + + - name: Verify munge works in spindle-slurm-srun-ubuntu + id: slurm-ubuntu-munge + run: | + docker exec slurm-srun-head bash -c 'munge -n | unmunge' + + - name: Run spindle-slurm-srun-ubuntu testsuite + id: slurm-ubuntu-testsuite + run: | + docker exec slurm-srun-head bash -c 'cd Spindle-build/testsuite && SPINDLE_DEBUG=3 salloc -n${workers} -N${workers} ./runTests ${workers}' + + - name: Collect Spindle logs from containers + if: ${{ always() }} + continue-on-error: true + run: | + failed=0 + for node in slurm-head slurm-srun-node-1 slurm-srun-node-1 slurm-srun-node-2 slurm-srun-node-3 slurm-srun-node-4; do + echo "::group::Collecting logs from $node" + + if ! workdir=$(docker inspect -f '{{.Config.WorkingDir}}' "$node" 2>/dev/null); then + echo "Container $node not found; skipping." + failed=1 + echo "::endgroup::" + continue + fi + + if [ -z "$workdir" ]; then + echo "WorkingDir empty for $node; cannot resolve log path." + failed=1 + echo "::endgroup::" + continue + fi + + src="$workdir/Spindle-build/testsuite" + mkdir -p "spindle-logs/$node" + + if docker cp "$node:$src/." "spindle-logs/$node/"; then + ls -la "spindle-logs/$node/" + else + echo "Copy failed for $node; listing container path:" + docker exec "$node" ls -la "$src" || docker exec "$node" ls -la "$workdir" || true + failed=1 + continue + fi + find "spindle-logs/$node/" -type f ! -name 'spindle_output.*' -delete + + echo "::endgroup::" + done + + echo "Collected tree:" + ls -R spindle-logs/ || true + exit "$failed" + + + - name: Upload Spindle logs + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: spindle-logs-slurm-srun + path: spindle-logs/ + retention-days: 7 + if-no-files-found: warn + + + - name: Bring spindle-slurm-ubuntu down + id: slurm-ubuntu-down + if: ${{ always() }} + continue-on-error: true + run: | + cd containers/spindle-slurm-ubuntu/testing-srun + docker compose down diff --git a/config.h.in b/config.h.in index 00b83ef3..4ebf63f2 100644 --- a/config.h.in +++ b/config.h.in @@ -9,8 +9,9 @@ /* Colon-separated list of potential back-end cache directories */ #undef CACHEPATHS -/* Back-end directory for communication and housekeeping */ -#undef COMMPATH +/* Colon-separated list of candidate paths for back-end communication and + housekeeping */ +#undef COMMPATHS /* Define if were using biter for client/server communication */ #undef COMM_BITER diff --git a/configure b/configure index 6b54926e..800f4ee8 100755 --- a/configure +++ b/configure @@ -850,7 +850,7 @@ with_default_num_ports with_localstorage with_cachepaths with_cachepath -with_commpath +with_commpaths with_default_local_prefix with_testrm with_rm @@ -1600,8 +1600,8 @@ Optional Packages: --with-cachepaths=DIR Colon-separated list of potential back-end cache directories , - --with-compath=DIR Back-end directory for communication and - housekeeping + --with-compaths=DIR Colon-separated list of candidate paths for back-end + communication and housekeeping --with-default-local-prefix=DIRS Colon-seperated list of directories that Spindle will not cache files out of @@ -16693,11 +16693,11 @@ if test "${with_cachepath+set}" = set; then : fi -# Check whether --with-commpath was given. -if test "${with_commpath+set}" = set; then : - withval=$with_commpath; COMMPATH=${withval} +# Check whether --with-commpaths was given. +if test "${with_commpaths+set}" = set; then : + withval=$with_commpaths; COMMPATHS=${withval} else - COMMPATH=$DEFAULT_LOC + COMMPATHS=$DEFAULT_LOC fi @@ -16705,7 +16705,7 @@ fi if test "${with_default_local_prefix+set}" = set; then : withval=$with_default_local_prefix; SPINDLE_LOCAL_PREFIX=${withval} else - SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$COMMPATH" + SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$COMMPATHS" fi @@ -16725,7 +16725,7 @@ _ACEOF cat >>confdefs.h <<_ACEOF -#define COMMPATH "$COMMPATH" +#define COMMPATHS "$COMMPATHS" _ACEOF diff --git a/configure.common.ac b/configure.common.ac index 1b187f2e..36f82883 100644 --- a/configure.common.ac +++ b/configure.common.ac @@ -18,29 +18,33 @@ AC_ARG_WITH(default-num-ports, [NUM_COBO_PORTS=${withval}], [NUM_COBO_PORTS=$DEFAULT_NUM_COBO_PORTS]) AC_ARG_WITH(localstorage, - [AS_HELP_STRING([--with-localstorage=DIR (obsolete)],[Use --with-cachepaths and --with-commpath instead.])], - [AC_MSG_ERROR(requested obsolete option --with-localstorage. Use --with-cachepaths and --with-commpath instead.)], + [AS_HELP_STRING([--with-localstorage=DIR (obsolete)],[Use --with-cachepaths and --with-commpaths instead.])], + [AC_MSG_ERROR(requested obsolete option --with-localstorage. Use --with-cachepaths and --with-commpaths instead.)], []) AC_ARG_WITH(cachepaths, [AS_HELP_STRING([--with-cachepaths=DIR],[Colon-separated list of potential back-end cache directories])], [CACHEPATHS=${withval}], [CACHEPATHS=$DEFAULT_LOC]) AC_ARG_WITH(cachepath, - [[],[]], - [AC_MSG_ERROR(use --with-cachepaths=DIRS (plural) instead of --with-cachepath=DIR to specify one or more cache paths)], + [AS_HELP_STRING([--with-cachepath=DIR],[Synonym for --with-cachepaths ])], + [CACHEPATHS=${withval}], []) +AC_ARG_WITH(commpaths, + [AS_HELP_STRING([--with-commpaths=DIR],[Colon-separated list of candidate paths for back-end communication and housekeeping])], + [COMMPATHS=${withval}], + [COMMPATHS=$DEFAULT_LOC]) AC_ARG_WITH(commpath, - [AS_HELP_STRING([--with-compath=DIR],[Back-end directory for communication and housekeeping])], - [COMMPATH=${withval}], - [COMMPATH=$DEFAULT_LOC]) + [AS_HELP_STRING([--with-commpath=DIR],[Synonym for --with-commpaths])], + [COMMPATHS=${withval}], + []) AC_ARG_WITH(default-local-prefix, [AS_HELP_STRING([--with-default-local-prefix=DIRS],[Colon-seperated list of directories that Spindle will not cache files out of])], [SPINDLE_LOCAL_PREFIX=${withval}], - [SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$COMMPATH"]) + [SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$COMMPATHS"]) AC_DEFINE_UNQUOTED([SPINDLE_PORT],[$SPINDLE_PORT],[The default port for Spindle]) AC_DEFINE_UNQUOTED([NUM_COBO_PORTS],[$NUM_COBO_PORTS],[Number of ports for COBO to search for an open port]) AC_DEFINE_UNQUOTED([SPINDLE_MAX_PORT],[$(($SPINDLE_PORT + $NUM_COBO_PORTS - 1))],[The maximum port value]) -AC_DEFINE_UNQUOTED([COMMPATH],"[$COMMPATH]",[Back-end directory for communication and housekeeping]) +AC_DEFINE_UNQUOTED([COMMPATHS],"[$COMMPATHS]",[Colon-separated list of candidate paths for back-end communication and housekeeping]) AC_DEFINE_UNQUOTED([CACHEPATHS],"[$CACHEPATHS]",[Colon-separated list of potential back-end cache directories]) AC_DEFINE_UNQUOTED([SPINDLE_LOCAL_PREFIX],"[$SPINDLE_LOCAL_PREFIX]",[The default colon-separated list of directories that Spindle will not cache files out of]) diff --git a/containers/spindle-flux-ubuntu/scripts/build_spindle.sh b/containers/spindle-flux-ubuntu/scripts/build_spindle.sh index 276c59a6..454bca88 100755 --- a/containers/spindle-flux-ubuntu/scripts/build_spindle.sh +++ b/containers/spindle-flux-ubuntu/scripts/build_spindle.sh @@ -4,7 +4,7 @@ set -euxo pipefail mkdir -p /home/${USER}/Spindle-build cd /home/${USER}/Spindle-build -/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=flux --enable-flux-plugin --with-cachepaths=/tmp/commpath/cachepath --with-commpath=/tmp/commpath --enable-crash-dedup CFLAGS="-O2 -g" CXXFLAGS="-O2 -g" +/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=flux --enable-flux-plugin --with-cachepaths=/bad_cachepath:/tmp/commpath/cachepath --with-commpaths=/bad_commpath:/tmp/commpaths CFLAGS="-O2 -g" CXXFLAGS="-O2 -g" make -j$(nproc) make install diff --git a/containers/spindle-serial-ubuntu/scripts/build_spindle.sh b/containers/spindle-serial-ubuntu/scripts/build_spindle.sh index b48e3b53..c18c202d 100755 --- a/containers/spindle-serial-ubuntu/scripts/build_spindle.sh +++ b/containers/spindle-serial-ubuntu/scripts/build_spindle.sh @@ -4,7 +4,7 @@ set -euxo pipefail mkdir -p /home/${USER}/Spindle-build cd /home/${USER}/Spindle-build -/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=serial --with-cachepaths=/tmp/commpath/cachepath --with-commpath=/tmp/commpath --enable-crash-dedup CFLAGS="-O2 -g" CXXFLAGS="-O2 -g" +/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=serial --with-cachepaths=/bad_cachepath:/tmp/commpath/cachepath --with-commpaths=/bad_commpath:/tmp/commpath CFLAGS="-O2 -g" CXXFLAGS="-O2 -g" make -j$(nproc) make install diff --git a/containers/spindle-slurm-ubuntu/testing-plugin/scripts/build_spindle.sh b/containers/spindle-slurm-ubuntu/testing-plugin/scripts/build_spindle.sh index a52917ba..c214630d 100755 --- a/containers/spindle-slurm-ubuntu/testing-plugin/scripts/build_spindle.sh +++ b/containers/spindle-slurm-ubuntu/testing-plugin/scripts/build_spindle.sh @@ -3,7 +3,7 @@ set -euxo pipefail mkdir -p /home/${USER}/Spindle-build cd /home/${USER}/Spindle-build -/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=slurm-plugin --enable-slurm-plugin --with-cachepaths=/tmp/commpath/cachepath --with-commpath='$TMPDIR/commpath' --enable-crash-dedup CFLAGS="-O2 -g" CXXFLAGS="-O2 -g" +/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=slurm-plugin --enable-slurm-plugin --with-cachepaths=/bad_cachepath:/tmp/commpath/cachepath --with-commpaths=/bad_commpath:/tmp/commpath CFLAGS="-O2 -g" CXXFLAGS="-O2 -g" make -j$(nproc) make install diff --git a/containers/spindle-slurm-ubuntu/testing-srun/scripts/build_spindle.sh b/containers/spindle-slurm-ubuntu/testing-srun/scripts/build_spindle.sh index 125fe2eb..165de7c9 100755 --- a/containers/spindle-slurm-ubuntu/testing-srun/scripts/build_spindle.sh +++ b/containers/spindle-slurm-ubuntu/testing-srun/scripts/build_spindle.sh @@ -3,7 +3,7 @@ set -euxo pipefail mkdir -p /home/${USER}/Spindle-build cd /home/${USER}/Spindle-build -/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=slurm --with-cachepaths=/tmp/commpath/cachepath --with-commpath=/tmp/commpath CFLAGS="-O2 -g" CXXFLAGS="-O2 -g" +/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=slurm --with-cachepaths=/bad_cachepath:/tmp/commpath/cachepath --with-commpaths=/bad_commpath:/tmp/commpath:/bar CFLAGS="-O2 -g" CXXFLAGS="-O2 -g" make -j$(nproc) make install diff --git a/containers/spindle-slurm-ubuntu/testing/scripts/build_spindle.sh b/containers/spindle-slurm-ubuntu/testing/scripts/build_spindle.sh index 4a4c5eee..7cdf58dd 100755 --- a/containers/spindle-slurm-ubuntu/testing/scripts/build_spindle.sh +++ b/containers/spindle-slurm-ubuntu/testing/scripts/build_spindle.sh @@ -3,7 +3,7 @@ set -euxo pipefail mkdir -p /home/${USER}/Spindle-build cd /home/${USER}/Spindle-build -/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=slurm --with-rsh-launch --with-rsh-cmd=/usr/bin/ssh --with-cachepaths=/tmp/commpath/cachepath --with-commpath=/tmp/commpath --enable-crash-dedup CFLAGS="-O2 -g" CXXFLAGS="-O2 -g" +/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=slurm --with-rsh-launch --with-rsh-cmd=/usr/bin/ssh --with-cachepaths=/bad_cachepath:/tmp/commpath/cachepath --with-commpaths=/bad_commpath:/tmp/commpath CFLAGS="-O2 -g" CXXFLAGS="-O2 -g" make -j$(nproc) make install diff --git a/src/client/beboot/spindle_bootstrap.c b/src/client/beboot/spindle_bootstrap.c index e769fe4c..b814c5e2 100644 --- a/src/client/beboot/spindle_bootstrap.c +++ b/src/client/beboot/spindle_bootstrap.c @@ -33,6 +33,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include "client_api.h" #include "exec_util.h" #include "shmcache.h" +#include "parseloc.h" #include "config.h" @@ -53,7 +54,7 @@ static int rankinfo[4]={-1,-1,-1,-1}; number_t number; static int use_cache; static unsigned int cachesize; -static char *commpath, *number_s, *symbolic_commpath; +static char *commpath, *number_s, *commpaths; static char **cmdline; static char *executable; static char *client_lib; @@ -168,7 +169,7 @@ static int parse_cmdline(int argc, char *argv[]) daemon_args[i - 3] = NULL; } - symbolic_commpath = argv[i++]; + commpaths = argv[i++]; i++; // Skip over candidate_cachepaths. number_s = argv[i++]; number = (number_t) strtoul(number_s, NULL, 0); @@ -182,7 +183,7 @@ static int parse_cmdline(int argc, char *argv[]) return 0; } -static void launch_daemon(char *commpath) +static void launch_daemon( void ) { /*grand-child fork, then execv daemon. By grand-child forking we ensure that the app won't get confused by seeing an unknown process as a child. */ @@ -192,11 +193,6 @@ static void launch_daemon(char *commpath) char unique_file[MAX_PATH_LEN+1]; char buffer[32]; - result = spindle_mkdir(commpath); - if (result == -1) { - debug_printf("Exiting due to spindle_mkdir error\n"); - exit(-1); - } snprintf(unique_file, MAX_PATH_LEN, "%s/spindle_daemon_pid", commpath); unique_file[MAX_PATH_LEN] = '\0'; fd = open(unique_file, O_CREAT | O_EXCL | O_WRONLY, 0600); @@ -352,14 +348,12 @@ int main(int argc, char *argv[]) } } - char *orig_commpath = parse_location(symbolic_commpath, number); - if (!orig_commpath) { - return -1; + if( -1 == getFirstValidPath( commpaths, &commpath, number ) ){ + return -1; } - commpath = realize(orig_commpath); if (daemon_args) { - launch_daemon(commpath); + launch_daemon(); } result = establish_connection(); diff --git a/src/client/client_comlib/client_api.c b/src/client/client_comlib/client_api.c index 07735226..86c8ffc7 100644 --- a/src/client/client_comlib/client_api.c +++ b/src/client/client_comlib/client_api.c @@ -43,8 +43,6 @@ int send_cachepath_query( int fd, char **chosen_realized_cachepath, char **chose int rc = 0; ldcs_message_t message; char buffer[2*(MAX_PATH_LEN+1)]; - buffer[MAX_PATH_LEN] = '\0'; - message.header.type = LDCS_MSG_CHOSEN_CACHEPATH_REQUEST; message.header.len = 0; @@ -70,8 +68,10 @@ int send_cachepath_query( int fd, char **chosen_realized_cachepath, char **chose err_printf("Got unexpected message of type %d\n", (int) message.header.type); return -1; } - char *local_crc = spindle_strdup( buffer ); - char *local_cpc = spindle_strdup( &buffer[ strlen(local_crc) + 1 ] ); + // message.data contains two strings, each of which is no more than MAX_PATH_LEN bytes. + message.data[2*(MAX_PATH_LEN+1)-1] = '\0'; + char *local_crc = spindle_strdup( message.data ); + char *local_cpc = spindle_strdup( &message.data[ strlen(message.data) + 1 ] ); if( chosen_realized_cachepath ){ *chosen_realized_cachepath = local_crc; } diff --git a/src/client/config.h.in b/src/client/config.h.in index 872b9278..5b61f2f3 100644 --- a/src/client/config.h.in +++ b/src/client/config.h.in @@ -9,8 +9,9 @@ /* Colon-separated list of potential back-end cache directories */ #undef CACHEPATHS -/* Back-end directory for communication and housekeeping */ -#undef COMMPATH +/* Colon-separated list of candidate paths for back-end communication and + housekeeping */ +#undef COMMPATHS /* Define if were using biter for client/server communication */ #undef COMM_BITER diff --git a/src/client/configure b/src/client/configure index 18f13fa5..290d99d8 100755 --- a/src/client/configure +++ b/src/client/configure @@ -813,7 +813,7 @@ with_default_num_ports with_localstorage with_cachepaths with_cachepath -with_commpath +with_commpaths with_default_local_prefix with_testrm with_rm @@ -1542,8 +1542,8 @@ Optional Packages: --with-cachepaths=DIR Colon-separated list of potential back-end cache directories , - --with-compath=DIR Back-end directory for communication and - housekeeping + --with-compaths=DIR Colon-separated list of candidate paths for back-end + communication and housekeeping --with-default-local-prefix=DIRS Colon-seperated list of directories that Spindle will not cache files out of @@ -12618,11 +12618,11 @@ if test "${with_cachepath+set}" = set; then : fi -# Check whether --with-commpath was given. -if test "${with_commpath+set}" = set; then : - withval=$with_commpath; COMMPATH=${withval} +# Check whether --with-commpaths was given. +if test "${with_commpaths+set}" = set; then : + withval=$with_commpaths; COMMPATHS=${withval} else - COMMPATH=$DEFAULT_LOC + COMMPATHS=$DEFAULT_LOC fi @@ -12630,7 +12630,7 @@ fi if test "${with_default_local_prefix+set}" = set; then : withval=$with_default_local_prefix; SPINDLE_LOCAL_PREFIX=${withval} else - SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$COMMPATH" + SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$COMMPATHS" fi @@ -12650,7 +12650,7 @@ _ACEOF cat >>confdefs.h <<_ACEOF -#define COMMPATH "$COMMPATH" +#define COMMPATHS "$COMMPATHS" _ACEOF diff --git a/src/fe/config.h.in b/src/fe/config.h.in index 20a30383..0fb0e6f0 100644 --- a/src/fe/config.h.in +++ b/src/fe/config.h.in @@ -9,8 +9,9 @@ /* Colon-separated list of potential back-end cache directories */ #undef CACHEPATHS -/* Back-end directory for communication and housekeeping */ -#undef COMMPATH +/* Colon-separated list of candidate paths for back-end communication and + housekeeping */ +#undef COMMPATHS /* Define if were using biter for client/server communication */ #undef COMM_BITER diff --git a/src/fe/configure b/src/fe/configure index 73def515..293bcfee 100755 --- a/src/fe/configure +++ b/src/fe/configure @@ -834,7 +834,7 @@ with_default_num_ports with_localstorage with_cachepaths with_cachepath -with_commpath +with_commpaths with_default_local_prefix with_testrm with_rm @@ -1580,8 +1580,8 @@ Optional Packages: --with-cachepaths=DIR Colon-separated list of potential back-end cache directories , - --with-compath=DIR Back-end directory for communication and - housekeeping + --with-compaths=DIR Colon-separated list of candidate paths for back-end + communication and housekeeping --with-default-local-prefix=DIRS Colon-seperated list of directories that Spindle will not cache files out of @@ -16468,11 +16468,11 @@ if test "${with_cachepath+set}" = set; then : fi -# Check whether --with-commpath was given. -if test "${with_commpath+set}" = set; then : - withval=$with_commpath; COMMPATH=${withval} +# Check whether --with-commpaths was given. +if test "${with_commpaths+set}" = set; then : + withval=$with_commpaths; COMMPATHS=${withval} else - COMMPATH=$DEFAULT_LOC + COMMPATHS=$DEFAULT_LOC fi @@ -16480,7 +16480,7 @@ fi if test "${with_default_local_prefix+set}" = set; then : withval=$with_default_local_prefix; SPINDLE_LOCAL_PREFIX=${withval} else - SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$COMMPATH" + SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$COMMPATHS" fi @@ -16500,7 +16500,7 @@ _ACEOF cat >>confdefs.h <<_ACEOF -#define COMMPATH "$COMMPATH" +#define COMMPATHS "$COMMPATHS" _ACEOF diff --git a/src/fe/startup/config_mgr.cc b/src/fe/startup/config_mgr.cc index e1368642..1d6e1889 100644 --- a/src/fe/startup/config_mgr.cc +++ b/src/fe/startup/config_mgr.cc @@ -50,14 +50,18 @@ using namespace std; #define SPINDLE_NUM_PORTS_STR "250" #endif -#if defined(COMMPATH) -#define SPINDLE_COMMPATH_STR COMMPATH +#if defined(COMMPATHS) +#define SPINDLE_COMMPATHS_STR COMMPATHS +#elif defined(COMMPATH) +#define SPINDLE_COMMPATHS_STR COMMPATH #else -#define SPINDLE_COMMPATH_STR "$TMPDIR" +#define SPINDLE_COMMPATHS_STR "$TMPDIR" #endif #if defined(CACHEPATHS) #define SPINDLE_CACHEPATHS_STR CACHEPATHS +#elif defined(CACHEPATH) +#define SPINDLE_CACHEPATHS_STR CACHEPATH #else #define SPINDLE_CACHEPATHS_STR "$TMPDIR" #endif @@ -279,10 +283,14 @@ void initOptionsList() "Provides a text file containing a white-space separated list of files that should be relocated to each node before execution begins" }, { confStrip, "strip", shortStrip, groupMisc, cvBool, {}, "true", "Strip debug and symbol information from binaries before distributing them." }, - { confCommPath, "commpath", shortCommPath, groupMisc, cvString, {}, SPINDLE_COMMPATH_STR, - "Back-end directory communication and housekeeping. Should be a non-shared location such as a ramdisk." }, + { confCommPaths, "commpaths", shortCommPaths, groupMisc, cvString, {}, SPINDLE_COMMPATHS_STR, + "Colon-separated list of candidate paths for back-end communication and housekeeping. Should be a non-shared location such as a ramdisk." }, + { confCommPaths, "commpath", shortNone, groupMisc, cvString, {}, SPINDLE_COMMPATHS_STR, + "Synonym for --commpaths." }, { confCachePaths, "cachepaths", shortCachePaths, groupMisc, cvString, {}, SPINDLE_CACHEPATHS_STR, "Colon-separated list of candidate paths for cached libraries."}, + { confCachePaths, "cachepath", shortNone, groupMisc, cvString, {}, SPINDLE_CACHEPATHS_STR, + "Synonym for --cachepaths." }, { confNoclean, "noclean", shortNoClean, groupMisc, cvBool, {}, "false", "Don't remove local file cache after execution." }, { confDisableLogging, "disable-logging", shortDisableLogging, groupMisc, cvBool, {}, DISABLE_LOGGING_STR, @@ -751,11 +759,8 @@ bool ConfigMap::toSpindleArgs(spindle_args_t &args, bool alloc_strs) const case confNumPorts: args.num_ports = numresult; break; - case confCommPath: { - string path = strresult + "/spindle.$NUMBER"; - args.commpath = strdup(path.c_str()); - break; - } + case confCommPaths: + __attribute__((fallthrough)); // gcc-specific case confCachePaths:{ // Paramemter values are colon-separated lists of paths. // Append "/spindle.$NUMBER" to each path in the list. @@ -768,10 +773,15 @@ bool ConfigMap::toSpindleArgs(spindle_args_t &args, bool alloc_strs) const idx = paths.find(":", idx + number_var_with_colon.size()); }; paths += number_var_without_colon; - args.candidate_cachepaths = strdup(paths.c_str()); + if( name == confCommPaths ){ + args.commpaths = strdup(paths.c_str()); + }else if( name == confCachePaths ){ + args.candidate_cachepaths = strdup(paths.c_str()); + } break; } case confCachePrefix: + __attribute__((fallthrough)); // gcc-specific case confPythonPrefix: if (args.pythonprefix) args.pythonprefix = getstr(string(args.pythonprefix) + string(":") + strresult, true); diff --git a/src/fe/startup/config_mgr.h b/src/fe/startup/config_mgr.h index 14f18b7a..ffebefc0 100644 --- a/src/fe/startup/config_mgr.h +++ b/src/fe/startup/config_mgr.h @@ -29,7 +29,7 @@ enum SpindleConfigID { confCmdlineNewgroup, confPort, confNumPorts, - confCommPath, + confCommPaths, confCachePaths, confCachePrefix, confPythonPrefix, @@ -84,7 +84,7 @@ enum CmdlineShortOptions { shortAuditType = 'k', shortRelocSO = 'l', shortNoClean = 'n', - shortCommPath = 'o', + shortCommPaths = 'o', shortPush = 'p', shortPull = 'q', shortPythonPrefix = 'r', diff --git a/src/fe/startup/parse_launcher.cc b/src/fe/startup/parse_launcher.cc index 4484e998..9edb3f23 100644 --- a/src/fe/startup/parse_launcher.cc +++ b/src/fe/startup/parse_launcher.cc @@ -26,7 +26,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include #include #include - +#include "parseloc.h" #include "spindle_launch.h" #include "spindle_debug.h" #include "config.h" @@ -286,12 +286,16 @@ void ModifyArgv::print_err(string msg) fprintf(stderr, " spindle mpirun -np 4 spindlemarker a.out arg1 arg2\n\n"); } -void ModifyArgv::modifyCmdLine() +int ModifyArgv::modifyCmdLine() { char options_str[32]; snprintf(options_str, 32, "%lu", (unsigned long) params->opts); string options(options_str); + int rc = getFirstValidPath( params->commpaths, &( params->commpath ), params->number ); + if( rc != 0 ){ + return -1; + } string commpath(params->commpath); char number_str[32]; @@ -323,7 +327,10 @@ void ModifyArgv::modifyCmdLine() #else char **a_argv; int a_argc; - getApplicationArgsFE(params, &a_argc, &a_argv); + rc = getApplicationArgsFE(params, &a_argc, &a_argv); + if( rc != 0 ){ + return -1; + } for (int i = 0; i < a_argc; i++) new_argv[n++] = a_argv[i]; (void) default_libstr; (void) intercept_libstr; //Not needed on linux @@ -336,6 +343,7 @@ void ModifyArgv::modifyCmdLine() new_argv[n] = NULL; assert(n < new_argv_size); new_argc = n; + return 0; } bool ModifyArgv::getNewArgv(int &newargc, char** &newargv) @@ -367,7 +375,10 @@ bool ModifyArgv::getNewArgv(int &newargc, char** &newargv) return false; } - modifyCmdLine(); + int rc = modifyCmdLine(); + if( rc == -1 ){ + return false; + } if (spindle_debug_prints) { string new_cmdline = string(new_argv[0]); diff --git a/src/fe/startup/parse_launcher.h b/src/fe/startup/parse_launcher.h index 89e57bd3..205413ca 100644 --- a/src/fe/startup/parse_launcher.h +++ b/src/fe/startup/parse_launcher.h @@ -43,7 +43,7 @@ class ModifyArgv { void print_err(std::string str); bool autodetectParser(); bool chooseParser(); - void modifyCmdLine(); + int modifyCmdLine(); public: ModifyArgv(int argc, char **argv, diff --git a/src/fe/startup/spindle_fe.cc b/src/fe/startup/spindle_fe.cc index f1daab9e..e0fc0fe1 100644 --- a/src/fe/startup/spindle_fe.cc +++ b/src/fe/startup/spindle_fe.cc @@ -70,7 +70,7 @@ static int pack_data(spindle_args_t *args, void* &buffer, unsigned &buffer_size) buffer_size += sizeof(number_t); buffer_size += sizeof(opt_t); buffer_size += sizeof(unique_id_t); - buffer_size += args->commpath ? strlen(args->commpath) + 1 : 1; + buffer_size += args->commpaths ? strlen(args->commpaths) + 1 : 1; buffer_size += args->candidate_cachepaths ? strlen(args->candidate_cachepaths) + 1 : 1; buffer_size += args->pythonprefix ? strlen(args->pythonprefix) + 1 : 1; buffer_size += args->preloadfile ? strlen(args->preloadfile) + 1 : 1; @@ -91,7 +91,7 @@ static int pack_data(spindle_args_t *args, void* &buffer, unsigned &buffer_size) pack_param(args->use_launcher, buf, pos); pack_param(args->startup_type, buf, pos); pack_param(args->shm_cache_size, buf, pos); - pack_param(args->commpath, buf, pos); + pack_param(args->commpaths, buf, pos); pack_param(args->candidate_cachepaths, buf, pos); pack_param(args->pythonprefix, buf, pos); pack_param(args->preloadfile, buf, pos); @@ -231,7 +231,7 @@ int getApplicationArgsFE(spindle_args_t *params, int *spindle_argc, char ***spin (*spindle_argv)[n++] = strdup(numports_s); (*spindle_argv)[n++] = strdup(uniqueid_s); } - (*spindle_argv)[n++] = strdup(params->commpath); + (*spindle_argv)[n++] = strdup(params->commpaths); (*spindle_argv)[n++] = strdup(params->candidate_cachepaths); (*spindle_argv)[n++] = strdup(number_s); (*spindle_argv)[n++] = strdup(opt_s); @@ -398,11 +398,11 @@ int spindleInitFE(const char **hosts, spindle_args_t *params) /* Start FE server */ debug_printf("spindle_args_t { number = %lu; port = %u; num_ports = %u; opts = %lu; unique_id = %lu; " - "use_launcher = %u; startup_type = %u; shm_cache_size = %u; commpath = %s; " + "use_launcher = %u; startup_type = %u; shm_cache_size = %u; commpaths = %s; " "cachepaths = %s; " "pythonprefix = %s; preloadfile = %s; bundle_timeout_ms = %u; bundle_cachesize_kb = %u }\n", (unsigned long) params->number, params->port, params->num_ports, params->opts, params->unique_id, - params->use_launcher, params->startup_type, params->shm_cache_size, params->commpath, + params->use_launcher, params->startup_type, params->shm_cache_size, params->commpaths, params->candidate_cachepaths, params->pythonprefix, params->preloadfile, params->bundle_timeout_ms, params->bundle_cachesize_kb); diff --git a/src/flux/flux-spindle.c b/src/flux/flux-spindle.c index a3abeef7..cb9a421a 100644 --- a/src/flux/flux-spindle.c +++ b/src/flux/flux-spindle.c @@ -382,7 +382,7 @@ static int sp_getopts (flux_shell_t *shell, struct spindle_ctx *ctx) int crash_dedup = 0; const char *relocaout = NULL, *reloclibs = NULL, *relocexec = NULL, *relocpython = NULL; const char *followfork = NULL, *preload = NULL, *level = NULL; - const char *pyprefix = NULL, *commpath = NULL; + const char *pyprefix = NULL, *commpaths = NULL; char *numafiles = NULL, *cachepaths = NULL; if (flux_shell_getopt_unpack (shell, "spindle", "o", &opts) < 0) @@ -416,7 +416,7 @@ static int sp_getopts (flux_shell_t *shell, struct spindle_ctx *ctx) "reloc-exec", &relocexec, "reloc-python", &relocpython, "python-prefix", &pyprefix, - "commpath", &commpath, + "commpaths", &commpaths, "numa", &numa, "numa-files", &numafiles, "preload", &preload, @@ -468,8 +468,8 @@ static int sp_getopts (flux_shell_t *shell, struct spindle_ctx *ctx) if( cachepaths ){ ctx->params.candidate_cachepaths = cachepaths; } - if (commpath) { - ctx->params.commpath = (char *) commpath; + if (commpaths) { + ctx->params.commpaths = (char *) commpaths; } if (crash_dedup) { ctx->params.opts |= OPT_CRASH_HANDLER; diff --git a/src/flux/sessionmgr.c b/src/flux/sessionmgr.c index 17027163..1267d302 100644 --- a/src/flux/sessionmgr.c +++ b/src/flux/sessionmgr.c @@ -106,27 +106,14 @@ char **strip_start_from_argv(int argc, char **argv) return new_argv; } -extern char *parse_location(char *loc, int number); -extern int spindle_mkdir(char *orig_path); - -#if !defined(COMMPATH) -#error COMMPATH must be defined in config.h -#endif +#define SESSIONPATH "$TMPDIR/spindle/session" const char *get_session_dir() { - int result; - char *dir; - dir = parse_location((char *) (COMMPATH "/spindle_session"), 0); - if (!dir) { - spindle_debug_printf(1, "ERROR: Could not parse directory for spindle session location from %s/spindle_session\n", COMMPATH); - return NULL; - } - - result = spindle_mkdir(dir); - if (result == -1) { - spindle_debug_printf(1, "ERROR: Could not mkdir spindle session location at %s\n", dir); - free(dir); - return NULL; - } - return dir; + /* This function was originally designed to return a realized version of COMMPATH. + * Since then, COMMPATH has been replaced by COMMPATHS, which can be modified by + * the user on a per-job basis and uses the number_t number as part of the path + * name. Sessions are designed to span multiple jobs, and thus shouldn't be + * piggybacking on COMMPATHS. For now, we'll put in a hardcoded directory. + */ + return strdup(SESSIONPATH); } diff --git a/src/include/spindle_launch.h b/src/include/spindle_launch.h index 98121396..c2db2680 100644 --- a/src/include/spindle_launch.h +++ b/src/include/spindle_launch.h @@ -127,6 +127,9 @@ typedef struct { /* The local-disk location for communication and housekeeping. */ char *commpath; + /* The initial colon-separated list of commpath candidates. */ + char *commpaths; + /* Path[s] for cached libraries. */ char *candidate_cachepaths; /* Colon-separated list of candidate paths (max 64) */ diff --git a/src/server/config.h.in b/src/server/config.h.in index 14c85b47..87e6242d 100644 --- a/src/server/config.h.in +++ b/src/server/config.h.in @@ -9,8 +9,9 @@ /* Colon-separated list of potential back-end cache directories */ #undef CACHEPATHS -/* Back-end directory for communication and housekeeping */ -#undef COMMPATH +/* Colon-separated list of candidate paths for back-end communication and + housekeeping */ +#undef COMMPATHS /* Define if were using biter for client/server communication */ #undef COMM_BITER diff --git a/src/server/configure b/src/server/configure index 71d5fb82..e6e08c67 100755 --- a/src/server/configure +++ b/src/server/configure @@ -840,7 +840,7 @@ with_default_num_ports with_localstorage with_cachepaths with_cachepath -with_commpath +with_commpaths with_default_local_prefix with_testrm with_rm @@ -1577,8 +1577,8 @@ Optional Packages: --with-cachepaths=DIR Colon-separated list of potential back-end cache directories , - --with-compath=DIR Back-end directory for communication and - housekeeping + --with-compaths=DIR Colon-separated list of candidate paths for back-end + communication and housekeeping --with-default-local-prefix=DIRS Colon-seperated list of directories that Spindle will not cache files out of @@ -16465,11 +16465,11 @@ if test "${with_cachepath+set}" = set; then : fi -# Check whether --with-commpath was given. -if test "${with_commpath+set}" = set; then : - withval=$with_commpath; COMMPATH=${withval} +# Check whether --with-commpaths was given. +if test "${with_commpaths+set}" = set; then : + withval=$with_commpaths; COMMPATHS=${withval} else - COMMPATH=$DEFAULT_LOC + COMMPATHS=$DEFAULT_LOC fi @@ -16477,7 +16477,7 @@ fi if test "${with_default_local_prefix+set}" = set; then : withval=$with_default_local_prefix; SPINDLE_LOCAL_PREFIX=${withval} else - SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$COMMPATH" + SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$COMMPATHS" fi @@ -16497,7 +16497,7 @@ _ACEOF cat >>confdefs.h <<_ACEOF -#define COMMPATH "$COMMPATH" +#define COMMPATHS "$COMMPATHS" _ACEOF diff --git a/src/server/startup/spindle_be.cc b/src/server/startup/spindle_be.cc index d5a234a1..17e072f6 100644 --- a/src/server/startup/spindle_be.cc +++ b/src/server/startup/spindle_be.cc @@ -58,7 +58,7 @@ static int unpack_data(spindle_args_t *args, void *buffer, int buffer_size) unpack_param(args->use_launcher, buf, pos); unpack_param(args->startup_type, buf, pos); unpack_param(args->shm_cache_size, buf, pos); - unpack_param(args->commpath, buf, pos); + unpack_param(args->commpaths, buf, pos); unpack_param(args->candidate_cachepaths, buf, pos); unpack_param(args->pythonprefix, buf, pos); unpack_param(args->preloadfile, buf, pos); @@ -146,17 +146,14 @@ int spindleRunBE(unsigned int port, unsigned int num_ports, unique_id_t unique_i assert(args.port == port); - /* Expand environment variables in commpath. */ - char *new_commpath = parse_location(args.commpath, args.number); - if (!new_commpath) { - err_printf("Failed to convert commpath %s\n", args.commpath); - if (args.startup_type == startup_external) - LOGGING_FINI; - return -1; + /* Find the first valid commpath in the commpaths list. */ + if( -1 == getFirstValidPath( args.commpaths, &( args.commpath ), args.number ) ){ + // Failed. + err_printf("Failed to find valid commpath in %s\n", args.commpaths); + if (args.startup_type == startup_external) + LOGGING_FINI; + return -1; } - debug_printf("Translated commpath from %s to %s\n", args.commpath, new_commpath); - free(args.commpath); - args.commpath = strdup(new_commpath); result = ldcs_audit_server_process(&args); if (result == -1) { diff --git a/src/slurm_plugin/Makefile.am b/src/slurm_plugin/Makefile.am index 489dffcb..f712463b 100644 --- a/src/slurm_plugin/Makefile.am +++ b/src/slurm_plugin/Makefile.am @@ -3,7 +3,7 @@ lib_LTLIBRARIES = libspindleslurm.la libver=`$(top_srcdir)/LIB_VERSION spindleslurm` libspindleslurm_la_SOURCES = encode_decode.c plugin_utils.c slurm_plugin.c $(top_srcdir)/src/utils/spindle_mkdir.c $(top_srcdir)/src/utils/parseloc.c -libspindleslurm_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(top_srcdir)/src/include -I$(top_srcdir)/src/logging -DUSE_PLUGIN_DEBUG -DDEBUG -DCUSTOM_GETENV -DCUSTOM_GETENV_FREE -DSCONTROL_BIN="$(SCONTROL_ABSPATH)" -DSINFO_BIN="$(SINFO_ABSPATH)" -DSPINDLE_DO_EXPORT +libspindleslurm_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(top_srcdir)/src/utils -I$(top_srcdir)/src/include -I$(top_srcdir)/src/logging -DUSE_PLUGIN_DEBUG -DDEBUG -DCUSTOM_GETENV -DCUSTOM_GETENV_FREE -DSCONTROL_BIN="$(SCONTROL_ABSPATH)" -DSINFO_BIN="$(SINFO_ABSPATH)" -DSPINDLE_DO_EXPORT libspindleslurm_la_LDFLAGS = $(AM_LDFLAGS) -ldl -version-info $(libver) libspindleslurm_la_LIBADD = $(top_builddir)/src/server/startup/libspindlebe.la $(top_builddir)/src/fe/startup/libspindlefe.la libspindleslurm_la_CFLAGS = $(CFLAGS) -fvisibility=hidden diff --git a/src/slurm_plugin/Makefile.in b/src/slurm_plugin/Makefile.in index a7fe188f..8d3f3643 100644 --- a/src/slurm_plugin/Makefile.in +++ b/src/slurm_plugin/Makefile.in @@ -383,7 +383,7 @@ top_srcdir = @top_srcdir@ lib_LTLIBRARIES = libspindleslurm.la libver = `$(top_srcdir)/LIB_VERSION spindleslurm` libspindleslurm_la_SOURCES = encode_decode.c plugin_utils.c slurm_plugin.c $(top_srcdir)/src/utils/spindle_mkdir.c $(top_srcdir)/src/utils/parseloc.c -libspindleslurm_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(top_srcdir)/src/include -I$(top_srcdir)/src/logging -DUSE_PLUGIN_DEBUG -DDEBUG -DCUSTOM_GETENV -DCUSTOM_GETENV_FREE -DSCONTROL_BIN="$(SCONTROL_ABSPATH)" -DSINFO_BIN="$(SINFO_ABSPATH)" -DSPINDLE_DO_EXPORT +libspindleslurm_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(top_srcdir)/src/utils -I$(top_srcdir)/src/include -I$(top_srcdir)/src/logging -DUSE_PLUGIN_DEBUG -DDEBUG -DCUSTOM_GETENV -DCUSTOM_GETENV_FREE -DSCONTROL_BIN="$(SCONTROL_ABSPATH)" -DSINFO_BIN="$(SINFO_ABSPATH)" -DSPINDLE_DO_EXPORT libspindleslurm_la_LDFLAGS = $(AM_LDFLAGS) -ldl -version-info $(libver) libspindleslurm_la_LIBADD = $(top_builddir)/src/server/startup/libspindlebe.la $(top_builddir)/src/fe/startup/libspindlefe.la libspindleslurm_la_CFLAGS = $(CFLAGS) -fvisibility=hidden diff --git a/src/slurm_plugin/plugin_utils.c b/src/slurm_plugin/plugin_utils.c index 58ebda9e..e29007fa 100644 --- a/src/slurm_plugin/plugin_utils.c +++ b/src/slurm_plugin/plugin_utils.c @@ -12,6 +12,7 @@ #include #include #include "plugin_utils.h" +#include "parseloc.h" #if !defined(STR) #define XSTR(X) #X @@ -297,46 +298,19 @@ int isFEHost(char **hostlist, unsigned int num_hosts) (void) error; return feresult; } - -static char* locSpecificDir(spindle_args_t *params) -{ - char *dir = NULL, *expanded_dir = NULL, *realized_dir = NULL; - - dir = params->commpath; - if (!dir) { - sdprintf(1, "ERROR: Location not filled in\n"); - goto done; - } - expanded_dir = parse_location(dir, params->number); - if (!expanded_dir) { - sdprintf(1, "ERROR: Could not expand file-system dir %s\n", dir); - goto done; - } - realized_dir = realize(expanded_dir); - if (!realized_dir) { - sdprintf(1, "ERROR: Could not turn dir to a real path\n"); - } - - done: - if (expanded_dir) - free(expanded_dir); - - return realized_dir; -} - #define SOCKET_PREFIX "spFE" static char* exitSocketPath(spindle_args_t *params) { - char *realized_dir, *socket_path = NULL; + char *realized_dir = NULL, *socket_path = NULL; char session_id_str[32]; size_t socket_path_len; - realized_dir = locSpecificDir(params); - if (!realized_dir) { + if( -1 == getFirstValidPath( params->commpaths, &( params->commpath ), params->number )){ err_printf("Could not get real path for FE exit socket\n"); goto done; } + realized_dir = strdup( params->commpath ); snprintf(session_id_str, sizeof(session_id_str), "%lu", (unsigned long) params->number); @@ -559,11 +533,10 @@ int isBEProc(spindle_args_t *params, unsigned int exit_phase) int beproc_result = -1; int fd = -1, error; - realized_dir = locSpecificDir(params); - if (!realized_dir) { - sdprintf(1, "ERROR: Could not resolve location directory in isBEProc\n"); - goto done; + if( -1 == getFirstValidPath( params->commpaths, &( params->commpath ), params->number ) ){ + return -1; } + realized_dir = strdup( params->commpath ); gethostname(hostname, sizeof(hostname)); hostname[sizeof(hostname)-1] = '\0'; diff --git a/src/slurm_plugin/slurm_plugin.c b/src/slurm_plugin/slurm_plugin.c index 1778acbf..644f1d37 100644 --- a/src/slurm_plugin/slurm_plugin.c +++ b/src/slurm_plugin/slurm_plugin.c @@ -32,7 +32,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include "spindle_launch.h" #include "plugin_utils.h" - +#include "parseloc.h" #include "config.h" #define SPINDLE_USE_SESSION "SPINDLE_USE_SESSION" @@ -766,7 +766,6 @@ static unique_id_t getUniqueID(spank_t spank, int session_enabled) static int fillInArgs(spank_t spank, spindle_args_t *args, int argc, char **argv, unique_id_t unique_id, int session_enabled) { int result; - char *symbolic_commpath, *orig_commpath; char *err_string; current_spank = spank; @@ -796,19 +795,9 @@ static int fillInArgs(spank_t spank, spindle_args_t *args, int argc, char **argv } - symbolic_commpath = args->commpath; - orig_commpath = parse_location(symbolic_commpath, args->number); - if( !orig_commpath ){ + if( -1 == getFirstValidPath( args->commpaths, &( args->commpath ), args->number ) ){ return -1; } - args->commpath = realize(orig_commpath); - if (args->commpath != orig_commpath) - free(orig_commpath); - if (!args->commpath) { - slurm_error("Spindle Options Error: Could not resolve commpath location\n"); - sdprintf(1, "ERROR: Could not realize commpath from '%s'\n", symbolic_commpath); - return -1; - } return 0; } diff --git a/src/utils/parseloc.c b/src/utils/parseloc.c index bdf5ca1f..01183d01 100644 --- a/src/utils/parseloc.c +++ b/src/utils/parseloc.c @@ -370,6 +370,24 @@ void determineValidCachePaths( uint64_t *validBitIdx, char *origPathList, number free( pathList ); } +int getFirstValidPath( char *origPathList, char **firstValidPath, number_t number ){ + char *saveptr, *candidatePath, *pathList = strdup( origPathList ); + int rc = 0; + + candidatePath = strtok_r( pathList, ":", &saveptr ); + while( NULL != candidatePath ){ + rc = validateCandidatePath( candidatePath, firstValidPath, NULL, NULL, number ); + if( 1 == rc ){ // success + break; + } + *firstValidPath = NULL; + candidatePath = strtok_r( NULL, ":", &saveptr ); + } + free(pathList); + + return (1 == rc) ? 0 : -1; +} + void getValidCachePathByIndex( uint64_t validBitIdx, char **realizedCachePath, char **parsedCachePath, char **symbolicCachePath ){ uint64_t bitoffset = 0; if (!validBitIdx){ diff --git a/src/utils/parseloc.h b/src/utils/parseloc.h index a99409c3..5100364e 100644 --- a/src/utils/parseloc.h +++ b/src/utils/parseloc.h @@ -31,7 +31,7 @@ int is_local_prefix(const char *path, char **local_prefixes); int validateCandidatePath( char *candidatePath, char **realizedPath, char **parsedPath, char **symbolicPath, number_t number ); void determineValidCachePaths( uint64_t *validBitIdx, char *origPathList, number_t number ); void getValidCachePathByIndex( uint64_t validBitIdx, char **realizedCachePath, char **parsedCachePath, char **symbolicCachePath ); - +int getFirstValidPath( char *origPathList, char **firstValidPath, number_t number ); #if defined(__cplusplus) } #endif