refactor github workflow: add LLVM coverage support and update test/run logic for compatibility and flexibility

Author: allnes

.github/workflows/ubuntu.yml

@@ -331,7 +331,7 @@ jobs:
         env:
           PPC_NUM_PROC: 1
           PPC_ASAN_RUN: 1
-  gcc-build-codecov:
+  clang-build-codecov:
 #    needs:
 #      - gcc-test-extended
 #      - clang-test-extended
@@ -343,65 +343,105 @@ jobs:
       - name: Setup environment
         run: |
           sudo apt-get update
-          sudo apt-get install --no-install-recommends -y \
-            gcc-14 g++-14 ninja-build libmpich-dev libomp-dev valgrind gcovr
+          sudo apt-get install --no-install-recommends -y ninja-build libmpich-dev libomp-dev valgrind python3-pip
+          wget https://apt.llvm.org/llvm.sh
+          chmod u+x llvm.sh
+          sudo ./llvm.sh 20 all
+          python3 -m pip install -r requirements.txt
       - name: ccache
         uses: hendrikmuhs/ccache-action@v1.2
         with:
-          key: ${{ runner.os }}-gcc
+          key: ${{ runner.os }}-clang-coverage
           create-symlink: true
           max-size: 1G
       - name: CMake configure
         run: >
           cmake -S . -B build -G Ninja
           -D CMAKE_C_COMPILER_LAUNCHER=ccache -D CMAKE_CXX_COMPILER_LAUNCHER=ccache
           -D CMAKE_BUILD_TYPE=RELEASE
-          -D CMAKE_VERBOSE_MAKEFILE=ON -D USE_COVERAGE=ON
+          -D CMAKE_VERBOSE_MAKEFILE=ON -D USE_LLVM_COVERAGE=ON
+        env:
+          CC: clang-20
+          CXX: clang++-20
       - name: Build project
         run: |
           cmake --build build --parallel
+        env:
+          CC: clang-20
+          CXX: clang++-20
       - name: Run tests (MPI)
-        run: scripts/run_tests.py --running-type="processes_coverage"
+        run: scripts/run_tests.py --running-type="processes_coverage" --counts 2
         env:
-          PPC_NUM_PROC: 2
           PPC_NUM_THREADS: 2
+          LLVM_PROFILE_FILE: "build/llvm_profile_%p_%m.profraw"
       - name: Run tests (threads)
         run: scripts/run_tests.py --running-type="threads" --counts 1 2 3 4
         env:
           PPC_NUM_PROC: 1
-      - name: Generate gcovr Coverage Data
+          LLVM_PROFILE_FILE: "build/llvm_profile_%p_%m.profraw"
+      - name: Generate LLVM Coverage Data
         run: |
           mkdir cov-report
           cd build
-          # Collect coverage data from all MPI rank directories and regular build directory
-          # Copy coverage files from MPI rank directories to main build directory
-          # Only copy files that don't already exist to avoid conflicts
-          if [ -d "gcov_data" ]; then
-            find gcov_data -name "*.gcda" -exec bash -c 'cp "$1" "$(basename "$1")" 2>/dev/null || true' _ {} \;
-            find gcov_data -name "*.gcno" -exec bash -c 'cp "$1" "$(basename "$1")" 2>/dev/null || true' _ {} \;
+          # Collect all profile data files (including from MPI ranks)
+          PROFILE_FILES=$(find . -name "*.profraw" -type f | tr '\n' ' ')
+          
+          if [ -z "$PROFILE_FILES" ]; then
+            echo "No profile files found!"
+            exit 1
           fi
-
-          gcovr -r ../ \
-                --gcov-object-directory . \
-                --exclude '.*3rdparty/.*' \
-                --exclude '/usr/.*' \
-                --exclude '.*tasks/.*/tests/.*' \
-                --exclude '.*modules/.*/tests/.*' \
-                --exclude '.*tasks/common/runners/.*' \
-                --exclude '.*modules/runners/.*' \
-                --exclude '.*modules/util/include/perf_test_util.hpp' \
-                --exclude '.*modules/util/include/func_test_util.hpp' \
-                --exclude '.*modules/util/src/func_test_util.cpp' \
-                --gcov-ignore-parse-errors \
-                --gcov-ignore-errors \
-                --xml --output ../coverage.xml \
-                --html=../cov-report/index.html \
-                --html-details=../cov-report/ \
-                .
+          
+          # Merge all profile data files
+          llvm-profdata-20 merge -sparse $PROFILE_FILES -o merged.profdata
+          
+          # Generate coverage summary
+          llvm-cov-20 report \
+            -instr-profile=merged.profdata \
+            bin/ppc_func_tests bin/core_func_tests \
+            -ignore-filename-regex='.*3rdparty/.*' \
+            -ignore-filename-regex='/usr/.*' \
+            -ignore-filename-regex='.*tasks/.*/tests/.*' \
+            -ignore-filename-regex='.*modules/.*/tests/.*' \
+            -ignore-filename-regex='.*tasks/common/runners/.*' \
+            -ignore-filename-regex='.*modules/runners/.*'
+          
+          # Generate LCOV report for Codecov
+          llvm-cov-20 export \
+            -instr-profile=merged.profdata \
+            bin/ppc_func_tests bin/core_func_tests \
+            -format=lcov \
+            -ignore-filename-regex='.*3rdparty/.*' \
+            -ignore-filename-regex='/usr/.*' \
+            -ignore-filename-regex='.*tasks/.*/tests/.*' \
+            -ignore-filename-regex='.*modules/.*/tests/.*' \
+            -ignore-filename-regex='.*tasks/common/runners/.*' \
+            -ignore-filename-regex='.*modules/runners/.*' \
+            -ignore-filename-regex='.*modules/util/include/perf_test_util.hpp' \
+            -ignore-filename-regex='.*modules/util/include/func_test_util.hpp' \
+            -ignore-filename-regex='.*modules/util/src/func_test_util.cpp' \
+            > ../coverage.lcov
+          
+          # Generate HTML report
+          llvm-cov-20 show \
+            -instr-profile=merged.profdata \
+            bin/ppc_func_tests bin/core_func_tests \
+            -format=html \
+            -output-dir=../cov-report \
+            -show-line-counts-or-regions \
+            -show-instantiations \
+            -ignore-filename-regex='.*3rdparty/.*' \
+            -ignore-filename-regex='/usr/.*' \
+            -ignore-filename-regex='.*tasks/.*/tests/.*' \
+            -ignore-filename-regex='.*modules/.*/tests/.*' \
+            -ignore-filename-regex='.*tasks/common/runners/.*' \
+            -ignore-filename-regex='.*modules/runners/.*' \
+            -ignore-filename-regex='.*modules/util/include/perf_test_util.hpp' \
+            -ignore-filename-regex='.*modules/util/include/func_test_util.hpp' \
+            -ignore-filename-regex='.*modules/util/src/func_test_util.cpp'
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v5.4.3
         with:
-          files: coverage.xml
+          files: coverage.lcov
       - name: Upload coverage report artifact
         id: upload-cov
         uses: actions/upload-artifact@v4

cmake/configure.cmake

@@ -28,7 +28,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
 set(CMAKE_COMPILE_WARNING_AS_ERROR ON)
 
-if(USE_COVERAGE)
+if(USE_COVERAGE OR USE_LLVM_COVERAGE)
   set(CMAKE_INSTALL_RPATH "${CMAKE_BINARY_DIR}/ppc_onetbb/install/lib")
 else()
   set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib")
@@ -75,6 +75,10 @@ if(UNIX)
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --coverage")
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --coverage")
   endif(USE_COVERAGE)
+  if(USE_LLVM_COVERAGE)
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-instr-generate -fcoverage-mapping")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-instr-generate -fcoverage-mapping")
+  endif(USE_LLVM_COVERAGE)
 endif()
 
 if(MSVC)

scripts/run_tests.py

@@ -146,6 +146,29 @@ def run_processes(self, additional_mpi_args):
                     + self.__get_gtest_settings(10, '_' + task_type)
                 )
 
+    def __create_coverage_wrapper(self, template_name, replacements):
+        """Create a coverage wrapper script from the template."""
+        template_path = (
+            Path(self.__get_project_path()) / "scripts" / "templates" / template_name
+        )
+        wrapper_path = (
+            Path(self.__get_project_path()) / "build" / template_name.replace('.template', '')
+        )
+
+        # Read template
+        with open(template_path, 'r', encoding='utf-8') as template_file:
+            content = template_file.read()
+
+        # Replace placeholders
+        for key, value in replacements.items():
+            content = content.replace(f"{{{key}}}", value)
+
+        # Write a wrapper script
+        wrapper_path.write_text(content)
+        wrapper_path.chmod(0o755)
+
+        return wrapper_path
+
     def run_processes_coverage(self, additional_mpi_args):
         """Run tests in multiprocessing mode with a coverage collection."""
         ppc_num_proc = self.__ppc_env.get("PPC_NUM_PROC")
@@ -156,50 +179,54 @@ def run_processes_coverage(self, additional_mpi_args):
 
         # Set up coverage environment for MPI processes
         if not self.__ppc_env.get("PPC_ASAN_RUN"):
-            # Enable coverage data collection for each MPI process
-            self.__ppc_env["GCOV_PREFIX_STRIP"] = "0"
-            # Use MPI rank to create unique coverage directories for each process
-            gcov_base_dir = Path(self.__get_project_path()) / "build" / "gcov_data"
-            gcov_base_dir.mkdir(parents=True, exist_ok=True)
-
-            # Set GCOV_PREFIX to include MPI rank - this creates separate directories
-            # for each MPI process at runtime
-            self.__ppc_env["GCOV_PREFIX"] = str(
-                gcov_base_dir / "rank_${PMI_RANK:-${OMPI_COMM_WORLD_RANK:-${SLURM_PROCID:-0}}}"
-            )
+            # Check if we're using LLVM coverage or gcov
+            llvm_profile_file = self.__ppc_env.get("LLVM_PROFILE_FILE")
+
+            if llvm_profile_file:
+                # LLVM coverage setup
+                wrapper_script = self.__create_coverage_wrapper(
+                    "mpi_llvm_coverage_wrapper.sh.template",
+                    {"llvm_profile_base": llvm_profile_file.replace('%p', '%p').replace('%m', '%m')}
+                )
 
-            # Create a wrapper script to set a unique prefix per process
-            wrapper_script = Path(self.__get_project_path()) / "build" / "mpi_coverage_wrapper.sh"
-            wrapper_content = f"""#!/bin/bash
-# Get MPI rank from environment variables
-if [ -n "$PMIX_RANK" ]; then
-    RANK=$PMIX_RANK
-elif [ -n "$PMI_RANK" ]; then
-    RANK=$PMI_RANK
-elif [ -n "$OMPI_COMM_WORLD_RANK" ]; then
-    RANK=$OMPI_COMM_WORLD_RANK
-elif [ -n "$SLURM_PROCID" ]; then
-    RANK=$SLURM_PROCID
-else
-    RANK=0
-fi
-
-export GCOV_PREFIX="{gcov_base_dir}/rank_$RANK"
-mkdir -p "$GCOV_PREFIX"
-exec "$@"
-"""
-            wrapper_script.write_text(wrapper_content)
-            wrapper_script.chmod(0o755)
+                # Run tests with the LLVM coverage wrapper
+                for task_type in ["all", "mpi"]:
+                    test_command = (
+                        mpi_running
+                        + [str(wrapper_script)]
+                        + [str(self.work_dir / 'ppc_func_tests')]
+                        + self.__get_gtest_settings(10, '_' + task_type)
+                    )
+                    self.__run_exec(test_command)
+            else:
+                # Original gcov coverage setup
+                # Enable coverage data collection for each MPI process
+                self.__ppc_env["GCOV_PREFIX_STRIP"] = "0"
+                # Use MPI rank to create unique coverage directories for each process
+                gcov_base_dir = Path(self.__get_project_path()) / "build" / "gcov_data"
+                gcov_base_dir.mkdir(parents=True, exist_ok=True)
+
+                # Set GCOV_PREFIX to include MPI rank - this creates separate directories
+                # for each MPI process at runtime
+                self.__ppc_env["GCOV_PREFIX"] = str(
+                    gcov_base_dir / "rank_${PMI_RANK:-${OMPI_COMM_WORLD_RANK:-${SLURM_PROCID:-0}}}"
+                )
 
-            # Run tests with a coverage wrapper
-            for task_type in ["all", "mpi"]:
-                test_command = (
-                    mpi_running
-                    + [str(wrapper_script)]
-                    + [str(self.work_dir / 'ppc_func_tests')]
-                    + self.__get_gtest_settings(10, '_' + task_type)
+                # Create a wrapper script to set a unique prefix per process
+                wrapper_script = self.__create_coverage_wrapper(
+                    "mpi_gcov_coverage_wrapper.sh.template",
+                    {"gcov_base_dir": str(gcov_base_dir)}
                 )
-                self.__run_exec(test_command)
+
+                # Run tests with a coverage wrapper
+                for task_type in ["all", "mpi"]:
+                    test_command = (
+                        mpi_running
+                        + [str(wrapper_script)]
+                        + [str(self.work_dir / 'ppc_func_tests')]
+                        + self.__get_gtest_settings(10, '_' + task_type)
+                    )
+                    self.__run_exec(test_command)
 
     def run_performance(self):
         """Run performance tests."""

scripts/templates/mpi_gcov_coverage_wrapper.sh.template

@@ -0,0 +1,17 @@
+#!/bin/bash
+# Get MPI rank from environment variables
+if [ -n "$PMIX_RANK" ]; then
+    RANK=$PMIX_RANK
+elif [ -n "$PMI_RANK" ]; then
+    RANK=$PMI_RANK
+elif [ -n "$OMPI_COMM_WORLD_RANK" ]; then
+    RANK=$OMPI_COMM_WORLD_RANK
+elif [ -n "$SLURM_PROCID" ]; then
+    RANK=$SLURM_PROCID
+else
+    RANK=0
+fi
+
+export GCOV_PREFIX="{gcov_base_dir}/rank_$RANK"
+mkdir -p "$GCOV_PREFIX"
+exec "$@"

scripts/templates/mpi_llvm_coverage_wrapper.sh.template

@@ -0,0 +1,17 @@
+#!/bin/bash
+# Get MPI rank from environment variables
+if [ -n "$PMIX_RANK" ]; then
+    RANK=$PMIX_RANK
+elif [ -n "$PMI_RANK" ]; then
+    RANK=$PMI_RANK
+elif [ -n "$OMPI_COMM_WORLD_RANK" ]; then
+    RANK=$OMPI_COMM_WORLD_RANK
+elif [ -n "$SLURM_PROCID" ]; then
+    RANK=$SLURM_PROCID
+else
+    RANK=0
+fi
+
+# Set unique profile file for each rank
+export LLVM_PROFILE_FILE="{llvm_profile_base}_rank_$RANK"
+exec "$@"