From f3311c97982107c16ff3e09848f45d17671696b4 Mon Sep 17 00:00:00 2001 From: Giovanni Baraldi Date: Thu, 13 Feb 2025 16:38:18 -0600 Subject: [PATCH 01/10] Adding tests for threading --- tests/rocprofv3/CMakeLists.txt | 1 + .../internal-threading/CMakeLists.txt | 30 +++++ .../internal-threading/pthread_wrapper.cpp | 106 ++++++++++++++++++ 3 files changed, 137 insertions(+) create mode 100644 tests/rocprofv3/internal-threading/CMakeLists.txt create mode 100644 tests/rocprofv3/internal-threading/pthread_wrapper.cpp diff --git a/tests/rocprofv3/CMakeLists.txt b/tests/rocprofv3/CMakeLists.txt index 1404bc5e33..6028af9af3 100644 --- a/tests/rocprofv3/CMakeLists.txt +++ b/tests/rocprofv3/CMakeLists.txt @@ -36,6 +36,7 @@ add_subdirectory(roctracer-roctx) add_subdirectory(scratch-memory) add_subdirectory(pc-sampling) add_subdirectory(collection-period) +add_subdirectory(internal-threading) if(ROCPROFILER_BUILD_ROCDECODE_TESTS) add_subdirectory(rocdecode-trace) endif() diff --git a/tests/rocprofv3/internal-threading/CMakeLists.txt b/tests/rocprofv3/internal-threading/CMakeLists.txt new file mode 100644 index 0000000000..79e56ff560 --- /dev/null +++ b/tests/rocprofv3/internal-threading/CMakeLists.txt @@ -0,0 +1,30 @@ +# +# rocprofv3 tool test +# +cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR) + +project( + rocprofiler-tests-threading-records + LANGUAGES CXX + VERSION 0.0.0) + +find_package(rocprofiler-sdk REQUIRED) + +add_library(pthread_wrapper SHARED) +target_sources(pthread_wrapper PRIVATE pthread_wrapper.cpp) +target_include_directories(pthread_wrapper PRIVATE $) + +add_test( + NAME rocprofv3-test-internal-threading + COMMAND + $ --sys-trace --pmc SQ_WAVES + --preload $ -- $ 1 15) + +string(REPLACE "LD_PRELOAD=" "ROCPROF_PRELOAD=" PRELOAD_ENV + "${ROCPROFILER_MEMCHECK_PRELOAD_ENV}") + +set_tests_properties( + rocprofv3-test-internal-threading + PROPERTIES TIMEOUT 45 LABELS "integration-tests" ENVIRONMENT + "${PRELOAD_ENV}" FAIL_REGULAR_EXPRESSION + "${ROCPROFILER_DEFAULT_FAIL_REGEX}") diff --git a/tests/rocprofv3/internal-threading/pthread_wrapper.cpp b/tests/rocprofv3/internal-threading/pthread_wrapper.cpp new file mode 100644 index 0000000000..8b4254b7f3 --- /dev/null +++ b/tests/rocprofv3/internal-threading/pthread_wrapper.cpp @@ -0,0 +1,106 @@ +#include +#include +#include +#include +#include + +#if defined(_MSC_VER) +# define ROCPROFILER_PUBLIC_API __declspec(dllexport) +#else +# define ROCPROFILER_PUBLIC_API __attribute__((visibility("default"))) +#endif + +typedef void (*rocprofiler_internal_thread_library_cb_t)(int, void*); +typedef int (*rocprofiler_at_internal_thread_create_t)(rocprofiler_internal_thread_library_cb_t precreate, rocprofiler_internal_thread_library_cb_t postcreate, int libs, void* data); + +typedef void*(routine_t)(void *); + +class RegisterProfiler +{ +public: + RegisterProfiler() = default; + + void try_register() + { + if (init.load()) return; + + auto lk = std::unique_lock{mut}; + if (init.load()) return; + + auto* handle = dlopen("librocprofiler-sdk.so", RTLD_LAZY | RTLD_NOLOAD); + if (!handle) return; + + auto* register_fn = (rocprofiler_at_internal_thread_create_t) dlsym(handle, "rocprofiler_at_internal_thread_create"); + if (!register_fn) throw std::runtime_error("Could not dlsym rocprofiler library"); + + register_fn(pre_callback, post_callback, ~0, this); + dlclose(handle); + init.store(true); + } + + static void pre_callback(int bitmask, void* arg) + { + static_cast(arg)->pre_library_list.fetch_or(bitmask); + } + + static void post_callback(int bitmask, void* arg) + { + static_cast(arg)->pre_library_list.fetch_and(~bitmask); + } + + std::atomic pre_library_list{0}; + std::atomic init{false}; + std::mutex mut; +}; + +class DL +{ + using PthreadFn = decltype(pthread_create); +public: + DL() + { + handle = dlopen("libpthread.so.0", RTLD_LAZY | RTLD_LOCAL); + if (!handle) throw std::runtime_error("Could not load pthread library"); + + pthread_create_fn = (PthreadFn*) dlsym(handle, "pthread_create"); + if (!pthread_create_fn) throw std::runtime_error("Could not dlsym pthread library"); + } + void* handle = nullptr; + PthreadFn* pthread_create_fn = nullptr; +}; + +RegisterProfiler* get_reg() +{ + static auto* reg = new RegisterProfiler(); + return reg; +} + +ROCPROFILER_PUBLIC_API +int pthread_create( + pthread_t* thread, + const pthread_attr_t* attr, + routine_t* start_routine, + void* arg +) { + static auto* dl = new DL(); + auto* reg = get_reg(); + + reg->try_register(); + + if (reg->pre_library_list.load() == 0) throw std::runtime_error("Thread not registered!"); + + std::cout << "Creating thread: " << std::hex << reg->pre_library_list << std::dec << std::endl; + return dl->pthread_create_fn(thread, attr, start_routine, arg); +} + +class ConstructCheck +{ +public: + ConstructCheck() = default; + ~ConstructCheck() + { + if (!get_reg()->init.load()) abort(); + } +}; + +ConstructCheck const_check(); From 0c9f59d4692b130ad39b799c146971a636116cbc Mon Sep 17 00:00:00 2001 From: Giovanni Baraldi Date: Mon, 17 Feb 2025 08:40:34 -0600 Subject: [PATCH 02/10] Adding pthread wrapper --- .../internal-threading/CMakeLists.txt | 1 - .../internal-threading/pthread_wrapper.cpp | 82 ++++++++++--------- 2 files changed, 42 insertions(+), 41 deletions(-) diff --git a/tests/rocprofv3/internal-threading/CMakeLists.txt b/tests/rocprofv3/internal-threading/CMakeLists.txt index 79e56ff560..63a1839ce9 100644 --- a/tests/rocprofv3/internal-threading/CMakeLists.txt +++ b/tests/rocprofv3/internal-threading/CMakeLists.txt @@ -12,7 +12,6 @@ find_package(rocprofiler-sdk REQUIRED) add_library(pthread_wrapper SHARED) target_sources(pthread_wrapper PRIVATE pthread_wrapper.cpp) -target_include_directories(pthread_wrapper PRIVATE $) add_test( NAME rocprofv3-test-internal-threading diff --git a/tests/rocprofv3/internal-threading/pthread_wrapper.cpp b/tests/rocprofv3/internal-threading/pthread_wrapper.cpp index 8b4254b7f3..1c1fea5e1d 100644 --- a/tests/rocprofv3/internal-threading/pthread_wrapper.cpp +++ b/tests/rocprofv3/internal-threading/pthread_wrapper.cpp @@ -1,7 +1,7 @@ -#include #include -#include +#include #include +#include #include #if defined(_MSC_VER) @@ -10,32 +10,47 @@ # define ROCPROFILER_PUBLIC_API __attribute__((visibility("default"))) #endif +#define ASSERT(x, msg) \ + if(!(x)) \ + { \ + std::cerr << __FILE__ << ':' << __LINE__ << " - " << msg << std::endl; \ + abort(); \ + } + +#define MAX_ALLOWED_THREADS 3 + typedef void (*rocprofiler_internal_thread_library_cb_t)(int, void*); -typedef int (*rocprofiler_at_internal_thread_create_t)(rocprofiler_internal_thread_library_cb_t precreate, rocprofiler_internal_thread_library_cb_t postcreate, int libs, void* data); +typedef int (*rocprofiler_at_internal_thread_create_t)( + rocprofiler_internal_thread_library_cb_t precreate, + rocprofiler_internal_thread_library_cb_t postcreate, + int libs, + void* data); -typedef void*(routine_t)(void *); +typedef void*(routine_t)(void*); class RegisterProfiler { public: RegisterProfiler() = default; - void try_register() + bool try_register() { - if (init.load()) return; + if(init.load()) return true; auto lk = std::unique_lock{mut}; - if (init.load()) return; + if(init.load()) return false; auto* handle = dlopen("librocprofiler-sdk.so", RTLD_LAZY | RTLD_NOLOAD); - if (!handle) return; + if(!handle) return false; - auto* register_fn = (rocprofiler_at_internal_thread_create_t) dlsym(handle, "rocprofiler_at_internal_thread_create"); - if (!register_fn) throw std::runtime_error("Could not dlsym rocprofiler library"); + auto* register_fn = (rocprofiler_at_internal_thread_create_t) dlsym( + handle, "rocprofiler_at_internal_thread_create"); + ASSERT(register_fn, "Could not dlsym rocprofiler library"); register_fn(pre_callback, post_callback, ~0, this); dlclose(handle); init.store(true); + return false; } static void pre_callback(int bitmask, void* arg) @@ -56,51 +71,38 @@ class RegisterProfiler class DL { using PthreadFn = decltype(pthread_create); + public: DL() { handle = dlopen("libpthread.so.0", RTLD_LAZY | RTLD_LOCAL); - if (!handle) throw std::runtime_error("Could not load pthread library"); + ASSERT(handle, "Could not load pthread library"); pthread_create_fn = (PthreadFn*) dlsym(handle, "pthread_create"); - if (!pthread_create_fn) throw std::runtime_error("Could not dlsym pthread library"); + ASSERT(pthread_create_fn, "Could not dlsym pthread library"); + } + ~DL() + { + pthread_create_fn = nullptr; + if(handle) dlclose(handle); } - void* handle = nullptr; + void* handle = nullptr; PthreadFn* pthread_create_fn = nullptr; }; -RegisterProfiler* get_reg() +ROCPROFILER_PUBLIC_API +int +pthread_create(pthread_t* thread, const pthread_attr_t* attr, routine_t* start_routine, void* arg) { static auto* reg = new RegisterProfiler(); - return reg; -} + static auto* dl = new DL(); -ROCPROFILER_PUBLIC_API -int pthread_create( - pthread_t* thread, - const pthread_attr_t* attr, - routine_t* start_routine, - void* arg -) { - static auto* dl = new DL(); - auto* reg = get_reg(); + static std::atomic unwrapped_threads{0}; - reg->try_register(); + if(reg->try_register() && reg->pre_library_list.load() == 0) unwrapped_threads++; - if (reg->pre_library_list.load() == 0) throw std::runtime_error("Thread not registered!"); + ASSERT(unwrapped_threads <= MAX_ALLOWED_THREADS, + "Limit reached for number of thread not inside a pre/post callback!"); - std::cout << "Creating thread: " << std::hex << reg->pre_library_list << std::dec << std::endl; return dl->pthread_create_fn(thread, attr, start_routine, arg); } - -class ConstructCheck -{ -public: - ConstructCheck() = default; - ~ConstructCheck() - { - if (!get_reg()->init.load()) abort(); - } -}; - -ConstructCheck const_check(); From 3029bccf4c4fefeae92eeb2f4642cd8f15edd2ff Mon Sep 17 00:00:00 2001 From: Giovanni Baraldi Date: Mon, 17 Feb 2025 11:40:19 -0600 Subject: [PATCH 03/10] Disable tests for asan --- .../internal-threading/CMakeLists.txt | 35 ++++++++++++++----- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/tests/rocprofv3/internal-threading/CMakeLists.txt b/tests/rocprofv3/internal-threading/CMakeLists.txt index 63a1839ce9..bc3fcce6a8 100644 --- a/tests/rocprofv3/internal-threading/CMakeLists.txt +++ b/tests/rocprofv3/internal-threading/CMakeLists.txt @@ -13,17 +13,36 @@ find_package(rocprofiler-sdk REQUIRED) add_library(pthread_wrapper SHARED) target_sources(pthread_wrapper PRIVATE pthread_wrapper.cpp) -add_test( - NAME rocprofv3-test-internal-threading - COMMAND - $ --sys-trace --pmc SQ_WAVES - --preload $ -- $ 1 15) - string(REPLACE "LD_PRELOAD=" "ROCPROF_PRELOAD=" PRELOAD_ENV "${ROCPROFILER_MEMCHECK_PRELOAD_ENV}") +if(DEFINED ROCPROFILER_MEMCHECK AND NOT ${ROCPROFILER_MEMCHECK} STREQUAL "") + set(IS_DISABLED True) +else() + set(IS_DISABLED False) +endif() + +message(STATUS "IS_DISABLED: ${IS_DISABLED} - MEMCHECK: ${ROCPROFILER_MEMCHECK}") + +add_test( + NAME rocprofv3-test-internal-threading-pmc + COMMAND $ --pmc SQ_WAVES + --preload $ -- $ 1 15) + +set_tests_properties( + rocprofv3-test-internal-threading-pmc + PROPERTIES TIMEOUT 45 LABELS "integration-tests" ENVIRONMENT + "${PRELOAD_ENV}" FAIL_REGULAR_EXPRESSION + "${ROCPROFILER_DEFAULT_FAIL_REGEX}" DISABLED ${IS_DISABLED}) + + +add_test( + NAME rocprofv3-test-internal-threading-sys-trace + COMMAND $ --sys-trace + --preload $ -- $ 1 15) + set_tests_properties( - rocprofv3-test-internal-threading + rocprofv3-test-internal-threading-sys-trace PROPERTIES TIMEOUT 45 LABELS "integration-tests" ENVIRONMENT "${PRELOAD_ENV}" FAIL_REGULAR_EXPRESSION - "${ROCPROFILER_DEFAULT_FAIL_REGEX}") + "${ROCPROFILER_DEFAULT_FAIL_REGEX}" DISABLED ${IS_DISABLED}) From d4a69e83806684eecedc15fee99e502def46d1e9 Mon Sep 17 00:00:00 2001 From: Giovanni Baraldi Date: Mon, 17 Feb 2025 11:46:54 -0600 Subject: [PATCH 04/10] Formatting --- .../internal-threading/CMakeLists.txt | 46 +++++++++++-------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/tests/rocprofv3/internal-threading/CMakeLists.txt b/tests/rocprofv3/internal-threading/CMakeLists.txt index bc3fcce6a8..c5245387fa 100644 --- a/tests/rocprofv3/internal-threading/CMakeLists.txt +++ b/tests/rocprofv3/internal-threading/CMakeLists.txt @@ -16,33 +16,43 @@ target_sources(pthread_wrapper PRIVATE pthread_wrapper.cpp) string(REPLACE "LD_PRELOAD=" "ROCPROF_PRELOAD=" PRELOAD_ENV "${ROCPROFILER_MEMCHECK_PRELOAD_ENV}") +# Disable test when using memcheck: The test needs to leak memory. if(DEFINED ROCPROFILER_MEMCHECK AND NOT ${ROCPROFILER_MEMCHECK} STREQUAL "") set(IS_DISABLED True) else() set(IS_DISABLED False) endif() -message(STATUS "IS_DISABLED: ${IS_DISABLED} - MEMCHECK: ${ROCPROFILER_MEMCHECK}") - -add_test( - NAME rocprofv3-test-internal-threading-pmc - COMMAND $ --pmc SQ_WAVES - --preload $ -- $ 1 15) +add_test(NAME rocprofv3-test-internal-threading-pmc + COMMAND $ --pmc SQ_WAVES --preload + $ -- $ 1 15) set_tests_properties( rocprofv3-test-internal-threading-pmc - PROPERTIES TIMEOUT 45 LABELS "integration-tests" ENVIRONMENT - "${PRELOAD_ENV}" FAIL_REGULAR_EXPRESSION - "${ROCPROFILER_DEFAULT_FAIL_REGEX}" DISABLED ${IS_DISABLED}) - - -add_test( - NAME rocprofv3-test-internal-threading-sys-trace - COMMAND $ --sys-trace - --preload $ -- $ 1 15) + PROPERTIES TIMEOUT + 45 + LABELS + "integration-tests" + ENVIRONMENT + "${PRELOAD_ENV}" + FAIL_REGULAR_EXPRESSION + "${ROCPROFILER_DEFAULT_FAIL_REGEX}" + DISABLED + ${IS_DISABLED}) + +add_test(NAME rocprofv3-test-internal-threading-sys-trace + COMMAND $ --sys-trace --preload + $ -- $ 1 15) set_tests_properties( rocprofv3-test-internal-threading-sys-trace - PROPERTIES TIMEOUT 45 LABELS "integration-tests" ENVIRONMENT - "${PRELOAD_ENV}" FAIL_REGULAR_EXPRESSION - "${ROCPROFILER_DEFAULT_FAIL_REGEX}" DISABLED ${IS_DISABLED}) + PROPERTIES TIMEOUT + 45 + LABELS + "integration-tests" + ENVIRONMENT + "${PRELOAD_ENV}" + FAIL_REGULAR_EXPRESSION + "${ROCPROFILER_DEFAULT_FAIL_REGEX}" + DISABLED + ${IS_DISABLED}) From 4d1fe32ae4e5f4c81ddc7c40b70184b14272e267 Mon Sep 17 00:00:00 2001 From: Giovanni Baraldi Date: Wed, 26 Feb 2025 10:15:12 -0600 Subject: [PATCH 05/10] Address review comments --- .../internal-threading/pthread_wrapper.cpp | 79 ++++++++++++++++--- 1 file changed, 67 insertions(+), 12 deletions(-) diff --git a/tests/rocprofv3/internal-threading/pthread_wrapper.cpp b/tests/rocprofv3/internal-threading/pthread_wrapper.cpp index 1c1fea5e1d..896f2a5e83 100644 --- a/tests/rocprofv3/internal-threading/pthread_wrapper.cpp +++ b/tests/rocprofv3/internal-threading/pthread_wrapper.cpp @@ -1,3 +1,33 @@ +// MIT License +// +// Copyright (c) 2024-2025 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +/** + * This test is supposed to check if rocprofiler is wrapping all threads inside calls to: + * rocprofiler_internal_thread_library_cb_t precreate + * rocprofiler_internal_thread_library_cb_t postcreate + * Currently, Runtime threads are not reported, so a tolerance "MAX_ALLOWED_THREADS" was added. + * Some memory needs to be leaked due to global destructor issues: Test disabled for sanitizers. + */ + #include #include #include @@ -17,8 +47,11 @@ abort(); \ } +// Tolerance for amount of threads not inside a pre/post callback +// "3" is the number seen for the given application #define MAX_ALLOWED_THREADS 3 +// We should not link to librocprofiler-sdk.so, so we need to define the callback types typedef void (*rocprofiler_internal_thread_library_cb_t)(int, void*); typedef int (*rocprofiler_at_internal_thread_create_t)( rocprofiler_internal_thread_library_cb_t precreate, @@ -26,46 +59,59 @@ typedef int (*rocprofiler_at_internal_thread_create_t)( int libs, void* data); +// Used in pthread_create typedef void*(routine_t)(void*); +// Bitmask for each library that is inside a pre callback +size_t& get_library_bitmask() +{ + // Pre-post callbacks are supposed to be called from the same thread as pthread_create + thread_local auto* bitmask = new size_t{0}; + return *bitmask; +} + class RegisterProfiler { public: RegisterProfiler() = default; + // Returns true if rocprofiler has already been registered before this call bool try_register() { if(init.load()) return true; auto lk = std::unique_lock{mut}; + // Case for which two threads were waiting on the lock. No need to initialize again. if(init.load()) return false; auto* handle = dlopen("librocprofiler-sdk.so", RTLD_LAZY | RTLD_NOLOAD); + // If handle is nullptr, that means pthread_create was called before + // librocprofiler-sdk.so initialized. We try again later. if(!handle) return false; auto* register_fn = (rocprofiler_at_internal_thread_create_t) dlsym( handle, "rocprofiler_at_internal_thread_create"); ASSERT(register_fn, "Could not dlsym rocprofiler library"); - register_fn(pre_callback, post_callback, ~0, this); + register_fn(pre_callback, post_callback, ~0, nullptr); dlclose(handle); init.store(true); return false; } - static void pre_callback(int bitmask, void* arg) + static void pre_callback(int bitmask, void* /* arg */) { - static_cast(arg)->pre_library_list.fetch_or(bitmask); + get_library_bitmask() |= bitmask; } - static void post_callback(int bitmask, void* arg) + static void post_callback(int bitmask, void* /* arg */) { - static_cast(arg)->pre_library_list.fetch_and(~bitmask); + get_library_bitmask() &= ~bitmask; } - std::atomic pre_library_list{0}; - std::atomic init{false}; - std::mutex mut; + // Indicates we have registered rocprofiler_at_internal_thread_create_t + std::atomic init{false}; + std::mutex mut{}; }; class DL @@ -97,12 +143,21 @@ pthread_create(pthread_t* thread, const pthread_attr_t* attr, routine_t* start_r static auto* reg = new RegisterProfiler(); static auto* dl = new DL(); - static std::atomic unwrapped_threads{0}; + // Number of threads not wrapped inside a pre/post callbac + static auto* unwrapped_threads = new std::atomic{0}; - if(reg->try_register() && reg->pre_library_list.load() == 0) unwrapped_threads++; + // If try_register returns false, SDK was not loaded yet or has just been loaded. + // We have to ignore the first initialization because we may have just missed a pre-callback. + // Comparing to zero checks if any library has called pre-callback. + if(reg->try_register() && get_library_bitmask() == 0) + { + size_t count = unwrapped_threads->fetch_add(1); - ASSERT(unwrapped_threads <= MAX_ALLOWED_THREADS, - "Limit reached for number of thread not inside a pre/post callback!"); + // This will fail if either rocprofiler does not wrap a thread, + // or if the runtime decides to create new threads. TODO: HIP/HSA callbacks + ASSERT(count < MAX_ALLOWED_THREADS, + "Limit reached for number of threads not inside a pre/post callback!"); + } return dl->pthread_create_fn(thread, attr, start_routine, arg); } From 279412007e116db06ca9260414987b7834591bf2 Mon Sep 17 00:00:00 2001 From: "Baraldi, Giovanni" Date: Wed, 26 Feb 2025 18:42:13 +0100 Subject: [PATCH 06/10] Update tests/rocprofv3/CMakeLists.txt --- tests/rocprofv3/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/rocprofv3/CMakeLists.txt b/tests/rocprofv3/CMakeLists.txt index ff784f816c..19d0983408 100644 --- a/tests/rocprofv3/CMakeLists.txt +++ b/tests/rocprofv3/CMakeLists.txt @@ -39,7 +39,6 @@ add_subdirectory(collection-period) add_subdirectory(internal-threading) add_subdirectory(rocdecode-trace) add_subdirectory(rocjpeg-trace) - if(TARGET att_decoder_testing) add_subdirectory(advanced-thread-trace) endif() From f6b7a7e272602cba69db5664e4ffc0c6b2627f04 Mon Sep 17 00:00:00 2001 From: Giovanni Baraldi Date: Wed, 26 Feb 2025 16:54:31 -0600 Subject: [PATCH 07/10] Format --- .../internal-threading/pthread_wrapper.cpp | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/tests/rocprofv3/internal-threading/pthread_wrapper.cpp b/tests/rocprofv3/internal-threading/pthread_wrapper.cpp index 896f2a5e83..a51d1a97e9 100644 --- a/tests/rocprofv3/internal-threading/pthread_wrapper.cpp +++ b/tests/rocprofv3/internal-threading/pthread_wrapper.cpp @@ -63,7 +63,8 @@ typedef int (*rocprofiler_at_internal_thread_create_t)( typedef void*(routine_t)(void*); // Bitmask for each library that is inside a pre callback -size_t& get_library_bitmask() +size_t& +get_library_bitmask() { // Pre-post callbacks are supposed to be called from the same thread as pthread_create thread_local auto* bitmask = new size_t{0}; @@ -99,15 +100,9 @@ class RegisterProfiler return false; } - static void pre_callback(int bitmask, void* /* arg */) - { - get_library_bitmask() |= bitmask; - } + static void pre_callback(int bitmask, void* /* arg */) { get_library_bitmask() |= bitmask; } - static void post_callback(int bitmask, void* /* arg */) - { - get_library_bitmask() &= ~bitmask; - } + static void post_callback(int bitmask, void* /* arg */) { get_library_bitmask() &= ~bitmask; } // Indicates we have registered rocprofiler_at_internal_thread_create_t std::atomic init{false}; From f06dd1d4519cd25d7a59abef7f3b4e6259b031a5 Mon Sep 17 00:00:00 2001 From: Giovanni Baraldi Date: Mon, 3 Mar 2025 16:38:52 -0600 Subject: [PATCH 08/10] Review comments --- .../internal-threading/CMakeLists.txt | 2 +- .../internal-threading/pthread_wrapper.cpp | 66 +++++++++++++------ 2 files changed, 47 insertions(+), 21 deletions(-) diff --git a/tests/rocprofv3/internal-threading/CMakeLists.txt b/tests/rocprofv3/internal-threading/CMakeLists.txt index c5245387fa..c6f1d212f5 100644 --- a/tests/rocprofv3/internal-threading/CMakeLists.txt +++ b/tests/rocprofv3/internal-threading/CMakeLists.txt @@ -17,7 +17,7 @@ string(REPLACE "LD_PRELOAD=" "ROCPROF_PRELOAD=" PRELOAD_ENV "${ROCPROFILER_MEMCHECK_PRELOAD_ENV}") # Disable test when using memcheck: The test needs to leak memory. -if(DEFINED ROCPROFILER_MEMCHECK AND NOT ${ROCPROFILER_MEMCHECK} STREQUAL "") +if("${ROCPROFILER_MEMCHECK}" STREQUAL "LeakSanitizer") set(IS_DISABLED True) else() set(IS_DISABLED False) diff --git a/tests/rocprofv3/internal-threading/pthread_wrapper.cpp b/tests/rocprofv3/internal-threading/pthread_wrapper.cpp index a51d1a97e9..53b6666d36 100644 --- a/tests/rocprofv3/internal-threading/pthread_wrapper.cpp +++ b/tests/rocprofv3/internal-threading/pthread_wrapper.cpp @@ -34,11 +34,7 @@ #include #include -#if defined(_MSC_VER) -# define ROCPROFILER_PUBLIC_API __declspec(dllexport) -#else -# define ROCPROFILER_PUBLIC_API __attribute__((visibility("default"))) -#endif +#define PUBLIC_API __attribute__((visibility("default"))) #define ASSERT(x, msg) \ if(!(x)) \ @@ -62,6 +58,11 @@ typedef int (*rocprofiler_at_internal_thread_create_t)( // Used in pthread_create typedef void*(routine_t)(void*); +namespace +{ +// Number of calls to pthread_create sucessfuly wrapped in pre/post callbacks +std::atomic wrapped_threads{0}; + // Bitmask for each library that is inside a pre callback size_t& get_library_bitmask() @@ -71,6 +72,18 @@ get_library_bitmask() return *bitmask; } +void +pre_callback(int bitmask, void* /* arg */) +{ + get_library_bitmask() |= bitmask; +} + +void +post_callback(int bitmask, void* /* arg */) +{ + get_library_bitmask() &= ~bitmask; +} + class RegisterProfiler { public: @@ -100,10 +113,6 @@ class RegisterProfiler return false; } - static void pre_callback(int bitmask, void* /* arg */) { get_library_bitmask() |= bitmask; } - - static void post_callback(int bitmask, void* /* arg */) { get_library_bitmask() &= ~bitmask; } - // Indicates we have registered rocprofiler_at_internal_thread_create_t std::atomic init{false}; std::mutex mut{}; @@ -131,27 +140,44 @@ class DL PthreadFn* pthread_create_fn = nullptr; }; -ROCPROFILER_PUBLIC_API +__attribute__((destructor)) void +check_wrapped_threads() +{ + // Ensures the test has actually run + ASSERT(wrapped_threads.load() > 0, "No thread was created inside a pre/post callback"); +} +} // namespace + +// This function wraps pthread_create defined in pthread.h +PUBLIC_API int pthread_create(pthread_t* thread, const pthread_attr_t* attr, routine_t* start_routine, void* arg) { static auto* reg = new RegisterProfiler(); static auto* dl = new DL(); - // Number of threads not wrapped inside a pre/post callbac - static auto* unwrapped_threads = new std::atomic{0}; + // Number of threads not wrapped inside a pre/post callback + static auto unwrapped_threads = std::atomic{0}; // If try_register returns false, SDK was not loaded yet or has just been loaded. // We have to ignore the first initialization because we may have just missed a pre-callback. - // Comparing to zero checks if any library has called pre-callback. - if(reg->try_register() && get_library_bitmask() == 0) + if(reg->try_register()) { - size_t count = unwrapped_threads->fetch_add(1); - - // This will fail if either rocprofiler does not wrap a thread, - // or if the runtime decides to create new threads. TODO: HIP/HSA callbacks - ASSERT(count < MAX_ALLOWED_THREADS, - "Limit reached for number of threads not inside a pre/post callback!"); + // Comparing to zero checks if any library has called pre-callback. + if(get_library_bitmask() == 0) + { + size_t count = unwrapped_threads.fetch_add(1); + + // This will fail if either rocprofiler does not wrap a thread, + // or if the runtime decides to create new threads. TODO: HIP/HSA callbacks + ASSERT(count < MAX_ALLOWED_THREADS, + "Limit reached for number of threads not inside a pre/post callback!"); + } + else + { + // We have sucessfuly created a thread wrapped in pre/post callbacks + wrapped_threads.fetch_add(1); + } } return dl->pthread_create_fn(thread, attr, start_routine, arg); From 76e4a753063756731e25c54383deaed091e1546c Mon Sep 17 00:00:00 2001 From: Giovanni Baraldi Date: Mon, 3 Mar 2025 17:29:51 -0600 Subject: [PATCH 09/10] Disable TSAN --- .../internal-threading/CMakeLists.txt | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/rocprofv3/internal-threading/CMakeLists.txt b/tests/rocprofv3/internal-threading/CMakeLists.txt index c6f1d212f5..6215f0b102 100644 --- a/tests/rocprofv3/internal-threading/CMakeLists.txt +++ b/tests/rocprofv3/internal-threading/CMakeLists.txt @@ -16,16 +16,18 @@ target_sources(pthread_wrapper PRIVATE pthread_wrapper.cpp) string(REPLACE "LD_PRELOAD=" "ROCPROF_PRELOAD=" PRELOAD_ENV "${ROCPROFILER_MEMCHECK_PRELOAD_ENV}") -# Disable test when using memcheck: The test needs to leak memory. -if("${ROCPROFILER_MEMCHECK}" STREQUAL "LeakSanitizer") +# Disable LeakSanitizer when using memcheck: The test needs to leak memory. +# ThreadSanitizer conflicts with this library. +if("${ROCPROFILER_MEMCHECK}" STREQUAL "LeakSanitizer" OR "${ROCPROFILER_MEMCHECK}" + STREQUAL "ThreadSanitizer") set(IS_DISABLED True) else() set(IS_DISABLED False) endif() add_test(NAME rocprofv3-test-internal-threading-pmc - COMMAND $ --pmc SQ_WAVES --preload - $ -- $ 1 15) + COMMAND $ --pmc SQ_WAVES -- + $ 1 15) set_tests_properties( rocprofv3-test-internal-threading-pmc @@ -34,15 +36,15 @@ set_tests_properties( LABELS "integration-tests" ENVIRONMENT - "${PRELOAD_ENV}" + "${PRELOAD_ENV} $" FAIL_REGULAR_EXPRESSION "${ROCPROFILER_DEFAULT_FAIL_REGEX}" DISABLED ${IS_DISABLED}) add_test(NAME rocprofv3-test-internal-threading-sys-trace - COMMAND $ --sys-trace --preload - $ -- $ 1 15) + COMMAND $ --sys-trace -- + $ 1 15) set_tests_properties( rocprofv3-test-internal-threading-sys-trace @@ -51,7 +53,7 @@ set_tests_properties( LABELS "integration-tests" ENVIRONMENT - "${PRELOAD_ENV}" + "${PRELOAD_ENV} $" FAIL_REGULAR_EXPRESSION "${ROCPROFILER_DEFAULT_FAIL_REGEX}" DISABLED From 360546cd951a86780b9caf8aed5e2d61cc1078f1 Mon Sep 17 00:00:00 2001 From: Giovanni Baraldi Date: Mon, 3 Mar 2025 17:38:46 -0600 Subject: [PATCH 10/10] Refactor --- tests/rocprofv3/internal-threading/CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/rocprofv3/internal-threading/CMakeLists.txt b/tests/rocprofv3/internal-threading/CMakeLists.txt index 6215f0b102..994bf5bcb4 100644 --- a/tests/rocprofv3/internal-threading/CMakeLists.txt +++ b/tests/rocprofv3/internal-threading/CMakeLists.txt @@ -14,7 +14,7 @@ add_library(pthread_wrapper SHARED) target_sources(pthread_wrapper PRIVATE pthread_wrapper.cpp) string(REPLACE "LD_PRELOAD=" "ROCPROF_PRELOAD=" PRELOAD_ENV - "${ROCPROFILER_MEMCHECK_PRELOAD_ENV}") + "${ROCPROFILER_MEMCHECK_PRELOAD_ENV} $") # Disable LeakSanitizer when using memcheck: The test needs to leak memory. # ThreadSanitizer conflicts with this library. @@ -36,7 +36,7 @@ set_tests_properties( LABELS "integration-tests" ENVIRONMENT - "${PRELOAD_ENV} $" + "${PRELOAD_ENV}" FAIL_REGULAR_EXPRESSION "${ROCPROFILER_DEFAULT_FAIL_REGEX}" DISABLED @@ -53,7 +53,7 @@ set_tests_properties( LABELS "integration-tests" ENVIRONMENT - "${PRELOAD_ENV} $" + "${PRELOAD_ENV}" FAIL_REGULAR_EXPRESSION "${ROCPROFILER_DEFAULT_FAIL_REGEX}" DISABLED