diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b713f34..1bf5cc85 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- **Allocation-free uncontended mutex locks**: Non-cancellable locks now defer + shared wake-state allocation until both the initial acquisition and the + suspension-entry recheck observe contention. Truly parked waiters retain + independent wake lifetime and FIFO ownership transfer, while token-aware + locks retain eager cancellation state (#1038). - **Allocation-free ready event waits**: Non-cancellable waits on an already-set manual-reset event now complete without allocating shared wake state. A wait that reaches the unset slow path creates the same independently owned state diff --git a/examples/microbench.cpp b/examples/microbench.cpp index 2e70bde4..74362320 100644 --- a/examples/microbench.cpp +++ b/examples/microbench.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -31,6 +32,21 @@ coro::task ready_event_waits(sync::event& ready_event, } } +coro::task uncontended_mutex_locks(sync::mutex& mutex, + size_t iterations) { + for (size_t i = 0; i < iterations; ++i) { + co_await mutex.lock(); + mutex.unlock(); + } +} + +coro::task mutex_handoffs(sync::mutex& mutex, size_t iterations) { + for (size_t i = 0; i < iterations; ++i) { + co_await mutex.lock(); + mutex.unlock(); + } +} + int main() { log::logger::instance().set_level(log::level::error); @@ -132,7 +148,76 @@ int main() { << " ns/wait" << std::endl; } - // 5. Measure MPSC push only (no scheduler overhead) + // 5. Measure uncontended mutex lock/unlock. As with the ready-event + // benchmark, construction happens before timing and one long-lived frame + // executes the whole loop without scheduler handoffs. + { + constexpr size_t lock_iterations = 1000000; + sync::mutex mutex; + auto locks = uncontended_mutex_locks(mutex, lock_iterations); + auto handle = coro::detail::task_access::handle(locks); + + auto start = high_resolution_clock::now(); + { + coro::detail::frame_context_scope frame_scope( + std::addressof(handle.promise())); + handle.resume(); + } + auto end = high_resolution_clock::now(); + if (!handle.done() || mutex.is_locked()) { + std::abort(); + } + auto ns = duration_cast(end - start).count(); + + std::cout << "Uncontended mutex lock/unlock: " + << (static_cast(ns) / lock_iterations) + << " ns/iteration" << std::endl; + } + + // 6. Measure forced handoff between two long-lived coroutine frames. Both + // frames are parked before timing; unlock then drives an alternating chain + // through the local trampoline without worker scheduling noise. + { + constexpr size_t handoff_iterations_per_task = 100000; + constexpr size_t total_handoffs = handoff_iterations_per_task * 2; + sync::mutex mutex; + if (!mutex.try_lock()) { + std::abort(); + } + + auto first = mutex_handoffs(mutex, handoff_iterations_per_task); + auto second = mutex_handoffs(mutex, handoff_iterations_per_task); + auto first_handle = coro::detail::task_access::handle(first); + auto second_handle = coro::detail::task_access::handle(second); + { + coro::detail::frame_context_scope frame_scope( + std::addressof(first_handle.promise())); + first_handle.resume(); + } + { + coro::detail::frame_context_scope frame_scope( + std::addressof(second_handle.promise())); + second_handle.resume(); + } + if (first_handle.done() || second_handle.done()) { + std::abort(); + } + + auto start = high_resolution_clock::now(); + mutex.unlock(); + auto end = high_resolution_clock::now(); + if (!first_handle.done() || !second_handle.done() || + mutex.is_locked()) { + std::abort(); + } + auto ns = duration_cast(end - start).count(); + + std::cout << "Forced two-task mutex handoff: " + << (static_cast(ns) / total_handoffs) + << " ns/handoff" << std::endl; + } + + // 7. Measure MPSC push only (no scheduler overhead) { runtime::mpsc_queue queue; @@ -149,7 +234,7 @@ int main() { while (queue.pop()) {} } - // 6. Measure Chase-Lev push only + // 8. Measure Chase-Lev push only { runtime::chase_lev_deque queue; @@ -166,7 +251,7 @@ int main() { while (queue.pop()) {} } - // 7. Compare atomic RMW with single-writer snapshot publication + // 9. Compare atomic RMW with single-writer snapshot publication { std::atomic published{0}; @@ -198,7 +283,7 @@ int main() { << " ns/update" << std::endl; } - // 8. Compare exact timestamps with the disabled diagnostic fast path + // 10. Compare exact timestamps with the disabled diagnostic fast path { std::atomic last_task_time{ steady_clock::now()}; @@ -233,7 +318,7 @@ int main() { << " ns/update" << std::endl; } - // 9. Measure atomic fence alone + // 11. Measure atomic fence alone { auto start = high_resolution_clock::now(); for (int i = 0; i < N; ++i) { @@ -245,7 +330,7 @@ int main() { std::cout << "Atomic release fence: " << (ns / N) << " ns" << std::endl; } - // 10. Measure eventfd write + // 12. Measure eventfd write { int fd = eventfd(0, EFD_NONBLOCK); uint64_t val = 1; @@ -261,7 +346,7 @@ int main() { close(fd); } - // 11. Full spawn path (with running scheduler) - includes alloc + spawn + // 13. Full spawn path (with running scheduler) - includes alloc + spawn { runtime::scheduler sched(4); sched.start(); @@ -283,7 +368,7 @@ int main() { sched.shutdown(); } - // 12. Measure warmed-up worker overhead + // 14. Measure warmed-up worker overhead { runtime::scheduler sched(4); sched.start(); diff --git a/include/elio/sync/detail/wake_state.hpp b/include/elio/sync/detail/wake_state.hpp index 0e0edbf9..c1d38c64 100644 --- a/include/elio/sync/detail/wake_state.hpp +++ b/include/elio/sync/detail/wake_state.hpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include "../../runtime/scheduler.hpp" @@ -11,6 +12,7 @@ namespace elio::sync::detail { #ifdef ELIO_RUNTIME_TEST_HOOKS inline std::atomic wake_state_allocations_for_test{0}; +inline std::atomic fail_next_wake_state_allocation_for_test{false}; #endif enum class wake_action { @@ -236,6 +238,12 @@ class wake_state { using wake_state_ptr = std::shared_ptr; inline wake_state_ptr make_wake_state() { +#ifdef ELIO_RUNTIME_TEST_HOOKS + if (fail_next_wake_state_allocation_for_test.exchange( + false, std::memory_order_acq_rel)) { + throw std::bad_alloc(); + } +#endif auto state = std::make_shared(); #ifdef ELIO_RUNTIME_TEST_HOOKS wake_state_allocations_for_test.fetch_add(1, std::memory_order_relaxed); diff --git a/include/elio/sync/mutex.hpp b/include/elio/sync/mutex.hpp index 2cef8b27..e9eaaef9 100644 --- a/include/elio/sync/mutex.hpp +++ b/include/elio/sync/mutex.hpp @@ -2,8 +2,11 @@ #include #include -#include #include +#include +#include +#include +#include #include #include "../coro/cancel_token.hpp" #include "../detail/intrusive_list.hpp" @@ -36,18 +39,16 @@ class mutex { class lock_waiter : public elio::detail::intrusive_list_node { public: explicit lock_waiter(mutex& m) - : mtx_(m) - , wake_state_(detail::make_wake_state()) {} + : mtx_(m) {} lock_waiter(mutex& m, bool cancellable) : mtx_(m) - , wake_state_(detail::make_wake_state()) - , cancellable_(cancellable) {} + , waiter_state_(cancellable) {} ~lock_waiter() { // Fast path: if we never suspended, we were never enqueued, // so no wake function could hold a reference to us. - if (!suspended_) return; + if (!waiter_state_.suspended) return; detail::wake_state_ptr to_schedule; // Slow path: acquire internal_mutex_ to prevent race with unlock() @@ -55,13 +56,14 @@ class mutex { std::lock_guard guard(mtx_.internal_mutex_); if (this->is_linked()) { mtx_.waiters_.remove(this); - detail::cancel_wake_state(wake_state_); - } else if (grant_pending_ && !resumed_) { - detail::cancel_wake_state(wake_state_); - grant_pending_ = false; + detail::cancel_wake_state(waiter_state_.wake()); + } else if (waiter_state_.grant_pending && + !waiter_state_.resumed) { + detail::cancel_wake_state(waiter_state_.wake()); + waiter_state_.grant_pending = false; to_schedule = mtx_.recover_cancelled_handoff_locked(); } else { - detail::cancel_wake_state(wake_state_); + detail::cancel_wake_state(waiter_state_.wake()); } } @@ -71,17 +73,17 @@ class mutex { } bool await_ready_impl() const noexcept { - if (!cancellable_) { + if (!waiter_state_.cancellable) { return mtx_.try_lock(); } - if (wake_state_->was_cancelled()) { + if (waiter_state_->was_cancelled()) { return true; } if (!mtx_.try_lock()) { return false; } - if (detail::claim_wake_state(wake_state_) != + if (detail::claim_wake_state(waiter_state_.wake()) != detail::wake_action::rejected) { return true; } @@ -91,35 +93,55 @@ class mutex { return true; } - bool await_suspend_impl(std::coroutine_handle<> awaiter) noexcept { - if (!cancellable_) { - void* expected = nullptr; - if (mtx_.state_.compare_exchange_strong( - expected, awaiter.address(), - std::memory_order_acq_rel, std::memory_order_acquire)) { - return false; - } + bool await_suspend_impl(std::coroutine_handle<> awaiter) { + if (waiter_state_.cancellable) { + return await_suspend_cancellable_impl(awaiter); + } + return await_suspend_non_cancellable_impl(awaiter); + } - std::lock_guard guard(mtx_.internal_mutex_); - expected = nullptr; - if (mtx_.state_.compare_exchange_strong( - expected, awaiter.address(), - std::memory_order_acq_rel, std::memory_order_acquire)) { - return false; - } + bool await_suspend_non_cancellable_impl( + std::coroutine_handle<> awaiter) { + assert(!waiter_state_.cancellable); - wake_state_->set_handle(awaiter); - mtx_.waiters_.push_back(this); - suspended_ = true; - return true; + // Recheck before allocating: unlock() may have released the mutex + // after await_ready() observed contention. + void* expected = nullptr; + if (mtx_.state_.compare_exchange_strong( + expected, awaiter.address(), + std::memory_order_acq_rel, std::memory_order_acquire)) { + return false; } + // A dequeued notifier may outlive the coroutine frame, so a + // genuinely contended wait still needs independent ownership. + // Allocate before taking the queue lock so failure leaves both the + // mutex state and waiter queue unchanged. + waiter_state_.emplace(); + + std::lock_guard guard(mtx_.internal_mutex_); + expected = nullptr; + if (mtx_.state_.compare_exchange_strong( + expected, awaiter.address(), + std::memory_order_acq_rel, std::memory_order_acquire)) { + return false; + } + + waiter_state_->set_handle(awaiter); + mtx_.waiters_.push_back(this); + waiter_state_.suspended = true; + return true; + } + + bool await_suspend_cancellable_impl( + std::coroutine_handle<> awaiter) noexcept { + assert(waiter_state_.cancellable); detail::wake_state_ptr to_schedule; { // Lock is held, add to wait queue std::lock_guard guard(mtx_.internal_mutex_); - if (wake_state_->was_cancelled()) { + if (waiter_state_->was_cancelled()) { return false; } @@ -128,7 +150,7 @@ class mutex { if (mtx_.state_.compare_exchange_strong( expected, awaiter.address(), std::memory_order_acq_rel, std::memory_order_acquire)) { - if (detail::claim_wake_state(wake_state_) != + if (detail::claim_wake_state(waiter_state_.wake()) != detail::wake_action::rejected) { return false; } @@ -137,18 +159,18 @@ class mutex { // another live waiter, or release it when none remain. to_schedule = mtx_.recover_cancelled_handoff_locked(); } else { - if (!wake_state_->set_handle_blocked(awaiter)) { + if (!waiter_state_->set_handle_blocked(awaiter)) { return false; } mtx_.waiters_.push_back(this); - suspended_ = true; + waiter_state_.suspended = true; - if (wake_state_->unblock_after_publish()) { + if (waiter_state_->unblock_after_publish()) { return true; } mtx_.waiters_.remove(this); - suspended_ = false; + waiter_state_.suspended = false; } } @@ -159,42 +181,111 @@ class mutex { } coro::cancel_result await_resume_impl() noexcept { - if (!cancellable_) { - resumed_ = true; - grant_pending_ = false; - suspended_ = false; + if (!waiter_state_.cancellable) { + waiter_state_.resumed = true; + waiter_state_.grant_pending = false; + waiter_state_.suspended = false; return coro::cancel_result::completed; } - if (wake_state_->was_cancelled()) { - if (suspended_) { + if (waiter_state_->was_cancelled()) { + if (waiter_state_.suspended) { std::lock_guard guard(mtx_.internal_mutex_); if (this->is_linked()) { mtx_.waiters_.remove(this); } - suspended_ = false; + waiter_state_.suspended = false; } return coro::cancel_result::cancelled; } - resumed_ = true; - grant_pending_ = false; - suspended_ = false; + waiter_state_.resumed = true; + waiter_state_.grant_pending = false; + waiter_state_.suspended = false; return coro::cancel_result::completed; } protected: const detail::wake_state_ptr& cancellation_wake_state() const noexcept { - return wake_state_; + return waiter_state_.wake(); } private: + class waiter_state { + public: + waiter_state() noexcept = default; + + explicit waiter_state(bool is_cancellable) + : cancellable(is_cancellable) { + if (is_cancellable) { + ::new (static_cast(storage_)) + detail::wake_state_ptr(detail::make_wake_state()); + // Publish engagement only after placement construction + // succeeds; a throwing allocation leaves no active object. + engaged = true; + } + } + + ~waiter_state() { + if (engaged) { + // Release shared wake ownership as this member's final + // non-trivial destruction step. + std::destroy_at(std::addressof(storage_ref())); + } + } + + waiter_state(const waiter_state&) = delete; + waiter_state& operator=(const waiter_state&) = delete; + waiter_state(waiter_state&&) = delete; + waiter_state& operator=(waiter_state&&) = delete; + + void emplace() { + assert(!engaged); + ::new (static_cast(storage_)) + detail::wake_state_ptr(detail::make_wake_state()); + // A failed allocation leaves the slot disengaged. + engaged = true; + } + + [[nodiscard]] const detail::wake_state_ptr& wake() const noexcept { + assert(engaged); + return storage_ref(); + } + + [[nodiscard]] detail::wake_state* operator->() const noexcept { + return wake().get(); + } + + bool engaged = false; + bool cancellable = false; + bool suspended = false; + bool resumed = false; + bool grant_pending = false; + + private: + detail::wake_state_ptr& storage_ref() noexcept { + return *std::launder(storage_ptr()); + } + + const detail::wake_state_ptr& storage_ref() const noexcept { + return *std::launder(storage_ptr()); + } + + detail::wake_state_ptr* storage_ptr() noexcept { + return reinterpret_cast(storage_); + } + + const detail::wake_state_ptr* storage_ptr() const noexcept { + return reinterpret_cast( + storage_); + } + + alignas(detail::wake_state_ptr) + std::byte storage_[sizeof(detail::wake_state_ptr)]; + }; + mutex& mtx_; - detail::wake_state_ptr wake_state_; - bool cancellable_ = false; - bool suspended_ = false; // True if enqueued in waiters_ - bool resumed_ = false; // True after a popped waiter resumes normally - bool grant_pending_ = false; // True after unlock() transfers ownership + waiter_state waiter_state_; friend class mutex; }; @@ -218,6 +309,10 @@ class mutex { return await_resume_impl(); } + bool await_suspend_impl(std::coroutine_handle<> awaiter) noexcept { + return await_suspend_cancellable_impl(awaiter); + } + private: coro::cancel_token::registration cancel_registration_; }; @@ -229,8 +324,8 @@ class mutex { explicit lock_awaitable(mutex& m) : waiter_(m) {} bool await_ready() const noexcept { return waiter_.await_ready_impl(); } - bool await_suspend(std::coroutine_handle<> awaiter) noexcept { - return waiter_.await_suspend_impl(awaiter); + bool await_suspend(std::coroutine_handle<> awaiter) { + return waiter_.await_suspend_non_cancellable_impl(awaiter); } void await_resume() noexcept { (void)waiter_.await_resume_impl(); @@ -281,7 +376,9 @@ class mutex { mutex& mtx_; }; - /// Lock the mutex (coroutine-aware). + /// Lock the mutex (coroutine-aware). An uncontended lock completes without + /// allocation. A contended lock allocates shared wake state and can + /// propagate std::bad_alloc from await_suspend(). [[nodiscard]] lock_awaitable lock() { return lock_awaitable(*this); } @@ -327,17 +424,17 @@ class mutex { detail::wake_state_ptr recover_cancelled_handoff_locked() noexcept { while (!waiters_.empty()) { auto* waiter = waiters_.pop_front(); - if (waiter->cancellable_) { + if (waiter->waiter_state_.cancellable) { const auto action = - detail::claim_wake_state(waiter->wake_state_); + detail::claim_wake_state(waiter->waiter_state_.wake()); if (action == detail::wake_action::rejected) { continue; } } state_.store(reinterpret_cast(1), std::memory_order_release); - waiter->grant_pending_ = true; - return waiter->wake_state_; + waiter->waiter_state_.grant_pending = true; + return waiter->waiter_state_.wake(); } state_.store(nullptr, std::memory_order_release); diff --git a/tests/unit/test_sync_cancellation.cpp b/tests/unit/test_sync_cancellation.cpp index 19bfee3d..6a44b3f5 100644 --- a/tests/unit/test_sync_cancellation.cpp +++ b/tests/unit/test_sync_cancellation.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -241,6 +242,181 @@ TEST_CASE("event ready path avoids wake-state allocation", } } +TEST_CASE("mutex fast paths defer wake-state allocation", + "[sync][mutex][allocation]") { + auto& allocations = + elio::sync::detail::wake_state_allocations_for_test; + auto& fail_next = + elio::sync::detail::fail_next_wake_state_allocation_for_test; + fail_next.store(false, std::memory_order_relaxed); + + SECTION("uncontended lock") { + mutex m; + allocations.store(0, std::memory_order_relaxed); + + for (int i = 0; i < 4; ++i) { + auto waiter = m.lock(); + static_assert(!noexcept( + waiter.await_suspend(std::noop_coroutine()))); + REQUIRE(waiter.await_ready()); + waiter.await_resume(); + REQUIRE(m.is_locked()); + m.unlock(); + } + + REQUIRE(allocations.load(std::memory_order_relaxed) == 0); + } + + SECTION("unlock between ready and suspend") { + mutex m; + REQUIRE(m.try_lock()); + allocations.store(0, std::memory_order_relaxed); + + auto waiter = m.lock(); + REQUIRE_FALSE(waiter.await_ready()); + m.unlock(); + REQUIRE_FALSE(waiter.await_suspend(std::noop_coroutine())); + waiter.await_resume(); + + REQUIRE(allocations.load(std::memory_order_relaxed) == 0); + REQUIRE(m.is_locked()); + m.unlock(); + } + + SECTION("explicit non-cancellable waiter configuration stays lazy") { + mutex m; + allocations.store(0, std::memory_order_relaxed); + + mutex::lock_waiter ready_waiter(m, false); + REQUIRE(ready_waiter.await_ready_impl()); + REQUIRE(ready_waiter.await_resume_impl() == cancel_result::completed); + REQUIRE(allocations.load(std::memory_order_relaxed) == 0); + REQUIRE(m.is_locked()); + + mutex::lock_waiter parked_waiter(m, false); + REQUIRE_FALSE(parked_waiter.await_ready_impl()); + REQUIRE(parked_waiter.await_suspend_impl(std::noop_coroutine())); + REQUIRE(allocations.load(std::memory_order_relaxed) == 1); + + m.unlock(); + REQUIRE(m.is_locked()); + REQUIRE(parked_waiter.await_resume_impl() == cancel_result::completed); + m.unlock(); + REQUIRE_FALSE(m.is_locked()); + } + + SECTION("explicit cancellable waiter configuration dispatches safely") { + mutex m; + REQUIRE(m.try_lock()); + allocations.store(0, std::memory_order_relaxed); + + mutex::lock_waiter waiter(m, true); + REQUIRE(allocations.load(std::memory_order_relaxed) == 1); + REQUIRE_FALSE(waiter.await_ready_impl()); + REQUIRE(waiter.await_suspend_impl(std::noop_coroutine())); + + m.unlock(); + REQUIRE(m.is_locked()); + REQUIRE(waiter.await_resume_impl() == cancel_result::completed); + m.unlock(); + REQUIRE_FALSE(m.is_locked()); + } + + SECTION("parked lock") { + mutex m; + REQUIRE(m.try_lock()); + allocations.store(0, std::memory_order_relaxed); + + auto waiter = m.lock(); + REQUIRE_FALSE(waiter.await_ready()); + REQUIRE(waiter.await_suspend(std::noop_coroutine())); + REQUIRE(allocations.load(std::memory_order_relaxed) == 1); + + m.unlock(); + REQUIRE(m.is_locked()); + waiter.await_resume(); + m.unlock(); + REQUIRE_FALSE(m.is_locked()); + } + + SECTION("cancellable ready lock keeps eager arbitration state") { + mutex m; + cancel_source source; + allocations.store(0, std::memory_order_relaxed); + + auto waiter = m.lock(source.get_token()); + static_assert(noexcept( + waiter.await_suspend(std::noop_coroutine()))); + REQUIRE(allocations.load(std::memory_order_relaxed) == 1); + REQUIRE(waiter.await_ready()); + REQUIRE(waiter.await_resume() == cancel_result::completed); + REQUIRE(m.is_locked()); + m.unlock(); + } + + SECTION("cancellable parked lock keeps eager arbitration state") { + mutex m; + REQUIRE(m.try_lock()); + cancel_source source; + allocations.store(0, std::memory_order_relaxed); + + auto waiter = m.lock(source.get_token()); + REQUIRE(allocations.load(std::memory_order_relaxed) == 1); + REQUIRE_FALSE(waiter.await_ready()); + REQUIRE(waiter.await_suspend(std::noop_coroutine())); + + m.unlock(); + REQUIRE(m.is_locked()); + REQUIRE(waiter.await_resume() == cancel_result::completed); + m.unlock(); + REQUIRE_FALSE(m.is_locked()); + } + + SECTION("allocation failure leaves mutex and queue unchanged") { + mutex m; + REQUIRE(m.try_lock()); + allocations.store(0, std::memory_order_relaxed); + + { + auto waiter = m.lock(); + REQUIRE_FALSE(waiter.await_ready()); + fail_next.store(true, std::memory_order_release); + REQUIRE_THROWS_AS( + waiter.await_suspend(std::noop_coroutine()), std::bad_alloc); + } + + REQUIRE(allocations.load(std::memory_order_relaxed) == 0); + REQUIRE(m.is_locked()); + m.unlock(); + REQUIRE_FALSE(m.is_locked()); + + auto next = m.lock(); + REQUIRE(next.await_ready()); + next.await_resume(); + REQUIRE(allocations.load(std::memory_order_relaxed) == 0); + m.unlock(); + } + + SECTION("cancellable construction failure leaves mutex unchanged") { + mutex m; + cancel_source source; + allocations.store(0, std::memory_order_relaxed); + fail_next.store(true, std::memory_order_release); + + REQUIRE_THROWS_AS( + (void)m.lock(source.get_token()), std::bad_alloc); + + REQUIRE_FALSE(fail_next.load(std::memory_order_acquire)); + REQUIRE(allocations.load(std::memory_order_relaxed) == 0); + REQUIRE_FALSE(m.is_locked()); + + auto next = m.lock(); + REQUIRE(next.await_ready()); + next.await_resume(); + m.unlock(); + } +} + TEST_CASE("runtime cancellation wakes basic sync waits", "[sync][cancellation][cancel_token][runtime]") { scheduler sched(2); diff --git a/wiki/API-Reference.md b/wiki/API-Reference.md index 95a875ec..c7f8fd62 100644 --- a/wiki/API-Reference.md +++ b/wiki/API-Reference.md @@ -3224,6 +3224,14 @@ public: }; ``` +The no-token `lock()` completes without allocation when the mutex is +uncontended, including when an unlock wins the race between `await_ready()` and +`await_suspend()`. A wait that remains contended creates independently owned +wake state before entering the waiter-queue critical section, so +`await_suspend()` may propagate `std::bad_alloc` without changing lock or queue +state. Token-aware waits create their arbitration state eagerly so cancellation +can race ownership transfer with exactly one terminal result. + ### `shared_mutex` Coroutine-aware read-write lock. Allows multiple concurrent readers or a single exclusive writer. diff --git a/wiki/Performance-Tuning.md b/wiki/Performance-Tuning.md index b5cf3b3c..386da46b 100644 --- a/wiki/Performance-Tuning.md +++ b/wiki/Performance-Tuning.md @@ -434,7 +434,16 @@ suspension. ### Mutex Performance -Elio's mutex uses atomic fast-path for uncontended cases: +Elio's mutex uses an atomic, allocation-free fast path for non-cancellable, +uncontended locks. It rechecks that fast path at suspension entry, so an unlock +that wins the `await_ready()` / `await_suspend()` race also avoids allocation. +A wait that remains contended at that recheck creates independently owned wake +state before taking the queue lock. It enters the FIFO waiter queue only if the +final locked acquisition recheck also fails; otherwise it acquires without +parking and releases the unused wake state. Token-aware locks create arbitration +state eagerly so cancellation can race ownership transfer safely. This deferral +favors workloads with a meaningful uncontended fraction; a permanently +contended handoff loop can be slightly slower. ```cpp #include