diff --git a/include/beman/big_int/basic_big_int.hpp b/include/beman/big_int/basic_big_int.hpp index 7de383de..99d642b4 100644 --- a/include/beman/big_int/basic_big_int.hpp +++ b/include/beman/big_int/basic_big_int.hpp @@ -620,6 +620,13 @@ class BEMAN_BIG_INT_TRIVIAL_ABI basic_big_int { constexpr void free_limbs(pointer p, size_type n); constexpr void free_storage(); constexpr void grow(size_type limbs_needed); + // Ensures room for `n` limbs and discards the contents (no copy, no zero-fill). Does not change the limb + // count or sign. The caller writes the limbs, sets the count, and zeroes any in-place tail + // (see `clear_inline_tail`). No operand of the caller may alias `*this`. + [[nodiscard]] constexpr limb_type* storage_for_overwrite(size_type n); + // For in-place storage, zeroes the limbs in [limb_count(), old_count). Keeps the invariant + // that in-place limbs past the limb count are zero, which `inplace_to_bit_uint` relies on. + constexpr void clear_inline_tail(size_type old_count) noexcept; constexpr void copy_n_to_allocation(const limb_type* p, size_type n, alloc_result out); // Copies the limbs of `x` into a freshly constructed `*this` whose control word already matches `x`. constexpr void copy_limbs_from(const basic_big_int& x); @@ -643,6 +650,10 @@ class BEMAN_BIG_INT_TRIVIAL_ABI basic_big_int { constexpr void shift_left(shift_type s); constexpr void shift_right(shift_type s); + // `*this = (src, neg) << s` / `>> s` (floor rounding) in a single pass. `src` must be canonical and + // must not alias `*this`. + constexpr void assign_shifted_left(std::span src, bool neg, shift_type s); + constexpr void assign_shifted_right(std::span src, bool neg, shift_type s); template [[nodiscard]] constexpr bool equals_integer(Integer x) const noexcept; @@ -713,13 +724,38 @@ class BEMAN_BIG_INT_TRIVIAL_ABI basic_big_int { constexpr uint_multiprecision_t divmod_in_place_short(uint_multiprecision_t divisor, bool divisor_neg, detail::division_op op); + // `*this *= b`, keeping the allocator. `b` may alias `*this`. A one-limb `b` multiplies in place; a product + // of at most 64 limbs is formed on the stack at run time; larger products go to a fresh buffer that replaces + // the old one only on success. + template + constexpr void multiply_in_place(std::span b, bool b_neg); + + // `*this /= divisor` or `*this %= divisor` for a multi-limb divisor in the schoolbook band, at run + // time and with all scratch on the stack: the dividend is divided where it sits, so nothing allocates. + // Returns false, leaving `*this` untouched, when the shape does not qualify. + template + constexpr bool divide_in_place_small(std::span divisor, + bool divisor_neg, + detail::division_op op); + + // A value holding exactly the trimmed magnitude `mag` (non-empty), with allocator `a`. + [[nodiscard]] static constexpr basic_big_int + make_from_magnitude(std::span mag, bool neg, const allocator_type& a); + // Shared implementation behind copy-assign, move-assign, and the lvalue // branches of `operator+` / `operator-`. - // Sets `*this` to a copy of `src`, reusing the existing allocation whenever - // the effective capacity already fits `src.limb_count() + extra_space` limbs. - // `extra_space` lets callers that know they are about to grow by a fixed - // amount (e.g., a carry-out of one limb in addition) reserve that space - // up front so that a subsequent grow is not needed. + // Sets `*this` to the value of `src`. + // + // An rvalue heap `src` is stolen (its buffer adopted, `src` left as inline zero) whenever our + // allocator can free it, i.e. the allocator propagates, is always-equal, or compares equal to + // `src`'s, regardless of the capacity we already hold. Our own buffer is released first, with + // our allocator. Every other case copies, reusing the existing allocation when it already + // fits `src.limb_count() + extra_space` limbs. `extra_space` lets callers that know they are + // about to grow by a fixed amount (e.g., a carry-out of one limb) reserve that space up front; + // it must be 0 for an rvalue `src`. + // + // When a propagating allocator compares unequal to ours, our heap buffer cannot be kept: it is + // released through the old allocator before the new one is adopted. template requires std::same_as, basic_big_int> constexpr void assign_value(Src&& src, const std::size_t extra_space = 0) { @@ -729,10 +765,6 @@ class BEMAN_BIG_INT_TRIVIAL_ABI basic_big_int { return; } - const std::size_t src_count = src.limb_count(); - const std::size_t needed = src_count + extra_space; - const std::size_t eff_cap = is_representation_inplace() ? inplace_capacity : m_capacity; - // Allocator propagation follows `std::allocator_traits`. Stateful allocators // such as `std::pmr::polymorphic_allocator` have a deleted copy/move // assignment operator, so any `m_alloc = ...` must be guarded by @@ -745,7 +777,39 @@ class BEMAN_BIG_INT_TRIVIAL_ABI basic_big_int { (std::is_lvalue_reference_v ? alloc_traits::propagate_on_container_copy_assignment::value : alloc_traits::propagate_on_container_move_assignment::value); - if (needed <= eff_cap) { + if constexpr (!std::is_lvalue_reference_v) { + BEMAN_BIG_INT_DEBUG_ASSERT(extra_space == 0); + if (!src.is_representation_inplace() && + (propagate_alloc || alloc_traits::is_always_equal::value || m_alloc == src.m_alloc)) { + free_storage(); + if constexpr (propagate_alloc) { + m_alloc = std::forward(src).m_alloc; + } + m_capacity = src.m_capacity; + m_storage.data = src.m_storage.data; + m_size_and_sign = src.m_size_and_sign; + src.m_capacity = 0; + src.m_size_and_sign = 1; + src.m_storage = {}; + return; + } + // Inline `src`, or a heap `src` that our allocator cannot free: copy below. + } + + const std::size_t src_count = src.limb_count(); + const std::size_t needed = src_count + extra_space; + const std::size_t eff_cap = is_representation_inplace() ? inplace_capacity : m_capacity; + + // Keeping our heap buffer is only valid if our allocator stays the one that frees it. + const bool keeps_buffer = [&] { + if constexpr (propagate_alloc && !alloc_traits::is_always_equal::value) { + return is_representation_inplace() || m_alloc == src.m_alloc; + } else { + return true; + } + }(); + + if (keeps_buffer && needed <= eff_cap) { // Fast path: current buffer is already big enough const auto old_count = limb_count(); m_size_and_sign = src.m_size_and_sign; @@ -755,22 +819,16 @@ class BEMAN_BIG_INT_TRIVIAL_ABI basic_big_int { limb_type* const dst_limbs = limb_ptr(); const limb_type* const src_limbs = src.limb_ptr(); std::copy_n(src_limbs, src_count, dst_limbs); - // Preserve the "limbs[limb_count..inplace_capacity) == 0" invariant that - // `inplace_to_bit_uint` relies on. Only relevant for inline storage. - if (is_representation_inplace()) { - for (std::size_t i = src_count; i < old_count; ++i) { - dst_limbs[i] = limb_type{0}; - } - } + clear_inline_tail(old_count); return; } - // Slow path: current buffer is too small. Each branch releases our buffer - // with our current allocator, before any propagation, and publishes the - // control words only once the storage they describe is in place. + // Slow path. Each branch releases our buffer with our current allocator, before any + // propagation, and publishes the control words only once the storage they describe is in place. if (src.is_representation_inplace() && needed <= inplace_capacity) { // Both src and the requested headroom fit inline. No buffer to - // adopt or allocate; just propagate (if applicable) and copy limbs. + // allocate; release ours (if any), propagate, and copy limbs. + free_storage(); if constexpr (propagate_alloc) { m_alloc = std::forward(src).m_alloc; } @@ -782,39 +840,9 @@ class BEMAN_BIG_INT_TRIVIAL_ABI basic_big_int { return; } - if constexpr (!std::is_lvalue_reference_v) { - // For a heap `src`, adopt its pointer when the allocators are - // compatible -- i.e., we are propagating from src, the allocator - // type is always-equal (e.g. std::allocator), or the two - // allocators compare equal at runtime. - if (!src.is_representation_inplace()) { - if constexpr (propagate_alloc || alloc_traits::is_always_equal::value) { - free_storage(); - if constexpr (propagate_alloc) { - m_alloc = std::forward(src).m_alloc; - } - m_capacity = src.m_capacity; - m_storage.data = src.m_storage.data; - m_size_and_sign = src.m_size_and_sign; - src.m_capacity = 0; - src.m_size_and_sign = 1; - src.m_storage.limbs[0] = limb_type{0}; - if (m_capacity < needed) { - grow(needed); - } - return; - } - // Allocators differ and don't propagate: fall through to a - // fresh allocation owned by our allocator. - } - // `src` is inline (or unstealable heap), fall through to the - // allocate-and-copy path below. - } - - // Fall back to a fresh allocation of `needed` limbs. Secure it before - // releasing ours, so a throwing allocation leaves `*this` unchanged. A - // propagating allocator has to serve the new block while ours still has to - // release the old, so allocate through a copy of `src`'s. + // Fresh allocation of `needed` limbs. Secure it before releasing ours, so a throwing allocation + // leaves `*this` unchanged. A propagating allocator has to serve the new block while ours still + // has to release the old, so allocate through a copy of `src`'s. const alloc_result allocation = [&] { if constexpr (propagate_alloc) { allocator_type src_alloc(src.m_alloc); @@ -1035,7 +1063,16 @@ constexpr void basic_big_int::negate() noexcept { template constexpr void basic_big_int::set_zero() noexcept { - limb_ptr()[0] = 0; + limb_type* const limbs = limb_ptr(); + limbs[0] = 0; + if constexpr (inplace_capacity != 1) { + // Keep "inline limbs past the limb count are zero" (see `clear_inline_tail`). + if (is_representation_inplace()) { + for (size_type i = 1; i < inplace_capacity; ++i) { + limbs[i] = 0; + } + } + } m_size_and_sign = 1; } @@ -1120,11 +1157,11 @@ constexpr basic_big_int& basic_big_int::operator=(const basic_ template constexpr basic_big_int& basic_big_int::operator=(basic_big_int&& x) noexcept { - // Invariant: `assign_value` with `extra_space == 0` and an rvalue `src` - // never allocates -- either `*this` already fits `x.limb_count()`, or we - // steal `x`'s heap buffer, or `x` is inline and fits inline in `*this`. - // So the `noexcept` contract holds even though `assign_value` itself is - // not marked `noexcept`. + // A heap `x` whose buffer our allocator can free (propagating, always-equal, or equal) is + // stolen whatever our capacity, leaving `x` as inline zero. Otherwise `x` is copied into the + // existing storage, which allocates only if it does not fit. That copy needs an unequal, + // non-propagating allocator (e.g. a different pmr resource); it is the one case in which this + // `noexcept` operator can throw, which terminates. assign_value(std::move(x)); return *this; } @@ -1318,101 +1355,167 @@ constexpr bool basic_big_int::unchecked_decrement_magnitude() { return borrow_in; } +// Zero stays zero. The result is sized exactly: the extra top limb exists only when the top limb's +// leading zeros cannot absorb the bit shift. template constexpr void basic_big_int::shift_left(const shift_type s) { BEMAN_BIG_INT_DEBUG_ASSERT(s <= shift_max); - if (s == 0) { + const size_type n = limb_count(); + if (s == 0 || limb_ptr()[n - 1] == 0) { return; } - const shift_type shifted_limbs = s / bits_per_limb; - const shift_type shifted_bits = s % bits_per_limb; + const size_type whole = s / bits_per_limb; + const unsigned bits = static_cast(s % bits_per_limb); + const bool extra = bits != 0 && static_cast(std::countl_zero(limb_ptr()[n - 1])) < bits; + const size_type new_n = n + whole + static_cast(extra); - // Only reserve an extra limb for the bit-shift when the top limb doesn't enough leading zeros. - // `countl_zero` tells us exactly how many bits of headroom the current top limb provides. - const bool needs_extra_limb = - shifted_bits != 0 && static_cast(std::countl_zero(limb_ptr()[limb_count() - 1])) < shifted_bits; - reserve_representation(limb_count() + shifted_limbs + static_cast(needs_extra_limb)); + grow(new_n); limb_type* const limbs = limb_ptr(); + if (bits != 0) { + const limb_type out = detail::lshift_copy(limbs + whole, limbs, n, bits); + if (extra) { + limbs[n + whole] = out; + } + } else { + std::copy_backward(limbs, limbs + n, limbs + n + whole); + } + std::fill_n(limbs, whole, limb_type{0}); + unchecked_set_limb_count(static_cast(new_n)); +} - if (shifted_limbs != 0) { - const auto current_count = limb_count(); - std::copy_backward(limbs, limbs + current_count, limbs + current_count + shifted_limbs); - std::fill_n(limbs, static_cast(shifted_limbs), limb_type{0}); - unchecked_set_limb_count(static_cast(current_count + shifted_limbs)); +// Floor division by `2^s`: a negative value whose discarded bits are not all zero gets its magnitude +// incremented after the (truncating) magnitude shift. +template +constexpr void basic_big_int::shift_right(const shift_type s) { + BEMAN_BIG_INT_DEBUG_ASSERT(s <= shift_max); + if (s == 0) { + return; } - if (shifted_bits != 0) { - const auto current_count = limb_count(); - const limb_type overflow = limbs[current_count - 1] >> (bits_per_limb - shifted_bits); - // Shift all limbs in place, top to bottom. - // Each iteration reads limbs[i-1] (original) and limbs[i] (original), writes limbs[i]. - // This avoids an out-of-bounds write - for (shift_type i = current_count - 1; i > shifted_limbs; --i) { - const detail::wide all_bits{.low_bits = limbs[i - 1], .high_bits = limbs[i]}; - limbs[i] = detail::funnel_shl(all_bits, static_cast(shifted_bits)); + const size_type n = limb_count(); + const size_type whole = s / bits_per_limb; + const unsigned bits = static_cast(s % bits_per_limb); + const bool neg = is_negative(); + + if (whole >= n) { + // Everything is discarded: zero, or -1 for a negative value (which is nonzero, so bits were lost). + set_zero(); + if (neg) { + limb_ptr()[0] = 1; + m_size_and_sign = 0x8000'0001U; } - limbs[shifted_limbs] <<= shifted_bits; + return; + } - if (overflow != 0) { - limbs[current_count] = overflow; - unchecked_set_limb_count(static_cast(current_count + 1)); + limb_type* const limbs = limb_ptr(); + bool inexact = false; + if (neg) { + for (size_type i = 0; i < whole && !inexact; ++i) { + inexact = limbs[i] != 0; } } + + const size_type new_n = n - whole; + if (bits != 0) { + inexact |= detail::rshift_copy(limbs, limbs + whole, new_n, bits) != 0; + } else if (whole != 0) { + std::copy(limbs + whole, limbs + n, limbs); + } + + size_type k = new_n; + while (k > 1 && limbs[k - 1] == 0) { + --k; + } + m_size_and_sign = (m_size_and_sign & 0x8000'0000U) | static_cast(k); + clear_inline_tail(n); + if (neg && inexact) { + // A magnitude that shifted down to zero is a negative zero here; incrementing makes it -1. + unchecked_increment_magnitude(); + BEMAN_BIG_INT_DEBUG_ASSERT(is_negative()); + } } template -constexpr void basic_big_int::shift_right(const shift_type s) { +constexpr void basic_big_int::assign_shifted_left(const std::span src, + const bool neg, + const shift_type s) { BEMAN_BIG_INT_DEBUG_ASSERT(s <= shift_max); - if (s == 0) { + const size_type n = src.size(); + if (src[n - 1] == 0) { + set_zero(); return; } - const shift_type shifted_limbs = s / bits_per_limb; - const shift_type shifted_bits = s % bits_per_limb; - limb_type* const limbs = limb_ptr(); + const size_type old_count = limb_count(); + const size_type whole = s / bits_per_limb; + const unsigned bits = static_cast(s % bits_per_limb); + const bool extra = bits != 0 && static_cast(std::countl_zero(src[n - 1])) < bits; + const size_type new_n = n + whole + static_cast(extra); - // Shifting to the right has the effect of dividing by `pow(2, N)` rounded towards negative infinity. - // When we "discard" limbs, this has truncating rounding; - // for positive numbers that is already what we need, but requires adjustment for negative numbers. - // In that case, we need to figure out whether the result is inexact - // by detecting whether any nonzero bits are shifted out. - const bool needs_decrement = is_negative() && [&] { - for (shift_type i = 0; i < shifted_limbs; ++i) { - if (i >= limb_count()) { - return false; - } - if (limbs[i] != 0) { - return true; - } + limb_type* const limbs = storage_for_overwrite(new_n); + if (bits != 0) { + const limb_type out = detail::lshift_copy(limbs + whole, src.data(), n, bits); + if (extra) { + limbs[n + whole] = out; } - const auto shift_mask = (limb_type{1} << shifted_bits) - 1; - return shifted_limbs < limb_count() && (limbs[shifted_limbs] & shift_mask) != 0; - }(); + } else { + std::copy_n(src.data(), n, limbs + whole); + } + std::fill_n(limbs, whole, limb_type{0}); + m_size_and_sign = (static_cast(neg) << 31) | static_cast(new_n); + clear_inline_tail(old_count); +} - if (shifted_limbs != 0) { - const auto current_count = limb_count(); - if (shifted_limbs >= current_count) { - limbs[0] = 0; - unchecked_set_limb_count(1); - } else { - std::shift_left(limbs, limbs + current_count, static_cast(shifted_limbs)); - unchecked_set_limb_count(static_cast(current_count - shifted_limbs)); +template +constexpr void basic_big_int::assign_shifted_right(const std::span src, + const bool neg, + const shift_type s) { + BEMAN_BIG_INT_DEBUG_ASSERT(s <= shift_max); + const size_type n = src.size(); + const size_type whole = s / bits_per_limb; + const unsigned bits = static_cast(s % bits_per_limb); + + if (whole >= n) { + set_zero(); + if (neg) { + limb_ptr()[0] = 1; + m_size_and_sign = 0x8000'0001U; } + return; } - if (shifted_bits != 0) { - for (size_type i = 0; i + 1 < limb_count(); ++i) { - const detail::wide all_bits{.low_bits = limbs[i], .high_bits = limbs[i + 1]}; - limbs[i] = detail::funnel_shr(all_bits, static_cast(shifted_bits)); + + bool inexact = false; + if (neg) { + for (size_type i = 0; i < whole && !inexact; ++i) { + inexact = src[i] != 0; } - BEMAN_BIG_INT_DEBUG_ASSERT(limb_count() != 0); - limbs[limb_count() - 1] >>= shifted_bits; } - unchecked_trim_magnitude(); - if (needs_decrement) { - // See above for rounding considerations. - // Also note that we may be holding a negative zero right now, - // and increasing the magnitude converts that into `-1`. + + // Size the result exactly (the source's top limb may shift away entirely) so a heap source whose + // shifted value fits inline does not allocate. + const size_type old_count = limb_count(); + const limb_type* from = src.data() + whole; + size_type k = n - whole; + const bool top_dies = k > 1 && bits != 0 && (from[k - 1] >> bits) == 0; + k -= static_cast(top_dies); + limb_type* const limbs = storage_for_overwrite(k); + if (bits != 0) { + inexact |= detail::rshift_copy(limbs, from, k, bits) != 0; + if (top_dies) { + // The vanished top limb still contributes its low bits to limb k - 1. + limbs[k - 1] |= static_cast(from[k] << (bits_per_limb - bits)); + } + } else { + std::copy_n(from, k, limbs); + } + + while (k > 1 && limbs[k - 1] == 0) { + --k; + } + m_size_and_sign = (static_cast(neg) << 31) | static_cast(k); + clear_inline_tail(old_count); + if (neg && inexact) { unchecked_increment_magnitude(); BEMAN_BIG_INT_DEBUG_ASSERT(is_negative()); } @@ -1892,15 +1995,9 @@ constexpr detail::common_big_int_type operator+(L&& x, R&& y) { r.add_in_place(x.representation(), x.is_negative()); return r; } else if constexpr (form == detail::binary_op_form::copy_copy) { - // Use add_into which combines copy and addition allocations. - // Pre-order operands so the larger goes first — this lets the hot loop - // drop the `i < a.size()` bounds check on every iteration. + // `add_into` writes the sum straight into the fresh result: one allocation, one pass. Result r{detail::result_allocator(x, y)}; - if (x.limb_count() >= y.limb_count()) { - r.add_into(x.representation(), x.is_negative(), y.representation(), y.is_negative()); - } else { - r.add_into(y.representation(), y.is_negative(), x.representation(), x.is_negative()); - } + r.add_into(x.representation(), x.is_negative(), y.representation(), y.is_negative()); return r; } else if constexpr (form == detail::binary_op_form::move_int) { const auto y_limbs = detail::to_limbs(detail::uabs(y)); @@ -1913,16 +2010,14 @@ constexpr detail::common_big_int_type operator+(L&& x, R&& y) { r.add_in_place(detail::to_fixed_span(x_limbs), detail::integer_signbit(x)); return r; } else if constexpr (form == detail::binary_op_form::copy_int) { - Result r{detail::result_allocator(x, y)}; - r.template assign_value(x, !x.is_representation_inplace()); + Result r{detail::result_allocator(x, y)}; const auto y_limbs = detail::to_limbs(detail::uabs(y)); - r.add_in_place(detail::to_fixed_span(y_limbs), detail::integer_signbit(y)); + r.add_into(x.representation(), x.is_negative(), detail::to_fixed_span(y_limbs), detail::integer_signbit(y)); return r; } else if constexpr (form == detail::binary_op_form::int_copy) { - Result r{detail::result_allocator(x, y)}; - r.template assign_value(y, !y.is_representation_inplace()); + Result r{detail::result_allocator(x, y)}; const auto x_limbs = detail::to_limbs(detail::uabs(x)); - r.add_in_place(detail::to_fixed_span(x_limbs), detail::integer_signbit(x)); + r.add_into(y.representation(), y.is_negative(), detail::to_fixed_span(x_limbs), detail::integer_signbit(x)); return r; } } @@ -1961,14 +2056,8 @@ constexpr detail::common_big_int_type operator-(L&& x, R&& y) { } else if constexpr (form == detail::binary_op_form::copy_copy) { // Both lvalue `basic_big_int`s: fold `x + (-y)` into a fresh buffer in a // single pass via `add_into`, fusing the allocation with the subtract. - // Pre-order operands so the larger goes first — `add_into` relies on - // this to keep the hot loop free of a per-iteration bounds check. Result r{detail::result_allocator(x, y)}; - if (x.limb_count() >= y.limb_count()) { - r.add_into(x.representation(), x.is_negative(), y.representation(), !y.is_negative()); - } else { - r.add_into(y.representation(), !y.is_negative(), x.representation(), x.is_negative()); - } + r.add_into(x.representation(), x.is_negative(), y.representation(), !y.is_negative()); return r; } else if constexpr (form == detail::binary_op_form::move_int) { const auto y_limbs = detail::to_limbs(detail::uabs(y)); @@ -1982,17 +2071,14 @@ constexpr detail::common_big_int_type operator-(L&& x, R&& y) { r.add_in_place(detail::to_fixed_span(x_limbs), detail::integer_signbit(x)); return r; } else if constexpr (form == detail::binary_op_form::copy_int) { - Result r{detail::result_allocator(x, y)}; - r.template assign_value(x, !x.is_representation_inplace()); + Result r{detail::result_allocator(x, y)}; const auto y_limbs = detail::to_limbs(detail::uabs(y)); - r.add_in_place(detail::to_fixed_span(y_limbs), !detail::integer_signbit(y)); + r.add_into(x.representation(), x.is_negative(), detail::to_fixed_span(y_limbs), !detail::integer_signbit(y)); return r; } else if constexpr (form == detail::binary_op_form::int_copy) { - Result r{detail::result_allocator(x, y)}; - r.template assign_value(y, !y.is_representation_inplace()); - r.negate(); + Result r{detail::result_allocator(x, y)}; const auto x_limbs = detail::to_limbs(detail::uabs(x)); - r.add_in_place(detail::to_fixed_span(x_limbs), detail::integer_signbit(x)); + r.add_into(y.representation(), !y.is_negative(), detail::to_fixed_span(x_limbs), detail::integer_signbit(x)); return r; } } @@ -2104,17 +2190,9 @@ constexpr std::remove_cvref_t operator<<(T&& x, const S s) { r.shift_left(shift); return r; } else if constexpr (form == detail::binary_op_form::copy_int) { - // lvalue: use assign_value with headroom so shift_left doesn't reallocate. - const shift_type shifted_limbs = shift / Result::bits_per_limb; - const shift_type shifted_bits = shift % Result::bits_per_limb; - const bool needs_extra = - shifted_bits != 0 && - static_cast(std::countl_zero(x.limb_ptr()[x.limb_count() - 1])) < shifted_bits; - const std::size_t headroom = shifted_limbs + static_cast(needs_extra); - + // lvalue: build the shifted value straight into a fresh result, one pass. Result r{detail::result_allocator(x, s)}; - r.template assign_value(x, headroom); - r.shift_left(shift); + r.assign_shifted_left(x.representation(), x.is_negative(), shift); return r; } } @@ -2140,95 +2218,17 @@ constexpr std::remove_cvref_t operator>>(T&& x, const S s) { r.shift_right(shift); return r; } else if constexpr (form == detail::binary_op_form::copy_int) { - // lvalue: copy then shift. Two fast paths avoid copying bits that are - // about to be discarded anyway - // - // First is the case where the value is discarded entirely - // All we need to do is copy the sign - // - // Second is the case where some of the value is discarded - // For positive values this is a simple copy what we need and make a final limb internal shift - // For negative values we need to account for the rounding at the end - // - // If neither of these situations applies, make a full copy and shift. - - using limb_type = uint_multiprecision_t; - - const shift_type shifted_limbs = shift / Result::bits_per_limb; - const shift_type shifted_bits = shift % Result::bits_per_limb; - const shift_type x_bits = static_cast(x.size()); - + // lvalue: shift straight from the source into a fresh result, one pass. + // Shifting out every bit leaves just the sign (-1 or 0), without touching the limbs. const auto alloc = detail::result_allocator(x, s); - - // Case 1: Everything is discarded except the sign - if (shift >= x_bits) { + if (shift >= static_cast(x.size())) { if (x.is_negative()) { return Result{-1, alloc}; } return Result{alloc}; } - - const limb_type* const src_limbs = x.limb_ptr(); - const shift_type src_count = static_cast(x.limb_count()); - - // Number of limbs the shifted result actually occupies. Smaller than - // `src_count` exactly when at least one source limb (or the top - // limb's surviving bits) shrinks away. - const auto new_count = detail::div_to_pos_inf(x_bits - shift, static_cast(Result::bits_per_limb)); - - // Case 2: Result is strictly smaller than the source - if (new_count < src_count) { - const shift_type src_offset = shifted_limbs; - - Result r{alloc}; - r.reserve_representation(new_count); - limb_type* const dst = r.limb_ptr(); - - if (shifted_bits == 0) { - std::copy_n(src_limbs + src_offset, new_count, dst); - } else { - // Funnel-shift directly from source limbs into `dst`. - // Handles the case where we can theoretically go from heap -> inline storage with our result - for (shift_type i = 0; i < new_count; ++i) { - const shift_type src_idx = src_offset + i; - const limb_type low = src_limbs[src_idx]; - const limb_type high = (src_idx + 1 < src_count) ? src_limbs[src_idx + 1] : limb_type{0}; - const detail::wide all_bits{.low_bits = low, .high_bits = high}; - dst[i] = detail::funnel_shr(all_bits, static_cast(shifted_bits)); - } - } - r.unchecked_set_limb_count(static_cast(new_count)); - - if (x.is_negative()) { - // Mirror `shift_right`'s rounding-toward-negative-infinity - // We have to account even for the bits that will never be copied - bool discarded_nonzero = false; - for (shift_type i = 0; i < src_offset; ++i) { - if (src_limbs[i] != 0) { - discarded_nonzero = true; - break; - } - } - - if (!discarded_nonzero && shifted_bits != 0) { - const auto mask = static_cast((limb_type{1} << shifted_bits) - 1); - if ((src_limbs[src_offset] & mask) != 0) { - discarded_nonzero = true; - } - } - - r.unchecked_set_sign(true); - if (discarded_nonzero) { - r.unchecked_increment_magnitude(); - } - } - return r; - } - - // Case 3: Make a full copy and shift Result r{alloc}; - r.template assign_value(x); - r.shift_right(shift); + r.assign_shifted_right(x.representation(), x.is_negative(), shift); return r; } } @@ -2360,295 +2360,187 @@ basic_big_int::compare_limbs(const std::span template -constexpr void basic_big_int::add_in_place(const std::span other, - const bool other_neg) { - const bool this_neg = is_negative(); +constexpr void +basic_big_int::add_in_place(const std::span other_in, + const bool other_neg) { + using limb_view = std::span; + const limb_view other = other_in.first(detail::trimmed_size_span(other_in)); + const bool this_neg = is_negative(); + const std::size_t n = limb_count(); + const std::size_t m = other.size(); + const std::uint32_t sign_bit = m_size_and_sign & 0x8000'0000U; + + if (n == 1 && m == 1) { + // Single-limb operands: one add or subtract, no comparison of spans. + const limb_type x = limb_ptr()[0]; + const limb_type y = other[0]; + if (this_neg == other_neg) { + const auto [sum, carry] = detail::carrying_add(x, y, false); + limb_ptr()[0] = sum; + if (carry) { + grow(2); + limb_ptr()[1] = limb_type{1}; + m_size_and_sign = sign_bit | 2U; + } + } else if (x == y) { + set_zero(); + } else { + limb_ptr()[0] = x > y ? x - y : y - x; + m_size_and_sign = (static_cast(x > y ? this_neg : other_neg) << 31) | 1U; + } + return; + } if (this_neg == other_neg) { - // Same sign: add magnitudes limb-by-limb. Target sign stays `this_neg`. - const std::uint32_t old_count = limb_count(); - const std::size_t big = std::max(old_count, other.size()); - - // `grow(big)` only allocates when `big` exceeds our current capacity; - // otherwise it's a no-op and we stay in our existing buffer. - grow(big); - limb_type* limbs = limb_ptr(); - - bool carry = false; - for (std::size_t i = 0; i < big; ++i) { - const limb_type li = i < old_count ? limbs[i] : limb_type{0}; - const limb_type ri = i < other.size() ? other[i] : limb_type{0}; - const auto [r_value, r_carry] = detail::carrying_add(li, ri, carry); - limbs[i] = r_value; - carry = r_carry; + // Same sign: the magnitude grows, the sign stays. + if (m <= n) { + limb_type* limbs = limb_ptr(); + const bool carry = detail::add_n_tail({limbs, n}, {limbs, n}, other); + if (carry) { + grow(n + 1); + limbs = limb_ptr(); + limbs[n] = limb_type{1}; + m_size_and_sign = sign_bit | static_cast(n + 1); + } + return; } - unchecked_set_limb_count(static_cast(big)); - - // Only allocate for the extra top limb if the ripple carry has actually escaped. + // `other` is longer, so it cannot alias us and growing first is safe. + grow(m); + limb_type* limbs = limb_ptr(); + const bool carry = detail::add_n_tail({limbs, m}, other, {limbs, n}); + m_size_and_sign = sign_bit | static_cast(m); if (carry) { - grow(big + 1); - limb_ptr()[big] = limb_type{1}; - unchecked_set_limb_count(static_cast(big + 1)); + // `grow` only keeps the limbs the count covers, so the count goes first. + grow(m + 1); + limb_ptr()[m] = limb_type{1}; + m_size_and_sign = sign_bit | static_cast(m + 1); } - - unchecked_set_sign(this_neg && !unchecked_is_magnitude_zero()); return; } - // Differing signs: subtract smaller magnitude from larger; take the sign of the - // larger-magnitude operand. - const auto magnitude_order = detail::compare_limb_magnitudes(representation(), other); - - if (std::is_gteq(magnitude_order)) { - // `|*this| >= |other|`: compute `*this - other` in place. Target sign is `this_neg`. - const std::uint32_t n = limb_count(); - limb_type* const limbs = limb_ptr(); - - bool borrow = false; - for (std::size_t i = 0; i < n; ++i) { - const limb_type li = limbs[i]; - const limb_type si = i < other.size() ? other[i] : limb_type{0}; - const auto [r_value, r_borrow] = detail::borrowing_sub(li, si, borrow); - limbs[i] = r_value; - borrow = r_borrow; - } - // Having picked `*this` as the larger operand, the final borrow must be zero. - BEMAN_BIG_INT_DEBUG_ASSERT(!borrow); - - unchecked_trim_magnitude(); - unchecked_set_sign(this_neg && !unchecked_is_magnitude_zero()); + // Differing signs: subtract the smaller magnitude from the larger; the larger one's sign wins. + const auto order = detail::compare_limb_magnitudes(limb_view{limb_ptr(), n}, other); + if (std::is_eq(order)) { + set_zero(); return; } - // `|other| > |*this|`: compute `other - *this` into our buffer. Target sign is `other_neg`. - const std::uint32_t old_count = limb_count(); - const std::size_t n = other.size(); - - // Subtraction can never produce more limbs than the larger operand, so this grow is tight. - grow(n); - limb_type* const limbs = limb_ptr(); - - bool borrow = false; - for (std::size_t i = 0; i < n; ++i) { - // Read our old limb at index `i` *before* overwriting it, so this loop is - // aliasing-safe if `other` happens to point into our own limb buffer. - const limb_type si = i < old_count ? limbs[i] : limb_type{0}; - const limb_type oi = other[i]; - const auto [r_value, r_borrow] = detail::borrowing_sub(oi, si, borrow); - limbs[i] = r_value; - borrow = r_borrow; - } - // Having picked `other` as the larger operand, the final borrow must be zero. + std::size_t k; + bool result_neg; + limb_type* limbs; + [[maybe_unused]] bool borrow; + if (std::is_gt(order)) { + // `|*this| > |other|`: subtract in place, nothing past `other` is touched once the borrow dies. + limbs = limb_ptr(); + borrow = detail::sub_n_tail({limbs, n}, {limbs, n}, other); + k = n; + result_neg = this_neg; + } else { + // `|other| > |*this|` (so `m >= n`): `other - *this` over our own limbs, reading each of ours first. + grow(m); + limbs = limb_ptr(); + borrow = detail::sub_n_tail({limbs, m}, other, {limbs, n}); + k = m; + result_neg = other_neg; + } BEMAN_BIG_INT_DEBUG_ASSERT(!borrow); - unchecked_set_limb_count(static_cast(n)); - unchecked_trim_magnitude(); - - // `|other| > |*this|` strictly, so the magnitude is guaranteed nonzero. - unchecked_set_sign(other_neg && !unchecked_is_magnitude_zero()); + while (k > 1 && limbs[k - 1] == 0) { + --k; + } + // Trimmed limbs are zero, so the inline tail needs no clearing. + m_size_and_sign = (static_cast(result_neg) << 31) | static_cast(k); } // Computes `(a, a_neg) + (b, b_neg)` directly into `*this`. -// Fuses copy and potential second allocation -// Precondition: `a.size() >= b.size()` +// Precondition: `*this` shares no storage with `a` or `b`. +// Either operand order is accepted, and operands may carry untrimmed high zero limbs. template template -constexpr void basic_big_int::add_into(const std::span a, - const bool a_neg, - const std::span b_span, - const bool b_neg) { - BEMAN_BIG_INT_DEBUG_ASSERT(a.size() >= b_span.size()); - - if (a_neg == b_neg) { - // Same sign: add magnitudes. - // Sign of the result is `a_neg`. - const std::size_t big = a.size(); - const std::size_t eff_cap = is_representation_inplace() ? inplace_capacity : m_capacity; - - // `common` is the range where both operands contribute, - // beyond it the remaining `a[i]` limbs just propagate the carry. - const std::size_t common = b_span.size(); - - // Unrolled addition loop - const auto run_add = [&](auto store) { - bool carry = false; - std::size_t i = 0; - for (; i + 4 <= common; i += 4) { - const auto [v0, c0] = detail::carrying_add(a[i + 0], b_span[i + 0], carry); - const auto [v1, c1] = detail::carrying_add(a[i + 1], b_span[i + 1], c0); - const auto [v2, c2] = detail::carrying_add(a[i + 2], b_span[i + 2], c1); - const auto [v3, c3] = detail::carrying_add(a[i + 3], b_span[i + 3], c2); - store(i + 0, v0); - store(i + 1, v1); - store(i + 2, v2); - store(i + 3, v3); - carry = c3; - } - for (; i < common; ++i) { - const auto [v, c] = detail::carrying_add(a[i], b_span[i], carry); - store(i, v); - carry = c; - } - // Carry-propagation tail over `a[common..big)`. Once the ripple - // stops carrying, the rest is a straight copy of `a`'s limbs. - while (i < big && carry) { - const auto [v, c] = detail::carrying_add(a[i], limb_type{0}, true); - store(i, v); - carry = c; - ++i; - } - for (; i < big; ++i) { - store(i, a[i]); - } - return carry; - }; - - if (big > inplace_capacity && big + 1 > eff_cap) { - // Heap allocation required for the result body. - // Write directly into the raw buffer via `construct_at`, - // skipping the `copy_n_to_allocation` zero-fill of the portion we're about to overwrite. - // Reserve `big + 1` so the carry limb folds into the same allocation. - const alloc_result allocation = alloc_limbs(big + 1); - limb_type* const limbs = allocation.ptr; - - const bool carry = run_add([limbs](std::size_t idx, limb_type v) { std::construct_at(limbs + idx, v); }); - - std::construct_at(limbs + big, carry ? limb_type{1} : limb_type{0}); - for (std::size_t i = big + 1; i < allocation.count; ++i) { - std::construct_at(limbs + i); - } - - free_storage(); - m_capacity = static_cast(allocation.count); - m_storage.data = allocation.ptr; - unchecked_set_limb_count(static_cast(carry ? big + 1 : big)); - } else { - // Either we fit inline, or we already own a heap buffer big enough to hold the result. - // Reuse it and let `grow` (almost always a no-op) handle the rare carry-out grow call. - limb_type* const limbs = limb_ptr(); - - const bool carry = run_add([limbs](std::size_t idx, limb_type v) { limbs[idx] = v; }); - unchecked_set_limb_count(static_cast(big)); - +constexpr void basic_big_int::add_into(const std::span a_in, + bool a_neg, + const std::span b_in, + bool b_neg) { + using limb_view = std::span; + limb_view a = a_in.first(detail::trimmed_size_span(a_in)); + limb_view bs = b_in.first(detail::trimmed_size_span(b_in)); + if (a.size() < bs.size()) { + std::swap(a, bs); + std::swap(a_neg, b_neg); + } + if (a.size() == 1) { // both single limbs (a is the longer operand) + const limb_type x = a[0]; + const limb_type y = bs[0]; + const size_type old_count = limb_count(); + if (a_neg == b_neg) { + const auto [sum, carry] = detail::carrying_add(x, y, false); + limb_type* const limbs = storage_for_overwrite(carry ? 2 : 1); + limbs[0] = sum; if (carry) { - grow(big + 1); - limb_ptr()[big] = limb_type{1}; - unchecked_set_limb_count(static_cast(big + 1)); + limbs[1] = limb_type{1}; } + m_size_and_sign = (static_cast(a_neg) << 31) | (carry ? 2U : 1U); + } else if (x == y) { + set_zero(); + return; + } else { + storage_for_overwrite(1)[0] = x > y ? x - y : y - x; + m_size_and_sign = (static_cast(x > y ? a_neg : b_neg) << 31) | 1U; } - - unchecked_set_sign(a_neg && !unchecked_is_magnitude_zero()); + clear_inline_tail(old_count); return; } - // Differing signs: subtract the smaller magnitude from the larger. - const auto magnitude_order = detail::compare_limb_magnitudes(a, b_span); - - // Both subtraction branches mirror the same-sign allocation strategy - const std::size_t eff_cap = is_representation_inplace() ? inplace_capacity : m_capacity; + const std::size_t old_count = limb_count(); + const std::size_t big = a.size(); + const std::size_t cap = is_representation_inplace() ? inplace_capacity : m_capacity; - const auto finalize_trim_and_sign = - [this](const limb_type* const limbs, const std::size_t n, const bool target_neg) { - unchecked_set_limb_count(static_cast(n)); - while (limb_count() > 1 && limbs[limb_count() - 1] == 0) { - unchecked_set_limb_count(limb_count() - 1); - } - unchecked_set_sign(target_neg && !unchecked_is_magnitude_zero()); - }; - - // Unrolled subtract - const auto run_sub = [&](auto larger, auto smaller, auto store, const std::size_t total) { - const std::size_t common = smaller.size(); - bool borrow = false; - std::size_t i = 0; - for (; i + 4 <= common; i += 4) { - const auto [v0, b0] = detail::borrowing_sub(larger[i + 0], smaller[i + 0], borrow); - const auto [v1, b1] = detail::borrowing_sub(larger[i + 1], smaller[i + 1], b0); - const auto [v2, b2] = detail::borrowing_sub(larger[i + 2], smaller[i + 2], b1); - const auto [v3, b3] = detail::borrowing_sub(larger[i + 3], smaller[i + 3], b2); - store(i + 0, v0); - store(i + 1, v1); - store(i + 2, v2); - store(i + 3, v3); - borrow = b3; - } - for (; i < common; ++i) { - const auto [v, br] = detail::borrowing_sub(larger[i], smaller[i], borrow); - store(i, v); - borrow = br; - } - while (i < total && borrow) { - const auto [v, br] = detail::borrowing_sub(larger[i], limb_type{0}, true); - store(i, v); - borrow = br; - ++i; - } - for (; i < total; ++i) { - store(i, larger[i]); - } - return borrow; - }; - - if (std::is_gteq(magnitude_order)) { - // `|a| >= |b|`: compute `a - b` into our buffer. - // Target sign is `a_neg`. - const std::size_t n = a.size(); - - if (n > inplace_capacity && n > eff_cap) { - const alloc_result allocation = alloc_limbs(n); - limb_type* const limbs = allocation.ptr; - - [[maybe_unused]] const bool borrow = - run_sub(a, b_span, [limbs](std::size_t idx, limb_type v) { std::construct_at(limbs + idx, v); }, n); - BEMAN_BIG_INT_DEBUG_ASSERT(!borrow); - for (std::size_t i = n; i < allocation.count; ++i) { - std::construct_at(limbs + i); - } - - free_storage(); - m_capacity = static_cast(allocation.count); - m_storage.data = allocation.ptr; - finalize_trim_and_sign(limbs, n, a_neg); - } else { - limb_type* const limbs = limb_ptr(); - [[maybe_unused]] const bool borrow = - run_sub(a, b_span, [limbs](std::size_t idx, limb_type v) { limbs[idx] = v; }, n); - BEMAN_BIG_INT_DEBUG_ASSERT(!borrow); - finalize_trim_and_sign(limbs, n, a_neg); + if (a_neg == b_neg) { + // Same sign: the sign of the result is `a_neg` (the magnitude is nonzero when that is negative). + // Ask for the carry limb only when the body alone would force an allocation anyway. + limb_type* const limbs = storage_for_overwrite(big > cap ? big + 1 : big); + const bool carry = detail::add_n_tail({limbs, big}, a, bs); + const auto sign = static_cast(a_neg) << 31; + m_size_and_sign = sign | static_cast(big); + if (carry) { + // `grow` only keeps the limbs the count covers, so the count goes first. + grow(big + 1); + limb_ptr()[big] = limb_type{1}; + m_size_and_sign = sign | static_cast(big + 1); } + clear_inline_tail(old_count); return; } - // `|b| > |a|`: compute `b - a` into our buffer. - // Target sign is `b_neg`. - const std::size_t n = b_span.size(); - - if (n > inplace_capacity && n > eff_cap) { - const alloc_result allocation = alloc_limbs(n); - limb_type* const limbs = allocation.ptr; - - [[maybe_unused]] const bool borrow = - run_sub(b_span, a, [limbs](std::size_t idx, limb_type v) { std::construct_at(limbs + idx, v); }, n); - BEMAN_BIG_INT_DEBUG_ASSERT(!borrow); - for (std::size_t i = n; i < allocation.count; ++i) { - std::construct_at(limbs + i); - } + // Differing signs: subtract the smaller magnitude from the larger. + const auto order = detail::compare_limb_magnitudes(a, bs); + if (std::is_eq(order)) { + set_zero(); + return; + } + const bool a_larger = std::is_gt(order); + const limb_view larger = a_larger ? a : bs; + const limb_view smaller = a_larger ? bs : a; + const bool result_neg = a_larger ? a_neg : b_neg; - free_storage(); - m_capacity = static_cast(allocation.count); - m_storage.data = allocation.ptr; - finalize_trim_and_sign(limbs, n, b_neg); - } else { - limb_type* const limbs = limb_ptr(); - [[maybe_unused]] const bool borrow = - run_sub(b_span, a, [limbs](std::size_t idx, limb_type v) { limbs[idx] = v; }, n); - BEMAN_BIG_INT_DEBUG_ASSERT(!borrow); - finalize_trim_and_sign(limbs, n, b_neg); + std::size_t k = larger.size(); + limb_type* const limbs = storage_for_overwrite(k); + [[maybe_unused]] const bool borrow = detail::sub_n_tail({limbs, k}, larger, smaller); + BEMAN_BIG_INT_DEBUG_ASSERT(!borrow); + while (k > 1 && limbs[k - 1] == 0) { + --k; } + m_size_and_sign = (static_cast(result_neg) << 31) | static_cast(k); + clear_inline_tail(old_count); } -// Computes `a * b` and stores the result into `*this`. +// Computes `a * b` and stores the result into `*this`. The operands must not alias `*this`. +// The product is written straight into `*this`'s storage (the multiply dispatchers need no pre-zeroed +// result); a product one limb past the inline capacity is formed on the stack at run time, so that a +// value that trims to fit inline never allocates. template template constexpr void basic_big_int::multiply_into(const std::span a, @@ -2665,38 +2557,31 @@ constexpr void basic_big_int::multiply_into(const std::span stack_span{stack_buf, result_size}; - const std::size_t sig = detail::multiply_dispatch(stack_span, a_trimmed, b_trimmed, m_alloc); - if (sig > inplace_capacity) { - grow(sig); - } - auto* const dst = limb_ptr(); - for (std::size_t i = 0; i < sig; ++i) { - dst[i] = stack_buf[i]; + const size_type old_count = limb_count(); + + // Both operands are nonzero, so the product is nonzero and its sign is the xor of the operand signs. + if constexpr (inplace_capacity + 1 <= 64) { + if BEMAN_BIG_INT_IS_NOT_CONSTEVAL { + if (result_size == inplace_capacity + 1) { + limb_type stack_buf[inplace_capacity + 1]; + const std::size_t sig = detail::multiply_dispatch( + std::span{stack_buf, result_size}, a_trimmed, b_trimmed, m_alloc); + limb_type* const dst = storage_for_overwrite(sig); + std::copy_n(stack_buf, sig, dst); + unchecked_set_limb_count(static_cast(sig)); + clear_inline_tail(old_count); + unchecked_set_sign(a_neg != b_neg); + return; } - unchecked_set_limb_count(static_cast(sig)); - unchecked_set_sign(a_neg != b_neg && !unchecked_is_magnitude_zero()); - return; } } - grow(result_size); - std::fill_n(limb_ptr(), result_size, limb_type{0}); - - std::span result_span{limb_ptr(), result_size}; - const std::size_t sig = detail::multiply_dispatch(result_span, a_trimmed, b_trimmed, m_alloc); + limb_type* const dst = storage_for_overwrite(result_size); + const std::size_t sig = + detail::multiply_dispatch(std::span{dst, result_size}, a_trimmed, b_trimmed, m_alloc); unchecked_set_limb_count(static_cast(sig)); - unchecked_set_sign(a_neg != b_neg && !unchecked_is_magnitude_zero()); + clear_inline_tail(old_count); + unchecked_set_sign(a_neg != b_neg); } template @@ -2845,6 +2730,68 @@ constexpr auto basic_big_int::operator-=(T&& rhs) -> basic_big_int& return *this; } +// Multiplies in place, keeping the allocator and, below the large-product threshold, the capacity. +template +template +constexpr void basic_big_int::multiply_in_place(const std::span b_span, + const bool b_neg) { + const std::span a_all{limb_ptr(), limb_count()}; + const auto a_trimmed = a_all.first(detail::trimmed_size_span(a_all)); + const auto b_trimmed = b_span.first(detail::trimmed_size_span(b_span)); + + if (detail::is_span_zero(a_trimmed) || detail::is_span_zero(b_trimmed)) { + set_zero(); + return; + } + + const bool result_neg = is_negative() != b_neg; + const std::size_t la = a_trimmed.size(); + const std::size_t lb = b_trimmed.size(); + + if (lb == 1) { + // `b` may alias our limbs, so take its value before anything moves. + const limb_type mul = b_trimmed[0]; + limb_type* limbs = limb_ptr(); + limb_type carry = 0; + for (std::size_t i = 0; i < la; ++i) { + const auto [lo, hi] = detail::widening_mul(limbs[i], mul); + const auto [sum, c] = detail::carrying_add(lo, carry); + limbs[i] = sum; + carry = hi + static_cast(c); + } + if (carry != 0) { + grow(la + 1); + limb_ptr()[la] = carry; + unchecked_set_limb_count(static_cast(la + 1)); + } + unchecked_set_sign(result_neg); + return; + } + + const std::size_t result_size = la + lb; + constexpr size_type stack_limbs = 64; + if BEMAN_BIG_INT_IS_NOT_CONSTEVAL { + if (result_size <= stack_limbs) { + limb_type stack_buf[stack_limbs]; + const std::size_t sig = + detail::multiply_dispatch(std::span{stack_buf, result_size}, a_trimmed, b_trimmed, m_alloc); + // The operands are dead now, so growth may release the old buffer. + const size_type old_count = limb_count(); + limb_type* const dst = storage_for_overwrite(sig); + std::copy_n(stack_buf, sig, dst); + unchecked_set_limb_count(static_cast(sig)); + clear_inline_tail(old_count); + unchecked_set_sign(result_neg); + return; + } + } + + // Large product: build it in a fresh buffer, then adopt that buffer. `*this` is untouched if this throws. + basic_big_int product{m_alloc}; + product.multiply_into(a_trimmed, is_negative(), b_trimmed, b_neg); + *this = std::move(product); +} + // Compound multiplication assignment. template template @@ -2852,30 +2799,82 @@ constexpr auto basic_big_int::operator*=(T&& rhs) -> basic_big_int& requires detail::common_big_int_type_with { if constexpr (detail::is_basic_big_int_v>) { - // Move *this to a temp so the old limbs become a read-only input, - // then multiply into a new *this. - const basic_big_int temp = std::move(*this); - *this = basic_big_int{}; - // `rhs` may alias `*this` (e.g. `y *= y`). - // After the move, `rhs` still references the moved-from destination, so read both operands from `temp`. - if (std::addressof(rhs) == this) { - multiply_into(temp.representation(), temp.is_negative(), temp.representation(), temp.is_negative()); - return *this; - } - multiply_into(temp.representation(), temp.is_negative(), rhs.representation(), rhs.is_negative()); + // `rhs` may be `*this`: multiply_in_place reads both operands before writing. + multiply_in_place(rhs.representation(), rhs.is_negative()); } else { - const basic_big_int temp = std::move(*this); - *this = basic_big_int{}; - const auto rhs_limbs = detail::to_limbs(detail::uabs(rhs)); - multiply_into( - temp.representation(), temp.is_negative(), detail::to_fixed_span(rhs_limbs), detail::integer_signbit(rhs)); + const auto rhs_limbs = detail::to_limbs(detail::uabs(rhs)); + multiply_in_place(detail::to_fixed_span(rhs_limbs), detail::integer_signbit(rhs)); } return *this; } +template +template +constexpr bool +basic_big_int::divide_in_place_small(const std::span divisor, + const bool divisor_neg, + const detail::division_op op) { + if BEMAN_BIG_INT_IS_NOT_CONSTEVAL { + const std::span dividend_all{limb_ptr(), limb_count()}; + const auto dividend = dividend_all.first(detail::trimmed_size_span(dividend_all)); + const auto d = divisor.first(detail::trimmed_size_span(divisor)); + const std::size_t n = dividend.size(); + const std::size_t m = d.size(); + if (m < 2 || n < m || detail::is_span_zero(dividend) || !detail::divide_takes_schoolbook(n, m)) { + return false; + } + const std::size_t q_cap = n - m + 1; + const std::size_t r_cap = n + 1; + const std::size_t need = + detail::divide_schoolbook_storage_size(n, m, true) + (op == detail::division_op::rem ? q_cap : 0); + if (need > detail::small_division_stack_limbs) { + return false; + } + + limb_type stack_buf[detail::small_division_stack_limbs]; + detail::scratch_allocator_base scratch(stack_buf, detail::small_division_stack_limbs); + limb_type* const limbs = limb_ptr(); + const size_type old_count = limb_count(); + if (op == detail::division_op::div) { + // The schoolbook kernel copies the dividend into scratch before its first quotient write. + const std::span quot{limbs, q_cap}; + detail::divide_dispatch_q(quot, dividend, d, scratch, m_alloc); + const std::size_t qsize = detail::trimmed_size_span(std::span{limbs, q_cap}); + unchecked_set_limb_count(static_cast(qsize)); + clear_inline_tail(old_count); + unchecked_set_sign(is_negative() != divisor_neg && !unchecked_is_magnitude_zero()); + } else { + const bool neg = is_negative(); + const std::span quot_span = scratch.allocate(q_cap); + const std::span rem_span = scratch.allocate(r_cap); + detail::divide_dispatch(quot_span, rem_span, dividend, d, scratch, m_alloc); + const std::size_t rsize = detail::trimmed_size_span(std::span{rem_span}); + std::copy_n(rem_span.data(), rsize, limbs); + unchecked_set_limb_count(static_cast(rsize)); + clear_inline_tail(old_count); + unchecked_set_sign(neg && !unchecked_is_magnitude_zero()); + } + return true; + } + return false; +} + +template +constexpr auto basic_big_int::make_from_magnitude(const std::span mag, + const bool neg, + const allocator_type& a) -> basic_big_int { + basic_big_int r{a}; + limb_type* dst = r.storage_for_overwrite(mag.size()); + std::copy(mag.begin(), mag.end(), dst); + r.unchecked_set_limb_count(static_cast(mag.size())); + r.unchecked_set_sign(neg && !r.unchecked_is_magnitude_zero()); + return r; +} + // Single-limb divisor fast path from Knuth -// Avoids allocating a remainder scratch buffer and a `t` buffer: we just stream the quotient through `*this`'s -// limbs and carry a single remainder limb between iterations. +// Avoids a scratch remainder buffer and a `t` buffer. The quotient streams through `*this`'s limbs when they +// already have room; otherwise a run-time stack buffer takes it, so a quotient that trims to fit inline never +// allocates. The remainder-only form needs no quotient at all. template template constexpr uint_multiprecision_t @@ -2886,13 +2885,15 @@ basic_big_int::divmod_into_short(const std::span::divmod_into_short(const std::span quot_span{limb_ptr(), dividend_trim.size()}; - const uint_multiprecision_t remainder = detail::divide_unsigned_short(quot_span, dividend_trim, divisor); + const size_type n = dividend_trim.size(); - if (want_quotient) { - const std::size_t qsize = - detail::trimmed_size_span(std::span{limb_ptr(), dividend_trim.size()}); - unchecked_set_limb_count(static_cast(qsize)); - } else { - limb_ptr()[0] = remainder; + if (!want_quotient) { + const uint_multiprecision_t remainder = detail::mod_unsigned_short(dividend_trim, divisor); + limb_ptr()[0] = remainder; unchecked_set_limb_count(1); + clear_inline_tail(old_count); + unchecked_set_sign(result_neg && !unchecked_is_magnitude_zero()); + return remainder; } + const size_type capacity = is_representation_inplace() ? inplace_capacity : m_capacity; + constexpr size_type stack_limbs = 64; + if BEMAN_BIG_INT_IS_NOT_CONSTEVAL { + if (n > capacity && n <= stack_limbs) { + limb_type stack_buf[stack_limbs]; + const uint_multiprecision_t remainder = + detail::divide_unsigned_short(std::span{stack_buf, n}, dividend_trim, divisor); + const size_type qsize = detail::trimmed_size_span(std::span{stack_buf, n}); + limb_type* const dst = storage_for_overwrite(qsize); + std::copy_n(stack_buf, qsize, dst); + unchecked_set_limb_count(static_cast(qsize)); + clear_inline_tail(old_count); + unchecked_set_sign(result_neg && !unchecked_is_magnitude_zero()); + return remainder; + } + } + + limb_type* const dst = storage_for_overwrite(n); + const uint_multiprecision_t remainder = + detail::divide_unsigned_short(std::span{dst, n}, dividend_trim, divisor); + const size_type qsize = detail::trimmed_size_span(std::span{dst, n}); + unchecked_set_limb_count(static_cast(qsize)); + clear_inline_tail(old_count); unchecked_set_sign(result_neg && !unchecked_is_magnitude_zero()); return remainder; } @@ -2938,8 +2959,10 @@ constexpr uint_multiprecision_t basic_big_int::divmod_in_place_short(co if (want_quotient) { unchecked_trim_magnitude(); } else { - limb_ptr()[0] = remainder; + const auto old_count = limb_count(); + limb_ptr()[0] = remainder; unchecked_set_limb_count(1); + clear_inline_tail(old_count); } unchecked_set_sign(result_neg && !unchecked_is_magnitude_zero()); return remainder; @@ -2979,26 +3002,25 @@ constexpr auto basic_big_int::divmod_into(const std::span(dividend_trim.size())); + clear_inline_tail(old_count); unchecked_set_sign(dividend_neg && !unchecked_is_magnitude_zero()); - if (op == detail::division_op::rem) { - return {}; - } - - BEMAN_BIG_INT_DEBUG_ASSERT(op == detail::division_op::div_rem); - basic_big_int result = std::move(*this); - set_zero(); - return result; + return {}; } // Single-limb divisor fast path. if (divisor_trim.size() == 1) { const uint_multiprecision_t rem_limb = divmod_into_short(dividend_trim, dividend_neg, divisor_trim[0], divisor_neg, op); - if (rem_limb == 0) { + if (op != detail::division_op::div_rem || rem_limb == 0) { return {}; } basic_big_int remainder{rem_limb, m_alloc}; @@ -3006,70 +3028,63 @@ constexpr auto basic_big_int::divmod_into(const std::span scratch(r_cap + t_cap, m_alloc); - detail::divide_dispatch_q( - std::span{limb_ptr(), q_cap}, dividend_trim, divisor_trim, scratch, m_alloc); - - const std::size_t qsize = detail::trimmed_size_span(std::span{limb_ptr(), q_cap}); - unchecked_set_limb_count(static_cast(qsize)); - unchecked_set_sign(unrounded_quotient_neg && !unchecked_is_magnitude_zero()); - return {}; - } + const auto divide_with = [&](detail::scratch_allocator_base& scratch) -> basic_big_int { + if (op == detail::division_op::rem) { + // Quotient and remainder both in scratch; only the remainder is kept. + const std::span quot_span = scratch.allocate(q_cap); + const std::span rem_span = scratch.allocate(r_cap); + detail::divide_dispatch(quot_span, rem_span, dividend_trim, divisor_trim, scratch, m_alloc); + const std::size_t rsize = detail::trimmed_size_span(std::span{rem_span}); + limb_type* const dst = storage_for_overwrite(rsize); + std::copy_n(rem_span.data(), rsize, dst); + unchecked_set_limb_count(static_cast(rsize)); + clear_inline_tail(old_count); + unchecked_set_sign(dividend_neg && !unchecked_is_magnitude_zero()); + return {}; + } - if (want_quotient) { - // *this will hold the quotient. - // Remainder and `t` go in scratch. - grow(q_cap); - std::fill_n(limb_ptr(), q_cap, limb_type{0}); - - detail::scratch_allocator scratch(r_cap + t_cap, m_alloc); - const std::span rem_span = scratch.allocate(r_cap); - - detail::divide_dispatch(std::span{limb_ptr(), q_cap}, - rem_span, - dividend_trim, - divisor_trim, - scratch, - m_alloc); - - const std::size_t qsize = detail::trimmed_size_span(std::span{limb_ptr(), q_cap}); + // Quotient only: the divide-and-conquer band can skip the remainder work through the + // approximate-quotient path. With div_rem the remainder is produced into scratch. + limb_type* const dst = storage_for_overwrite(q_cap); + const std::span quot = {dst, q_cap}; + std::span rem_span = {}; + if (op == detail::division_op::div) { + detail::divide_dispatch_q(quot, dividend_trim, divisor_trim, scratch, m_alloc); + } else { + rem_span = scratch.allocate(r_cap); + detail::divide_dispatch(quot, rem_span, dividend_trim, divisor_trim, scratch, m_alloc); + } + const std::size_t qsize = detail::trimmed_size_span(std::span{dst, q_cap}); unchecked_set_limb_count(static_cast(qsize)); + clear_inline_tail(old_count); unchecked_set_sign(unrounded_quotient_neg && !unchecked_is_magnitude_zero()); + if (op == detail::division_op::div) { + return {}; + } + const std::size_t rsize = detail::trimmed_size_span(std::span{rem_span}); + return make_from_magnitude( + std::span{rem_span.data(), rsize}, dividend_neg, m_alloc); + }; - BEMAN_BIG_INT_DEBUG_ASSERT(op == detail::division_op::div_rem); - basic_big_int rem(rem_span.data(), rem_span.data() + rem_span.size(), m_alloc); - rem.unchecked_set_sign(dividend_neg && !rem.unchecked_is_magnitude_zero()); - return rem; + if BEMAN_BIG_INT_IS_NOT_CONSTEVAL { + if (scratch_limbs <= detail::small_division_stack_limbs) { + limb_type stack_buf[detail::small_division_stack_limbs]; + detail::scratch_allocator_base scratch(stack_buf, detail::small_division_stack_limbs); + return divide_with(scratch); + } } - - // *this will hold the remainder. - // Quotient and `t` go in scratch. - grow(r_cap); - std::fill_n(limb_ptr(), r_cap, limb_type{0}); - - detail::scratch_allocator scratch(q_cap + t_cap, m_alloc); - const std::span quot_span = scratch.allocate(q_cap); - - detail::divide_dispatch( - quot_span, std::span{limb_ptr(), r_cap}, dividend_trim, divisor_trim, scratch, m_alloc); - - const std::size_t rsize = detail::trimmed_size_span(std::span{limb_ptr(), r_cap}); - unchecked_set_limb_count(static_cast(rsize)); - unchecked_set_sign(dividend_neg && !unchecked_is_magnitude_zero()); - return {}; + detail::scratch_allocator scratch(scratch_limbs, m_alloc); + return divide_with(scratch); } // Simultaneously computes the quotient and remainder of a division, @@ -3195,15 +3210,16 @@ constexpr auto basic_big_int::operator/=(T&& rhs) -> basic_big_int& // storage when `&rhs == this`. if (&rhs == this) { BEMAN_BIG_INT_ASSERT(!is_zero()); - *this = basic_big_int{1}; + set_zero(); + limb_ptr()[0] = 1; return *this; } } if (rhs.limb_count() == 1) { static_cast(divmod_in_place_short(rhs.limb_ptr()[0], rhs.is_negative(), detail::division_op::div)); - } else { - const basic_big_int temp = std::move(*this); - *this = basic_big_int{}; + } else if (!divide_in_place_small(rhs.representation(), rhs.is_negative(), detail::division_op::div)) { + const basic_big_int temp(std::move(*this), m_alloc); + set_zero(); divmod_into(temp.representation(), temp.is_negative(), rhs.representation(), @@ -3215,9 +3231,10 @@ constexpr auto basic_big_int::operator/=(T&& rhs) -> basic_big_int& if (detail::trimmed_size_span(rhs_limbs) == 1) { static_cast( divmod_in_place_short(rhs_limbs[0], detail::integer_signbit(rhs), detail::division_op::div)); - } else { - const basic_big_int temp = std::move(*this); - *this = basic_big_int{}; + } else if (!divide_in_place_small( + detail::to_fixed_span(rhs_limbs), detail::integer_signbit(rhs), detail::division_op::div)) { + const basic_big_int temp(std::move(*this), m_alloc); + set_zero(); divmod_into(temp.representation(), temp.is_negative(), detail::to_fixed_span(rhs_limbs), @@ -3238,15 +3255,15 @@ constexpr auto basic_big_int::operator%=(T&& rhs) -> basic_big_int& // Self-modulus: `std::move(*this)` below would zero `rhs` when &rhs == this. if (&rhs == this) { BEMAN_BIG_INT_ASSERT(!is_zero()); - *this = basic_big_int{}; + set_zero(); return *this; } } if (rhs.limb_count() == 1) { static_cast(divmod_in_place_short(rhs.limb_ptr()[0], rhs.is_negative(), detail::division_op::rem)); - } else { - const basic_big_int temp = std::move(*this); - *this = basic_big_int{}; + } else if (!divide_in_place_small(rhs.representation(), rhs.is_negative(), detail::division_op::rem)) { + const basic_big_int temp(std::move(*this), m_alloc); + set_zero(); divmod_into(temp.representation(), temp.is_negative(), rhs.representation(), @@ -3258,9 +3275,10 @@ constexpr auto basic_big_int::operator%=(T&& rhs) -> basic_big_int& if (detail::trimmed_size_span(rhs_limbs) == 1) { static_cast( divmod_in_place_short(rhs_limbs[0], detail::integer_signbit(rhs), detail::division_op::rem)); - } else { - const basic_big_int temp = std::move(*this); - *this = basic_big_int{}; + } else if (!divide_in_place_small( + detail::to_fixed_span(rhs_limbs), detail::integer_signbit(rhs), detail::division_op::rem)) { + const basic_big_int temp(std::move(*this), m_alloc); + set_zero(); divmod_into(temp.representation(), temp.is_negative(), detail::to_fixed_span(rhs_limbs), @@ -3447,6 +3465,46 @@ constexpr void basic_big_int::grow(const size_type limbs_needed) { m_capacity = static_cast(allocation.count); } +// Same growth policy as `grow`. The new block is allocated before the old one is released. +// Runtime blocks are left uninitialized (poisoned in debug builds); constant evaluation +// must construct every limb before use. +template +constexpr auto basic_big_int::storage_for_overwrite(const size_type n) -> limb_type* { + const size_type current_cap = is_representation_inplace() ? inplace_capacity : m_capacity; + if (n > current_cap) { + if (n > max_limbs) { + detail::throw_length_error(); + } + const size_type new_cap = std::min(std::max(n, 2 * current_cap), max_limbs); + const alloc_result allocation = alloc_limbs(new_cap); + if BEMAN_BIG_INT_IS_NOT_CONSTEVAL { +#ifdef BEMAN_BIG_INT_DEBUG + std::uninitialized_fill_n(allocation.ptr, allocation.count, limb_type{~limb_type{0} / 0xFF * 0xA5}); +#endif + } else { + for (size_type i = 0; i < allocation.count; ++i) { + std::construct_at(allocation.ptr + i); + } + } + free_storage(); + m_storage.data = allocation.ptr; + m_capacity = static_cast(allocation.count); + } + return limb_ptr(); +} + +template +constexpr void basic_big_int::clear_inline_tail(const size_type old_count) noexcept { + if constexpr (inplace_capacity != 1) { + if (is_representation_inplace()) { + limb_type* const limbs = m_storage.limbs; + for (size_type i = limb_count(); i < old_count; ++i) { + limbs[i] = limb_type{0}; + } + } + } +} + BEMAN_BIG_INT_DIAGNOSTIC_POP() template diff --git a/include/beman/big_int/charconv.hpp b/include/beman/big_int/charconv.hpp index bff86253..16deda52 100644 --- a/include/beman/big_int/charconv.hpp +++ b/include/beman/big_int/charconv.hpp @@ -423,6 +423,8 @@ from_chars(const char* const begin, const char* const end, basic_big_int= 2, dividend.size() >= // divisor.size(), quotient.size() >= dividend.size() - divisor.size() + 1, // remainder.size() >= dividend.size() + 1, no aliasing, and `scratch` -// provides at least divide_unsigned_storage_size(...) limbs. +// provides at least divide_unsigned_storage_size(...) limbs. The one +// exception to no aliasing: `quotient` may start at the dividend's first limb +// (same data()), because the dividend is copied whole into scratch before the +// first quotient limb is written and never read again. // --------------------------------------------------------------------------- constexpr void divide_unsigned(const std::span quotient, const std::span remainder, @@ -66,7 +69,6 @@ constexpr void divide_unsigned(const std::span quot BEMAN_BIG_INT_DEBUG_ASSERT(dividend.size() >= divisor.size()); BEMAN_BIG_INT_DEBUG_ASSERT(quotient.size() >= dividend.size() - divisor.size() + 1); BEMAN_BIG_INT_DEBUG_ASSERT(remainder.size() >= dividend.size() + 1); - BEMAN_BIG_INT_DEBUG_ASSERT(quotient.data() != dividend.data()); BEMAN_BIG_INT_DEBUG_ASSERT(quotient.data() != divisor.data()); BEMAN_BIG_INT_DEBUG_ASSERT(remainder.data() != dividend.data()); BEMAN_BIG_INT_DEBUG_ASSERT(remainder.data() != divisor.data()); @@ -908,6 +910,37 @@ void divide_barrett_preinv(std::span quotient, std::span inv, scratch_allocator_base& scratch); +// True when the runtime dispatchers send an m-limb by s-limb division to the +// Barrett tier (see the gate rules above). +[[nodiscard]] constexpr bool divide_takes_barrett(const std::size_t m, const std::size_t s) noexcept { + const bool barrett_march = s >= barrett_march_cutoff && m / 16 >= s; + const bool barrett_march8 = s >= barrett_march8_cutoff && m / 8 >= s; + const bool barrett_quarter = m >= barrett_quarter_cutoff && m / 4 >= s; + const bool barrett_balanced = m >= barrett_balanced_cutoff && m - s >= s; + return barrett_march || barrett_march8 || barrett_quarter || barrett_balanced; +} + +// True when the runtime dispatchers take the schoolbook kernel (divide_unsigned) +// for an m-limb by s-limb division: neither the Barrett nor the +// divide-and-conquer gate fires. Constant evaluation always takes it. +[[nodiscard]] constexpr bool divide_takes_schoolbook(const std::size_t m, const std::size_t s) noexcept { + if (divide_takes_barrett(m, s)) { + return false; + } + return !(s >= burnikel_ziegler_cutoff && m - s >= burnikel_ziegler_offset); +} + +// Scratch limbs the schoolbook path of divide_dispatch (remainder_in_scratch = +// false) or divide_dispatch_q (true: the discarded remainder also lives in +// scratch) needs for an m-limb by s-limb division. +[[nodiscard]] constexpr std::size_t +divide_schoolbook_storage_size(const std::size_t m, const std::size_t s, const bool remainder_in_scratch) noexcept { + return divide_unsigned_storage_size(m, s) + (remainder_in_scratch ? m + 1 : 0); +} + +// Scratch limbs up to which a caller may keep division workspace on the stack. +inline constexpr std::size_t small_division_stack_limbs = 128; + // --------------------------------------------------------------------------- // Top-level division dispatcher (counterpart of multiply_dispatch): the Barrett // gates above are checked first, then the divide-and-conquer path, which needs @@ -930,16 +963,12 @@ constexpr void divide_dispatch(const std::span quot const std::size_t m = dividend.size(); const std::size_t s = divisor.size(); - const bool barrett_march = s >= barrett_march_cutoff && m / 16 >= s; - const bool barrett_march8 = s >= barrett_march8_cutoff && m / 8 >= s; - const bool barrett_quarter = m >= barrett_quarter_cutoff && m / 4 >= s; - const bool barrett_balanced = m >= barrett_balanced_cutoff && m - s >= s; - if (barrett_march || barrett_march8 || barrett_quarter || barrett_balanced) { - divide_barrett(quotient, remainder, dividend, divisor, alloc); - return; - } - if (s >= burnikel_ziegler_cutoff && m - s >= burnikel_ziegler_offset) { - divide_burnikel_ziegler(quotient, remainder, dividend, divisor, alloc); + if (!divide_takes_schoolbook(m, s)) { + if (divide_takes_barrett(m, s)) { + divide_barrett(quotient, remainder, dividend, divisor, alloc); + } else { + divide_burnikel_ziegler(quotient, remainder, dividend, divisor, alloc); + } return; } } @@ -967,17 +996,13 @@ constexpr void divide_dispatch_q(const std::span qu const std::size_t s = divisor.size(); if BEMAN_BIG_INT_IS_NOT_CONSTEVAL { - const bool barrett_march = s >= barrett_march_cutoff && m / 16 >= s; - const bool barrett_march8 = s >= barrett_march8_cutoff && m / 8 >= s; - const bool barrett_quarter = m >= barrett_quarter_cutoff && m / 4 >= s; - const bool barrett_balanced = m >= barrett_balanced_cutoff && m - s >= s; - if (barrett_march || barrett_march8 || barrett_quarter || barrett_balanced) { + if (divide_takes_barrett(m, s)) { const std::span r = scratch.allocate(m + 1); divide_barrett(quotient, r, dividend, divisor, alloc); scratch.deallocate(m + 1); return; } - if (s >= burnikel_ziegler_cutoff && m - s >= burnikel_ziegler_offset) { + if (!divide_takes_schoolbook(m, s)) { divide_quotient(quotient, dividend, divisor, alloc); return; } diff --git a/include/beman/big_int/detail/mul_impl.hpp b/include/beman/big_int/detail/mul_impl.hpp index cb15ee84..b7afb66d 100644 --- a/include/beman/big_int/detail/mul_impl.hpp +++ b/include/beman/big_int/detail/mul_impl.hpp @@ -1307,18 +1307,37 @@ static_assert(mul_slice_zones_valid(), "invalid slicing zone table (see mul_slic // the shape-aware dispatch names above. #define BEMAN_BIG_INT_HAS_SHAPE_DISPATCH 1 +// --------------------------------------------------------------------------- +// multiply_single_limb that also writes result[a.size()] when the product has +// no carry limb, so all a.size() + 1 limbs are defined. Same contract +// otherwise: `result` has space for a.size() + 1 limbs, no aliasing. Returns +// the number of significant result limbs. +// --------------------------------------------------------------------------- +constexpr std::size_t multiply_by_limb_full(const std::span result, + const std::span a, + const uint_multiprecision_t val) noexcept { + const std::size_t n = multiply_single_limb(result, a, val); + if (n == a.size()) { + result[n] = 0; + } + return n; +} + // --------------------------------------------------------------------------- // Runtime multiplication tier ladders (src/mul_dispatch.cpp): operands must -// be trimmed with at least two limbs; `result` pre-zeroed, sized for the -// full product, non-aliasing. Kernel workspaces come from the type-erased -// heap hooks. Returns the trimmed significant size. +// be trimmed with at least two limbs; `result` sized for the full product, +// non-aliasing, and need NOT be pre-zeroed: limbs [0, a.size() + b.size()) are +// all written (tiers that accumulate zero-fill them first), limbs beyond are +// untouched. Kernel workspaces come from the type-erased heap hooks. Returns +// the trimmed significant size. // --------------------------------------------------------------------------- std::size_t multiply_runtime(std::span result, std::span a, std::span b, const scratch_heap_source& heap); -// Benchmark/test-only variants of multiply_runtime, same contract. The +// Benchmark/test-only variants of multiply_runtime, same contract (a result +// that is not pre-zeroed is fine). The // schoolbook (min < karatsuba_cutoff) and power-of-two shortcuts still apply in // both. _sliced forces slicing of the top-level product whenever max > min, // ignoring the ratio test, and skips the FFT gate even where it would take the @@ -1341,18 +1360,37 @@ std::size_t square_runtime(std::span result, // Untrimmed/short-operand-tolerant runtime product for the src-side callers // (the division tiers and mulmod): trims, takes the single-limb shortcuts, -// then runs the tier ladder. The runtime mirror of multiply_dispatch. +// then runs the tier ladder. The runtime mirror of multiply_dispatch, with the +// same non-zeroed-result contract. std::size_t multiply_runtime_any(std::span result, std::span a_untrimmed, std::span b_untrimmed, const scratch_heap_source& heap); +// Master switch for the header-side basecase shortcut in multiply_dispatch. +inline constexpr bool mul_header_basecase_enabled = true; + +// Operand length below which multiply_dispatch calls multiply_basecase_runtime +// instead of building the scratch hooks and entering the ladder. One constant +// for every configuration; src/mul_dispatch.cpp static_asserts that it is at +// most every Karatsuba cutoff of the compiled configuration. +inline constexpr std::size_t mul_header_basecase_limbs = 32; + +// Product (or square, when a and b are the same span) of trimmed operands with +// at least two limbs each, min(a.size(), b.size()) < mul_header_basecase_limbs, +// via the basecase kernels only: no scratch. Same result contract as +// multiply_runtime (no pre-zeroing needed). Returns the trimmed size. +std::size_t multiply_basecase_runtime(std::span result, + std::span a, + std::span b) noexcept; + // --------------------------------------------------------------------------- // result <- a * p2 where p2 is a trimmed power of two: a shifted copy of `a` // placed at limb offset p2.size() - 1, bit-shifted by countr_zero(p2.back()). // O(n) with no scratch, versus a full kernel dispatch (see GMP mpz_mul_2exp). -// `result` must be pre-zeroed, have space for a.size() + p2.size() limbs, and -// must NOT alias `a`. Returns the number of significant result limbs. +// Writes all a.size() + p2.size() limbs of `result` (it need NOT be pre-zeroed), +// which must have space for them and must NOT alias `a`. Returns the number of +// significant result limbs. // --------------------------------------------------------------------------- constexpr std::size_t multiply_power_of_two(const std::span result, const std::span a, @@ -1367,14 +1405,20 @@ constexpr std::size_t multiply_power_of_two(const std::span(std::countr_zero(p2.back())); const auto dst = result.subspan(limb_offset); + std::ranges::fill(result.first(limb_offset), uint_multiprecision_t{0}); std::ranges::copy(a, dst.begin()); - return limb_offset + shift_left_n(dst, a.size(), bit_shift); + const std::size_t shifted = shift_left_n(dst, a.size(), bit_shift); + if (shifted == a.size()) { + dst[shifted] = 0; + } + return limb_offset + shifted; } // --------------------------------------------------------------------------- // Squaring dispatcher: result <- a * a using kernels that exploit the // symmetric cross products. Same contract as multiply_dispatch: `result` must -// be pre-zeroed with space for 2 * a.size() limbs and must NOT alias `a`. +// have space for 2 * a.size() limbs (it need NOT be pre-zeroed) and must NOT +// alias `a`. // `a` must be trimmed with at least two limbs. Returns the number of // significant result limbs. // --------------------------------------------------------------------------- @@ -1393,7 +1437,10 @@ std::size_t square_dispatch(const std::span result, // --------------------------------------------------------------------------- // Top-level multiplication dispatcher. -// `result` must be pre-zeroed and have space for a.size() + b.size() limbs. +// `result` must have space for a.size() + b.size() limbs; it need NOT be +// pre-zeroed. On return limbs [0, trimmed a.size() + trimmed b.size()) hold the +// product (the high ones zero when the product is shorter); limbs beyond are +// untouched, so a caller that wants zeros there fills them itself. // `result` must NOT alias `a` or `b`. // Returns the number of significant result limbs (trimmed). // --------------------------------------------------------------------------- @@ -1419,10 +1466,10 @@ constexpr std::size_t multiply_dispatch(const std::span return hi != 0 ? 2 : 1; } if (a.size() == 1) { - return multiply_single_limb(result, b, a[0]); + return multiply_by_limb_full(result, b, a[0]); } if (b.size() == 1) { - return multiply_single_limb(result, a, b[0]); + return multiply_by_limb_full(result, a, b[0]); } // This check has 0 runtime cost, but could speed up/reduce depth of constant evaluation @@ -1435,6 +1482,7 @@ constexpr std::size_t multiply_dispatch(const std::span } // Squaring halves the widening muls, and so the consteval step count. if (a.data() == b.data() && a.size() == b.size()) { + std::ranges::fill(result.first(2 * a.size()), uint_multiprecision_t{0}); square_long(result, a); return trimmed_size_span(std::span{result.data(), 2 * a.size()}); } @@ -1444,6 +1492,11 @@ constexpr std::size_t multiply_dispatch(const std::span // keeps long multiplication (the recursive tiers could blow up consteval // step limits, and the kernels are runtime-compiled). if BEMAN_BIG_INT_IS_NOT_CONSTEVAL { + if constexpr (mul_header_basecase_enabled) { + if (std::min(a.size(), b.size()) < mul_header_basecase_limbs) { + return multiply_basecase_runtime(result, a, b); + } + } const scratch_allocator hooks(alloc); return multiply_runtime(result, a, b, hooks.heap()); } else { diff --git a/include/beman/big_int/detail/span_ops.hpp b/include/beman/big_int/detail/span_ops.hpp index a66b63c7..b091be8b 100644 --- a/include/beman/big_int/detail/span_ops.hpp +++ b/include/beman/big_int/detail/span_ops.hpp @@ -708,12 +708,23 @@ constexpr std::size_t multiply_single_limb(const std::span; + for (std::size_t i = 0; i < size; ++i) { + const wide_type product = static_cast(s[i]) * mul + carry; + s[i] = static_cast(product); + carry = static_cast(product >> width_v); + } +#else for (std::size_t i = 0; i < size; ++i) { const auto [lo, hi] = widening_mul(s[i], mul); const uint_multiprecision_t next = lo + carry; carry = hi + static_cast(next < lo); s[i] = next; } +#endif if (carry == 0) { return size; } @@ -1142,6 +1153,164 @@ struct subsystem_signs { bool inner; }; +// ===== Front-end primitives: tail-aware add/sub and copying shifts ================================== +// Single-pass building blocks for the basic_big_int front end. They are independent of the +// kernels above, which stay untouched as a fixed yardstick. + +// dst[0..a.size()) = a + b, returns the carry out of the top limb. Requires a.size() >= b.size() and +// dst.size() >= a.size(). `dst` may equal `a` exactly (in-place add); otherwise it must not overlap `a`. +// Once the carry dies the rest of `a` is copied (skipped entirely when dst == a). +[[nodiscard]] constexpr bool add_n_tail(const std::span dst, + const std::span a, + const std::span b) noexcept { + BEMAN_BIG_INT_DEBUG_ASSERT(a.size() >= b.size()); + BEMAN_BIG_INT_DEBUG_ASSERT(dst.size() >= a.size()); + const std::size_t common = b.size(); + const std::size_t total = a.size(); + bool carry = false; + std::size_t i = 0; +#ifndef BEMAN_BIG_INT_GCC + // GCC re-materializes the carry flag per limb in an unrolled chain and runs the one-limb loop below faster. + for (; i + 4 <= common; i += 4) { + const auto [v0, c0] = carrying_add(a[i + 0], b[i + 0], carry); + const auto [v1, c1] = carrying_add(a[i + 1], b[i + 1], c0); + const auto [v2, c2] = carrying_add(a[i + 2], b[i + 2], c1); + const auto [v3, c3] = carrying_add(a[i + 3], b[i + 3], c2); + dst[i + 0] = v0; + dst[i + 1] = v1; + dst[i + 2] = v2; + dst[i + 3] = v3; + carry = c3; + } +#endif + for (; i < common; ++i) { + const auto [v, c] = carrying_add(a[i], b[i], carry); + dst[i] = v; + carry = c; + } + while (i < total && carry) { + const auto [v, c] = carrying_add(a[i], uint_multiprecision_t{0}, true); + dst[i] = v; + carry = c; + ++i; + } + if (dst.data() != a.data()) { + for (; i < total; ++i) { + dst[i] = a[i]; + } + } + return carry; +} + +// dst[0..a.size()) = a - b, returns the borrow out of the top limb. Same contract as `add_n_tail`. +[[nodiscard]] constexpr bool sub_n_tail(const std::span dst, + const std::span a, + const std::span b) noexcept { + BEMAN_BIG_INT_DEBUG_ASSERT(a.size() >= b.size()); + BEMAN_BIG_INT_DEBUG_ASSERT(dst.size() >= a.size()); + const std::size_t common = b.size(); + const std::size_t total = a.size(); + bool borrow = false; + std::size_t i = 0; +#ifndef BEMAN_BIG_INT_GCC + for (; i + 4 <= common; i += 4) { + const auto [v0, b0] = borrowing_sub(a[i + 0], b[i + 0], borrow); + const auto [v1, b1] = borrowing_sub(a[i + 1], b[i + 1], b0); + const auto [v2, b2] = borrowing_sub(a[i + 2], b[i + 2], b1); + const auto [v3, b3] = borrowing_sub(a[i + 3], b[i + 3], b2); + dst[i + 0] = v0; + dst[i + 1] = v1; + dst[i + 2] = v2; + dst[i + 3] = v3; + borrow = b3; + } +#endif + for (; i < common; ++i) { + const auto [v, br] = borrowing_sub(a[i], b[i], borrow); + dst[i] = v; + borrow = br; + } + while (i < total && borrow) { + const auto [v, br] = borrowing_sub(a[i], uint_multiprecision_t{0}, true); + dst[i] = v; + borrow = br; + ++i; + } + if (dst.data() != a.data()) { + for (; i < total; ++i) { + dst[i] = a[i]; + } + } + return borrow; +} + +// dst[0..n) = src[0..n) << bits for bits in [1, limb_width), returning the bits shifted out of the top +// limb. Walks top-down, so it is valid when dst >= src (including dst == src) and the ranges do not +// otherwise overlap. +constexpr uint_multiprecision_t lshift_copy(uint_multiprecision_t* const dst, + const uint_multiprecision_t* const src, + const std::size_t n, + const unsigned bits) noexcept { + constexpr unsigned limb_bits = width_v; + BEMAN_BIG_INT_DEBUG_ASSERT(bits >= 1 && bits < limb_bits); + if (n == 0) { + return 0; + } + const unsigned back = limb_bits - bits; +#ifdef BEMAN_BIG_INT_GCC + // GCC reloads src[i - 1] each iteration when dst may alias src; carry it in a register. + // Clang compiles the two-load form below better. + uint_multiprecision_t hi = src[n - 1]; + const uint_multiprecision_t out = hi >> back; + for (std::size_t i = n - 1; i > 0; --i) { + const uint_multiprecision_t lo = src[i - 1]; + dst[i] = (hi << bits) | (lo >> back); + hi = lo; + } + dst[0] = hi << bits; +#else + const uint_multiprecision_t out = src[n - 1] >> back; + for (std::size_t i = n - 1; i > 0; --i) { + dst[i] = (src[i] << bits) | (src[i - 1] >> back); + } + dst[0] = src[0] << bits; +#endif + return out; +} + +// dst[0..n) = src[0..n) >> bits for bits in [1, limb_width), returning the bits shifted out of the bottom +// limb (left-aligned in the returned limb). Walks bottom-up, so it is valid when dst <= src (including +// dst == src) and the ranges do not otherwise overlap. +constexpr uint_multiprecision_t rshift_copy(uint_multiprecision_t* const dst, + const uint_multiprecision_t* const src, + const std::size_t n, + const unsigned bits) noexcept { + constexpr unsigned limb_bits = width_v; + BEMAN_BIG_INT_DEBUG_ASSERT(bits >= 1 && bits < limb_bits); + if (n == 0) { + return 0; + } + const unsigned back = limb_bits - bits; +#ifdef BEMAN_BIG_INT_GCC + // See lshift_copy. + uint_multiprecision_t lo = src[0]; + const uint_multiprecision_t out = lo << back; + for (std::size_t i = 0; i + 1 < n; ++i) { + const uint_multiprecision_t hi = src[i + 1]; + dst[i] = (lo >> bits) | (hi << back); + lo = hi; + } + dst[n - 1] = lo >> bits; +#else + const uint_multiprecision_t out = src[0] << back; + for (std::size_t i = 0; i + 1 < n; ++i) { + dst[i] = (src[i] >> bits) | (src[i + 1] << back); + } + dst[n - 1] = src[n - 1] >> bits; +#endif + return out; +} + } // namespace detail BEMAN_BIG_INT_END_NAMESPACE diff --git a/src/mul_dispatch.cpp b/src/mul_dispatch.cpp index 1f1ff152..3c17d001 100644 --- a/src/mul_dispatch.cpp +++ b/src/mul_dispatch.cpp @@ -71,6 +71,9 @@ std::size_t square_runtime(const std::span result, } } + // The Karatsuba/Toom squaring tiers accumulate into the result. + std::ranges::fill(result.first(result_total), uint_multiprecision_t{0}); + const auto in_heap_scratch = [&](const std::size_t limbs, auto&& kernel) { scratch_heap_array buf(heap, limbs); scratch_allocator_base scratch(buf.data(), limbs); @@ -172,9 +175,10 @@ std::size_t multiply_runtime_impl(std::span result, // (it may slice the other way or take the FFT) and allocates its own scratch, // possibly after piece 0 has been written. On x86 (model off, gate = floor on // min) slicing is only entered when min = m is below the floor, so no piece -// takes the FFT and this path is a no-op there. Callers pre-zero the result and -// discard it if an allocation throws. Requires trimmed operands, -// min >= karatsuba_cutoff and a pre-zeroed result. +// takes the FFT and this path is a no-op there. The caller discards the result +// if an allocation throws. Requires trimmed operands, min >= karatsuba_cutoff +// and a result whose first a.size() + b.size() limbs are pre-zeroed (the +// pieces accumulate into it). std::size_t multiply_sliced(const std::span result, const std::span a, const std::span b, @@ -306,6 +310,9 @@ std::size_t multiply_runtime_impl(const std::span r } } + // The slicing and Karatsuba/Toom tiers accumulate into the result. + std::ranges::fill(result.first(result_total), uint_multiprecision_t{0}); + const bool slice = mode == slice_mode::forced ? max_size > min_size : mode == slice_mode::automatic ? mul_should_slice(min_size, max_size) : false; @@ -358,14 +365,47 @@ std::size_t multiply_runtime_any(const std::span re return hi != 0 ? 2 : 1; } if (a.size() == 1) { - return multiply_single_limb(result, b, a[0]); + return multiply_by_limb_full(result, b, a[0]); } if (b.size() == 1) { - return multiply_single_limb(result, a, b[0]); + return multiply_by_limb_full(result, a, b[0]); } return multiply_runtime(result, a, b, heap); } +// The header only calls this below mul_header_basecase_limbs, so every +// operand-size gate used here must stay under the compiled cutoffs. +static_assert(mul_header_basecase_limbs <= karatsuba_cutoff); +static_assert(mul_header_basecase_limbs <= square_karatsuba_cutoff); + +std::size_t multiply_basecase_runtime(const std::span result, + const std::span a, + const std::span b) noexcept { + BEMAN_BIG_INT_DEBUG_ASSERT(a.size() >= 2); + BEMAN_BIG_INT_DEBUG_ASSERT(b.size() >= 2); + BEMAN_BIG_INT_DEBUG_ASSERT(a.back() != 0); + BEMAN_BIG_INT_DEBUG_ASSERT(b.back() != 0); + BEMAN_BIG_INT_DEBUG_ASSERT(std::min(a.size(), b.size()) < mul_header_basecase_limbs); + BEMAN_BIG_INT_DEBUG_ASSERT(result.size() >= a.size() + b.size()); + + const std::size_t result_total = a.size() + b.size(); + + // Mirrors multiply_runtime_impl / square_runtime below their Karatsuba cutoffs. + if (a.data() == b.data() && a.size() == b.size()) { + const std::size_t n = a.size(); + if (n < square_long_cutoff) { + ::beman_big_int_multiply_long_runtime(result.data(), a.data(), n, a.data(), n); + } else if (is_power_of_two_span(a)) { + return multiply_power_of_two(result, a, a); + } else { + ::beman_big_int_square_long_runtime(result.data(), a.data(), n); + } + } else { + ::beman_big_int_multiply_long_runtime(result.data(), a.data(), a.size(), b.data(), b.size()); + } + return trimmed_size_span(std::span{result.data(), result_total}); +} + } // namespace detail BEMAN_BIG_INT_END_NAMESPACE diff --git a/tests/beman/big_int/alloc_count.test.cpp b/tests/beman/big_int/alloc_count.test.cpp new file mode 100644 index 00000000..d4dba9fa --- /dev/null +++ b/tests/beman/big_int/alloc_count.test.cpp @@ -0,0 +1,516 @@ +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-License-Identifier: BSL-1.0 + +// Steady-state heap allocations per operation. Every case runs one warm-up call (so destinations reach their working +// capacity), then K calls, and reports the allocator traffic per call. Three environments: +// counting basic_big_int<64, limb, counting_allocator> (stateful allocator, util_counting_allocator.hpp) +// pmr pmr::big_int with a counting memory_resource installed through std::pmr::set_default_resource, so +// results (which take select_on_container_copy_construction of an operand allocator) land on it +// wide256 basic_big_int<256, limb, counting_allocator> (inline capacity of 4 64-bit limbs) +// wide128, wide512 the same with 128 and 512 inline bits (the inline-capacity study; no baseline column) +// The measured counts are printed to stdout as "[alloc_count] ...". The expectations below are the item 1 targets +// (see the front-end cost plan); with `item1_pending` true, cases that exceed their target at the baseline (cf1cf54) +// are skipped instead of failing. The constant is false now that the front end changes have landed. +// +// Baseline counts measured at cf1cf54 (allocations per call, K = 64 after one warm-up call; `want` is the target the +// case asserts; the cases whose baseline exceeds `want` are the pending ones): +// +// case want counting pmr wide256 +// ---------------------------------------------------------------------------- +// c = a + b (16x16) 1 1 1 1 +// c = a - b (16x16) 1 1 1 1 +// c += b; c -= b (16x16) 0 0 0 0 +// c = a << 13 (16 limbs) 1 1 1 1 +// c = a >> 13 (16 limbs) 1 1 1 1 +// c <<= 13; c >>= 13 (16 limbs) 0 0 0 0 +// c = a * b (8x8) 1 1 1 1 +// c = a; c *= b (8x8 -> 16) 0 1 1 1 +// c = a; c *= 7 (8 limbs) 0 1 1 1 +// c = a / b (8x4) 1 2 2 2 +// c = a % b (8x4) 1 2 2 2 +// div_rem_to_zero (8x4) <= 2 4 4 3 +// c = a / b (2x1) 0 (>= 2 inline) 1 1 0 +// c = a; c /= b (8x4) [stretch] 0 3 3 2 +// c = a; c %= b (8x4) [stretch] 0 2 2 2 +// from_chars 2000 digits (reserved) <= 2 2 2 2 +// +// Every case is enforced (item1_pending is false). The div_rem_to_zero baseline counts the quotient (5 limbs), the +// 22-limb scratch and the over-reserved remainder (9 limbs) plus one more on the counting environments for operands +// that need normalising; an earlier version of this table (5, 5, 4) also counted the check's own temporaries. Set +// `item1_pending` back to true to skip a case whose baseline count exceeds its target. + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include "util/util_counting_allocator.hpp" + +namespace { + +using BEMAN_BIG_INT_NAMESPACE::basic_big_int; +using BEMAN_BIG_INT_NAMESPACE::div_rem_to_zero; +using BEMAN_BIG_INT_NAMESPACE::from_chars; +using BEMAN_BIG_INT_NAMESPACE::uint_multiprecision_t; +using BEMAN_BIG_INT_NAMESPACE::test_util::counting_allocator; +using BEMAN_BIG_INT_NAMESPACE::test_util::counting_state; + +using limb = uint_multiprecision_t; + +// Cases that exceed their target at the baseline are skipped while this is true. +constexpr bool item1_pending = false; + +constexpr int calls = 64; // K + +std::vector random_limbs(const std::size_t n, std::uint64_t seed) { + std::vector v(n); + for (auto& x : v) { + seed ^= seed << 13U; + seed ^= seed >> 7U; + seed ^= seed << 17U; + x = static_cast(seed); + } + if (v.back() == 0) { + v.back() = 1; + } + return v; +} + +// Like random_limbs with the top bit set, so sums, shifts and products have the full worst-case limb count. +std::vector full_limbs(const std::size_t n, const std::uint64_t seed) { + auto v = random_limbs(n, seed); + v.back() |= limb{1} << (std::numeric_limits::digits - 1); + return v; +} + +// ----- environments ----- + +struct counting_env { + static constexpr const char* name = "counting"; + static constexpr std::size_t index = 0; + + using int_t = basic_big_int<64, limb, counting_allocator>; + + counting_state state; + counting_allocator alloc{&state}; + + [[nodiscard]] int_t empty() { return int_t(alloc); } + [[nodiscard]] int_t make(const std::vector& v) { return int_t(v.begin(), v.end(), alloc); } + [[nodiscard]] std::size_t count() const { return state.total_allocations; } +}; + +struct wide256_env { + static constexpr const char* name = "wide256"; + static constexpr std::size_t index = 2; + + using int_t = basic_big_int<256, limb, counting_allocator>; + + counting_state state; + counting_allocator alloc{&state}; + + [[nodiscard]] int_t empty() { return int_t(alloc); } + [[nodiscard]] int_t make(const std::vector& v) { return int_t(v.begin(), v.end(), alloc); } + [[nodiscard]] std::size_t count() const { return state.total_allocations; } +}; + +struct wide128_env { + static constexpr const char* name = "wide128"; + static constexpr std::size_t index = 3; + + using int_t = basic_big_int<128, limb, counting_allocator>; + + counting_state state; + counting_allocator alloc{&state}; + + [[nodiscard]] int_t empty() { return int_t(alloc); } + [[nodiscard]] int_t make(const std::vector& v) { return int_t(v.begin(), v.end(), alloc); } + [[nodiscard]] std::size_t count() const { return state.total_allocations; } +}; + +struct wide512_env { + static constexpr const char* name = "wide512"; + static constexpr std::size_t index = 4; + + using int_t = basic_big_int<512, limb, counting_allocator>; + + counting_state state; + counting_allocator alloc{&state}; + + [[nodiscard]] int_t empty() { return int_t(alloc); } + [[nodiscard]] int_t make(const std::vector& v) { return int_t(v.begin(), v.end(), alloc); } + [[nodiscard]] std::size_t count() const { return state.total_allocations; } +}; + +class counting_resource final : public std::pmr::memory_resource { + public: + [[nodiscard]] std::size_t allocations() const noexcept { return m_allocs; } + + private: + [[nodiscard]] void* do_allocate(const std::size_t bytes, const std::size_t align) override { + ++m_allocs; + return std::pmr::new_delete_resource()->allocate(bytes, align); + } + void do_deallocate(void* p, const std::size_t bytes, const std::size_t align) override { + std::pmr::new_delete_resource()->deallocate(p, bytes, align); + } + [[nodiscard]] bool do_is_equal(const std::pmr::memory_resource& other) const noexcept override { + return this == &other; + } + + std::size_t m_allocs{0}; +}; + +struct pmr_env { + static constexpr const char* name = "pmr"; + static constexpr std::size_t index = 1; + + using int_t = BEMAN_BIG_INT_NAMESPACE::pmr::big_int; + + counting_resource resource; + std::pmr::memory_resource* previous; + + pmr_env() : previous(std::pmr::set_default_resource(&resource)) {} + ~pmr_env() { std::pmr::set_default_resource(previous); } + pmr_env(const pmr_env&) = delete; + pmr_env& operator=(const pmr_env&) = delete; + + [[nodiscard]] int_t empty() { return int_t(); } + [[nodiscard]] int_t make(const std::vector& v) { return int_t(v.begin(), v.end()); } + [[nodiscard]] std::size_t count() const { return resource.allocations(); } +}; + +// One warm-up call, then `k` calls; returns allocations per call. +template +double steady_allocs(Env& env, F&& f, const int k = calls) { + f(); + const std::size_t before = env.count(); + for (int i = 0; i < k; ++i) { + f(); + } + return static_cast(env.count() - before) / static_cast(k); +} + +// Keeps a result observable without allocating. +volatile std::size_t escape_sink = 0; + +template +void escape_value(const T& x) { + escape_sink = x.size(); +} + +// Baseline (cf1cf54) count per environment, in the order counting, pmr, wide256, wide128, wide512 (-1: not measured). +using baseline_counts = std::array; + +} // namespace + +// Prints the measurement; fails when it exceeds `want`, or skips when the case is known to exceed it at the baseline +// and the item 1 changes are still pending. +#define ALLOC_EXPECT(Env, case_name, measured, want, baseline) \ + do { \ + const double alloc_measured_ = (measured); \ + const double alloc_want_ = (want); \ + const double alloc_base_ = (baseline)[Env::index]; \ + std::printf("[alloc_count] %-8s %-34s measured=%6.3f want<=%5.2f baseline=%6.3f\n", \ + Env::name, \ + case_name, \ + alloc_measured_, \ + alloc_want_, \ + alloc_base_); \ + if (alloc_measured_ > alloc_want_ && item1_pending && alloc_base_ > alloc_want_) { \ + GTEST_SKIP() << "item 1 pending: " << case_name << " measures " << alloc_measured_ \ + << ", want <= " << alloc_want_; \ + } \ + EXPECT_LE(alloc_measured_, alloc_want_) << case_name; \ + } while (false) + +namespace { + +template +class AllocCount : public ::testing::Test {}; + +using envs = ::testing::Types; +TYPED_TEST_SUITE(AllocCount, envs); + +// Baseline columns are filled from the baseline run (see the table at the top of the file). + +// ----- add and subtract ----- + +TYPED_TEST(AllocCount, AddHeapResult) { + TypeParam env; + using int_t = typename TypeParam::int_t; + const int_t a = env.make(random_limbs(16, 1)); + const int_t b = env.make(random_limbs(16, 2)); + int_t c = env.empty(); + const double n = steady_allocs(env, [&] { c = a + b; }); + EXPECT_TRUE(c - b == a); + ALLOC_EXPECT(TypeParam, "c = a + b (16x16)", n, 1.0, (baseline_counts{1.0, 1.0, 1.0, -1.0, -1.0})); +} + +TYPED_TEST(AllocCount, SubHeapResult) { + TypeParam env; + using int_t = typename TypeParam::int_t; + const int_t a = env.make(random_limbs(16, 3)); + const int_t b = env.make(random_limbs(16, 4)); + int_t c = env.empty(); + const double n = steady_allocs(env, [&] { c = a - b; }); + EXPECT_TRUE(c + b == a); + ALLOC_EXPECT(TypeParam, "c = a - b (16x16)", n, 1.0, (baseline_counts{1.0, 1.0, 1.0, -1.0, -1.0})); +} + +TYPED_TEST(AllocCount, AddSubInPlaceWithCapacity) { + TypeParam env; + using int_t = typename TypeParam::int_t; + const int_t a = env.make(random_limbs(16, 5)); + const int_t b = env.make(random_limbs(16, 6)); + int_t c = a; + const double n = steady_allocs(env, [&] { + c += b; + c -= b; + }); + EXPECT_TRUE(c == a); + ALLOC_EXPECT(TypeParam, "c += b; c -= b (16x16)", n, 0.0, (baseline_counts{0.0, 0.0, 0.0, -1.0, -1.0})); +} + +// ----- shifts ----- + +TYPED_TEST(AllocCount, ShiftLeftHeapResult) { + TypeParam env; + using int_t = typename TypeParam::int_t; + const int_t a = env.make(random_limbs(16, 7)); + int_t c = env.empty(); + const double n = steady_allocs(env, [&] { c = a << 13; }); + EXPECT_TRUE((c >> 13) == a); + ALLOC_EXPECT(TypeParam, "c = a << 13 (16 limbs)", n, 1.0, (baseline_counts{1.0, 1.0, 1.0, -1.0, -1.0})); +} + +TYPED_TEST(AllocCount, ShiftRightHeapResult) { + TypeParam env; + using int_t = typename TypeParam::int_t; + const int_t a = env.make(random_limbs(16, 8)); + int_t c = env.empty(); + const double n = steady_allocs(env, [&] { c = a >> 13; }); + EXPECT_TRUE((c << 13) <= a); + ALLOC_EXPECT(TypeParam, "c = a >> 13 (16 limbs)", n, 1.0, (baseline_counts{1.0, 1.0, 1.0, -1.0, -1.0})); +} + +TYPED_TEST(AllocCount, ShiftInPlaceWithCapacity) { + TypeParam env; + using int_t = typename TypeParam::int_t; + const int_t a = env.make(random_limbs(16, 9)); + int_t c = a; + const double n = steady_allocs(env, [&] { + c <<= 13; + c >>= 13; + }); + EXPECT_TRUE(c == a); + ALLOC_EXPECT(TypeParam, "c <<= 13; c >>= 13 (16 limbs)", n, 0.0, (baseline_counts{0.0, 0.0, 0.0, -1.0, -1.0})); +} + +// ----- multiply ----- + +TYPED_TEST(AllocCount, MultiplyHeapResult) { + TypeParam env; + using int_t = typename TypeParam::int_t; + const int_t a = env.make(random_limbs(8, 10)); + const int_t b = env.make(random_limbs(8, 11)); + int_t c = env.empty(); + const double n = steady_allocs(env, [&] { c = a * b; }); + EXPECT_TRUE(c / b == a); + ALLOC_EXPECT(TypeParam, "c = a * b (8x8)", n, 1.0, (baseline_counts{1.0, 1.0, 1.0, -1.0, -1.0})); +} + +// c is restored from a inside the timed call (a copy into existing capacity), so the pair is steady. +TYPED_TEST(AllocCount, MultiplyInPlaceByBigInt) { + TypeParam env; + using int_t = typename TypeParam::int_t; + const int_t a = env.make(random_limbs(8, 12)); + const int_t b = env.make(random_limbs(8, 13)); + int_t c = env.empty(); + const double n = steady_allocs(env, [&] { + c = a; + c *= b; + }); + EXPECT_TRUE(c == a * b); + ALLOC_EXPECT(TypeParam, "c = a; c *= b (8x8 -> 16)", n, 0.0, (baseline_counts{1.0, 1.0, 1.0, -1.0, -1.0})); +} + +TYPED_TEST(AllocCount, MultiplyInPlaceBySmall) { + TypeParam env; + using int_t = typename TypeParam::int_t; + const int_t a = env.make(random_limbs(8, 14)); + int_t c = env.empty(); + const double n = steady_allocs(env, [&] { + c = a; + c *= 7; + }); + EXPECT_TRUE(c == a * 7); + ALLOC_EXPECT(TypeParam, "c = a; c *= 7 (8 limbs)", n, 0.0, (baseline_counts{1.0, 1.0, 1.0, -1.0, -1.0})); +} + +// ----- divide ----- + +TYPED_TEST(AllocCount, DivideHeapResult) { + TypeParam env; + using int_t = typename TypeParam::int_t; + const int_t a = env.make(random_limbs(8, 15)); + const int_t b = env.make(random_limbs(4, 16)); + int_t c = env.empty(); + const double n = steady_allocs(env, [&] { c = a / b; }); + EXPECT_TRUE(c * b + a % b == a); + ALLOC_EXPECT(TypeParam, "c = a / b (8x4)", n, 1.0, (baseline_counts{2.0, 2.0, 2.0, -1.0, -1.0})); +} + +TYPED_TEST(AllocCount, RemainderHeapResult) { + TypeParam env; + using int_t = typename TypeParam::int_t; + const int_t a = env.make(random_limbs(8, 17)); + const int_t b = env.make(random_limbs(4, 18)); + int_t c = env.empty(); + const double n = steady_allocs(env, [&] { c = a % b; }); + EXPECT_TRUE(a / b * b + c == a); + ALLOC_EXPECT(TypeParam, "c = a % b (8x4)", n, 1.0, (baseline_counts{2.0, 2.0, 2.0, -1.0, -1.0})); +} + +TYPED_TEST(AllocCount, DivRemToZero) { + TypeParam env; + using int_t = typename TypeParam::int_t; + const int_t a = env.make(random_limbs(8, 19)); + const int_t b = env.make(random_limbs(4, 20)); + // The result is checked after the measurement so the check's own temporaries are not counted. + const double n = steady_allocs(env, [&] { + const auto qr = div_rem_to_zero(a, b); + escape_value(qr.quotient); + }); + const auto qr = div_rem_to_zero(a, b); + EXPECT_TRUE(qr.quotient * b + qr.remainder == a); + ALLOC_EXPECT(TypeParam, "div_rem_to_zero (8x4)", n, 2.0, (baseline_counts{4.0, 4.0, 3.0, -1.0, -1.0})); +} + +// A two-limb dividend over a one-limb divisor, quotient two limbs: it fits inline only when the inline capacity is at +// least two limbs, so the case applies to wide256 only. +TYPED_TEST(AllocCount, DivideTwoByOneQuotientInline) { + TypeParam env; + using int_t = typename TypeParam::int_t; + const int_t a = env.make(std::vector{random_limbs(1, 21)[0], ~limb{0}}); + const int_t b = env.make(std::vector{limb{0x12345678U}}); + int_t c = env.empty(); + const double n = steady_allocs(env, [&] { c = a / b; }); + EXPECT_TRUE(c * b + a % b == a); + if constexpr (int_t::inplace_capacity < 2) { + std::printf("[alloc_count] %-8s %-34s measured=%6.3f (inline capacity %zu: quotient needs the heap, n/a)\n", + TypeParam::name, + "c = a / b (2x1)", + n, + static_cast(int_t::inplace_capacity)); + GTEST_SKIP() << "the 2-limb quotient only fits inline at inline capacity >= 2"; + } else { + ALLOC_EXPECT(TypeParam, "c = a / b (2x1, inline q)", n, 0.0, (baseline_counts{1.0, 1.0, 0.0, -1.0, -1.0})); + } +} + +// Stretch: in the schoolbook band the quotient could be built in the object's own limbs. +TYPED_TEST(AllocCount, DivideInPlace) { + TypeParam env; + using int_t = typename TypeParam::int_t; + const int_t a = env.make(random_limbs(8, 22)); + const int_t b = env.make(random_limbs(4, 23)); + int_t c = env.empty(); + const double n = steady_allocs(env, [&] { + c = a; + c /= b; + }); + EXPECT_TRUE(c == a / b); + ALLOC_EXPECT(TypeParam, "c = a; c /= b (8x4) [stretch]", n, 0.0, (baseline_counts{3.0, 3.0, 2.0, -1.0, -1.0})); +} + +TYPED_TEST(AllocCount, RemainderInPlace) { + TypeParam env; + using int_t = typename TypeParam::int_t; + const int_t a = env.make(random_limbs(8, 24)); + const int_t b = env.make(random_limbs(4, 25)); + int_t c = env.empty(); + const double n = steady_allocs(env, [&] { + c = a; + c %= b; + }); + EXPECT_TRUE(c == a % b); + ALLOC_EXPECT(TypeParam, "c = a; c %= b (8x4) [stretch]", n, 0.0, (baseline_counts{2.0, 2.0, 2.0, -1.0, -1.0})); +} + +// ----- decimal from_chars ----- + +TYPED_TEST(AllocCount, FromCharsDecimalIntoReserved) { + TypeParam env; + using int_t = typename TypeParam::int_t; + std::string digits(2000, '0'); + for (std::size_t i = 0; i < digits.size(); ++i) { + digits[i] = static_cast('0' + (i * 7 + 3) % 10); + } + digits[0] = '9'; + int_t c = env.empty(); + c.reserve(2000 * 3 + 800); // 2000 decimal digits are about 6644 bits + std::errc ec = std::errc{}; + const double n = steady_allocs( + env, + [&] { + const auto r = from_chars(digits.data(), digits.data() + digits.size(), c); + ec = r.ec; + }, + 16); + EXPECT_EQ(ec, std::errc{}); + // The value must be 2000 decimal digits long. + EXPECT_GT(c.size(), 6600u); + ALLOC_EXPECT(TypeParam, "from_chars 2000 digits (reserved)", n, 2.0, (baseline_counts{2.0, 2.0, 2.0, -1.0, -1.0})); +} + +// ----- small shapes (the inline-capacity study): allocations per call at 1, 2, 4 and 8 limbs ----- +// A result that fits the inline capacity allocates nothing; otherwise one block. Counts are printed for the study +// table. + +TYPED_TEST(AllocCount, SmallShapes) { + TypeParam env; + using int_t = typename TypeParam::int_t; + for (const std::size_t n : {std::size_t{1}, std::size_t{2}, std::size_t{4}, std::size_t{8}}) { + const int_t a = env.make(full_limbs(n, 40 + n)); + const int_t b = env.make(full_limbs(n, 50 + n)); + const int_t d = env.make(full_limbs((n + 1) / 2, 60 + n)); + int_t c = env.empty(); + int_t q = env.empty(); + int_t r = env.empty(); + const std::string shape = std::to_string(n) + " limbs"; + const auto row = [&](const char* op, const double allocs, const double want) { + const std::string name = std::string(op) + " (" + shape + ")"; + std::printf( + "[alloc_count] %-8s %-34s measured=%6.3f small-shape\n", TypeParam::name, name.c_str(), allocs); + EXPECT_LE(allocs, want) << name; + }; + row("c = a + b", steady_allocs(env, [&] { c = a + b; }), 1.0); + row("c = a - b", steady_allocs(env, [&] { c = a - b; }), 1.0); + row("c = a << 13", steady_allocs(env, [&] { c = a << 13; }), 1.0); + row("c = a >> 13", steady_allocs(env, [&] { c = a >> 13; }), 1.0); + row("c = a * b", steady_allocs(env, [&] { c = a * b; }), 1.0); + row("c = a * a", steady_allocs(env, [&] { c = a * a; }), 1.0); + row("c = a / d (n x n/2)", steady_allocs(env, [&] { c = a / d; }), 1.0); + row("div_rem_to_zero (n x n/2)", + steady_allocs(env, + [&] { + const auto qr = div_rem_to_zero(a, d); + escape_value(qr.quotient); + }), + 2.0); + q = a / d; + r = a % d; + EXPECT_TRUE(q * d + r == a); + } +} + +} // namespace diff --git a/tests/beman/big_int/allocation.test.cpp b/tests/beman/big_int/allocation.test.cpp index 67c651ca..570f6603 100644 --- a/tests/beman/big_int/allocation.test.cpp +++ b/tests/beman/big_int/allocation.test.cpp @@ -2,7 +2,10 @@ // SPDX-License-Identifier: BSL-1.0 #include +#include +#include #include +#include #include #include #include @@ -310,6 +313,79 @@ TEST(Allocation, PropagatingAssignmentIsStrongWhenAllocationThrows) { EXPECT_EQ(dst, src); } +// Compound multiplication, division and modulus keep the object's own allocator. These objects use a +// propagating-on-move-assignment allocator, so replacing `*this` with a default-constructed one would +// reset its id. +TEST(Allocation, CompoundMultiplyDivideModulusKeepPropagatingAllocator) { + using alloc_t = pocca_alloc; + const auto big = [](std::size_t id, unsigned shift) { + pocca_big_int x{1, alloc_t{id}}; + x <<= shift; + x += 12345; + return x; + }; + // Small (inline), a few limbs (stack product), and large (fresh-buffer product) operands. + for (const unsigned shift : {10U, 200U, 700U, 6000U}) { + pocca_big_int a = big(7U, shift); + pocca_big_int b = big(7U, 130U); + pocca_big_int c = big(7U, 64U); + const pocca_big_int a0 = a; + + a *= b; + EXPECT_EQ(a.get_allocator().id, 7U) << "*= shift " << shift; + EXPECT_EQ(a, a0 * b); + + a /= b; + EXPECT_EQ(a.get_allocator().id, 7U) << "/= shift " << shift; + EXPECT_EQ(a, a0); + + a *= a; + EXPECT_EQ(a.get_allocator().id, 7U) << "x *= x shift " << shift; + EXPECT_EQ(a, a0 * a0); + + a %= b; + EXPECT_EQ(a.get_allocator().id, 7U) << "%= shift " << shift; + EXPECT_EQ(a, (a0 * a0) % b); + + a = a0; + a /= a; + EXPECT_EQ(a.get_allocator().id, 7U) << "x /= x shift " << shift; + EXPECT_EQ(a, 1); + + a = a0; + a %= a; + EXPECT_EQ(a.get_allocator().id, 7U) << "x %= x shift " << shift; + EXPECT_EQ(a, 0); + + a = a0; + a *= 3; + a /= 3; + a %= 1000003; + EXPECT_EQ(a.get_allocator().id, 7U) << "integer rhs shift " << shift; + + // Divisors and multipliers that fit one limb take the in-place paths. + a = a0; + a *= c; + a /= c; + EXPECT_EQ(a.get_allocator().id, 7U); + EXPECT_EQ(a, a0); + } +} + +TEST(Allocation, MultiplyAndDivideResultsTakeAllocatorFromOperand) { + using alloc_t = pocca_alloc; + pocca_big_int a{1, alloc_t{3U}}; + a <<= 500; + pocca_big_int b{1, alloc_t{3U}}; + b <<= 190; + EXPECT_EQ((a * b).get_allocator().id, 3U); + EXPECT_EQ((a / b).get_allocator().id, 3U); + EXPECT_EQ((a % b).get_allocator().id, 3U); + const auto both = div_rem_to_zero(a + 1, b + 1); + EXPECT_EQ(both.quotient.get_allocator().id, 3U); + EXPECT_EQ(both.remainder.get_allocator().id, 3U); +} + // A stateful allocator whose `select_on_container_copy_construction` hands back a // distinct allocator (`id + 1`) instead of a copy. A copy-constructed container // must adopt that allocator, so `id` shows whether the constructor consulted the @@ -847,6 +923,10 @@ TEST(Allocation, SelfAssignment) { } // ----- operator= storage reuse ----- +// Limbs in 2^64, whatever the limb width. +constexpr std::size_t two_pow_64_limbs = + 64U / static_cast(std::numeric_limits::digits) + 1U; + // The shared `assign_value` helper keeps the destination's allocation if its // effective capacity already fits the source. These tests verify the fast path // by checking that the destination's data pointer does not change. @@ -859,35 +939,51 @@ TEST(Allocation, CopyAssignReusesDstStorage) { const BEMAN_BIG_INT_NAMESPACE::big_int src = BEMAN_BIG_INT_NAMESPACE::big_int{0xFFFFFFFFFFFFFFFFU} + BEMAN_BIG_INT_NAMESPACE::big_int{1}; - ASSERT_EQ(src.representation().size(), 2U); // heap, 2 limbs -- fits in dst's capacity + ASSERT_EQ(src.representation().size(), two_pow_64_limbs); // heap -- fits in dst's capacity dst = src; EXPECT_EQ(dst.representation().data(), dst_data); // no reallocation EXPECT_EQ(dst.representation_capacity(), dst_cap); - ASSERT_EQ(dst.representation().size(), 2U); + ASSERT_EQ(dst.representation().size(), two_pow_64_limbs); EXPECT_EQ(dst, src); } -TEST(Allocation, MoveAssignReusesDstStorageWhenLarger) { - // When dst's capacity already covers src's limb count, assign_value should - // copy src's limbs into dst's buffer rather than stealing src's (smaller) - // buffer. +TEST(Allocation, MoveAssignStealsHeapSrcEvenWhenDstLarger) { + // A heap source is stolen whatever capacity dst already holds: the source buffer is + // adopted, dst's own buffer is released, and the source is left as inline zero. BEMAN_BIG_INT_NAMESPACE::big_int dst{1U}; dst.reserve_representation(16); // big dst buffer - const auto* const dst_data = dst.representation().data(); - const auto dst_cap = dst.representation_capacity(); + const auto dst_cap = dst.representation_capacity(); BEMAN_BIG_INT_NAMESPACE::big_int src = BEMAN_BIG_INT_NAMESPACE::big_int{0xFFFFFFFFFFFFFFFFU} + BEMAN_BIG_INT_NAMESPACE::big_int{1}; - ASSERT_EQ(src.representation().size(), 2U); - const auto src_cap = src.representation_capacity(); + ASSERT_EQ(src.representation().size(), two_pow_64_limbs); + const auto* const src_data = src.representation().data(); + const auto src_cap = src.representation_capacity(); ASSERT_LT(src_cap, dst_cap); // dst has more capacity than src dst = std::move(src); - // dst retained its larger buffer rather than adopting src's smaller one. + EXPECT_EQ(dst.representation().data(), src_data); + EXPECT_EQ(dst.representation_capacity(), src_cap); + ASSERT_EQ(dst.representation().size(), two_pow_64_limbs); + EXPECT_TRUE(is_inplace(src)); + EXPECT_EQ(src, 0U); +} + +TEST(Allocation, MoveAssignInlineSrcReusesDstStorage) { + // An inline source has no buffer to steal, so it is copied into dst's existing storage. + BEMAN_BIG_INT_NAMESPACE::big_int dst{1U}; + dst.reserve_representation(16); + const auto* const dst_data = dst.representation().data(); + const auto dst_cap = dst.representation_capacity(); + + BEMAN_BIG_INT_NAMESPACE::big_int src{5U}; + ASSERT_TRUE(is_inplace(src)); + + dst = std::move(src); EXPECT_EQ(dst.representation().data(), dst_data); EXPECT_EQ(dst.representation_capacity(), dst_cap); - ASSERT_EQ(dst.representation().size(), 2U); + EXPECT_EQ(dst, 5U); } TEST(Allocation, MoveAssignStealsSrcWhenDstTooSmall) { @@ -920,7 +1016,7 @@ TEST(Allocation, CopyAssignAllocatesWhenDstTooSmall) { ASSERT_FALSE(is_inplace(src)); dst = src; - ASSERT_EQ(dst.representation().size(), 2U); + ASSERT_EQ(dst.representation().size(), two_pow_64_limbs); EXPECT_FALSE(is_inplace(dst)); EXPECT_NE(dst.representation().data(), src.representation().data()); EXPECT_EQ(dst, src); @@ -1030,3 +1126,171 @@ consteval bool test_grow_shrink_grow() { return x.representation()[0] == 42U; } static_assert(test_grow_shrink_grow()); + +// ----- Allocator ownership across assignment ----- + +// Records which allocator id handed out each block, and counts blocks released through an allocator other than +// the one that allocated them. +struct ownership_tracker { + std::map owner; + std::size_t mismatched_frees = 0; + std::size_t live = 0; +}; + +ownership_tracker& tracker() { + static ownership_tracker t; + return t; +} + +template +struct tracking_pocca_alloc { + using value_type = T; + using propagate_on_container_copy_assignment = std::true_type; + using propagate_on_container_move_assignment = std::true_type; + + std::size_t id = 0; + + tracking_pocca_alloc() = default; + explicit tracking_pocca_alloc(std::size_t allocator_id) noexcept : id{allocator_id} {} + template + tracking_pocca_alloc(const tracking_pocca_alloc& other) noexcept : id{other.id} {} + + [[nodiscard]] T* allocate(std::size_t n) { + T* const p = std::allocator{}.allocate(n); + tracker().owner[p] = id; + ++tracker().live; + return p; + } + void deallocate(T* p, std::size_t n) noexcept { + if (tracker().owner[p] != id) { + ++tracker().mismatched_frees; + } + tracker().owner.erase(p); + --tracker().live; + std::allocator{}.deallocate(p, n); + } + + template + bool operator==(const tracking_pocca_alloc& other) const noexcept { + return id == other.id; + } +}; + +using tracking_big_int = + BEMAN_BIG_INT_NAMESPACE::basic_big_int<64, + BEMAN_BIG_INT_NAMESPACE::uint_multiprecision_t, + tracking_pocca_alloc>; + +namespace { +using tracking_id = tracking_pocca_alloc; + +tracking_big_int make_heap_value(const tracking_id alloc, const std::size_t bits) { + tracking_big_int x{1, alloc}; + x <<= bits; + return x; +} +} // namespace + +TEST(Allocation, PropagatingCopyAssignFreesLargerDstWithItsOwnAllocator) { + tracker() = {}; + { + tracking_big_int dst = make_heap_value(tracking_id{1U}, 4000); + dst.reserve_representation(200); // larger than the source, so the old fast path kept it + const tracking_big_int src = make_heap_value(tracking_id{2U}, 100); + ASSERT_FALSE(is_inplace(src)); + + dst = src; + + EXPECT_EQ(dst, src); + EXPECT_EQ(dst.get_allocator().id, 2U); + } + EXPECT_EQ(tracker().mismatched_frees, 0U); + EXPECT_EQ(tracker().live, 0U); +} + +TEST(Allocation, PropagatingCopyAssignOfInlineSrcFreesDstWithItsOwnAllocator) { + tracker() = {}; + { + tracking_big_int dst = make_heap_value(tracking_id{1U}, 4000); + const tracking_big_int src{7, tracking_id{2U}}; + + dst = src; + + EXPECT_EQ(dst, 7); + EXPECT_EQ(dst.get_allocator().id, 2U); + } + EXPECT_EQ(tracker().mismatched_frees, 0U); + EXPECT_EQ(tracker().live, 0U); +} + +TEST(Allocation, PropagatingMoveAssignFreesDstWithItsOwnAllocator) { + tracker() = {}; + { + tracking_big_int dst = make_heap_value(tracking_id{1U}, 4000); + tracking_big_int heap_src = make_heap_value(tracking_id{2U}, 100); + const auto* const src_data = heap_src.representation().data(); + + dst = std::move(heap_src); + EXPECT_EQ(dst.representation().data(), src_data); + EXPECT_EQ(dst.get_allocator().id, 2U); + + tracking_big_int inline_src{9, tracking_id{3U}}; + dst = std::move(inline_src); + EXPECT_EQ(dst, 9); + EXPECT_EQ(dst.get_allocator().id, 3U); + } + EXPECT_EQ(tracker().mismatched_frees, 0U); + EXPECT_EQ(tracker().live, 0U); +} + +TEST(Allocation, PmrMoveAssignStealsFromTheSameResource) { + std::pmr::monotonic_buffer_resource resource; + BEMAN_BIG_INT_NAMESPACE::pmr::big_int dst{1, &resource}; + dst.reserve_representation(16); + BEMAN_BIG_INT_NAMESPACE::pmr::big_int src{1, &resource}; + src <<= 200; + const auto* const src_data = src.representation().data(); + const auto expected = src; + + dst = std::move(src); + + EXPECT_EQ(dst.representation().data(), src_data); + EXPECT_EQ(dst, expected); + EXPECT_EQ(dst.get_allocator().resource(), &resource); + EXPECT_EQ(src, 0U); +} + +TEST(Allocation, PmrMoveAssignCopiesFromAnotherResourceAndKeepsItsOwn) { + std::pmr::monotonic_buffer_resource dst_resource; + std::pmr::monotonic_buffer_resource src_resource; + BEMAN_BIG_INT_NAMESPACE::pmr::big_int dst{1, &dst_resource}; + BEMAN_BIG_INT_NAMESPACE::pmr::big_int src{1, &src_resource}; + src <<= 200; + const auto* const src_data = src.representation().data(); + const auto expected = src; + + dst = std::move(src); + + EXPECT_NE(dst.representation().data(), src_data); + EXPECT_EQ(dst, expected); + EXPECT_EQ(dst.get_allocator().resource(), &dst_resource); + EXPECT_EQ(src.get_allocator().resource(), &src_resource); + EXPECT_EQ(src, expected); // not stolen, so the source keeps its value +} + +TEST(Allocation, PmrMoveAssignFromAnotherResourceReusesLargerDstStorage) { + std::pmr::monotonic_buffer_resource dst_resource; + std::pmr::monotonic_buffer_resource src_resource; + BEMAN_BIG_INT_NAMESPACE::pmr::big_int dst{1, &dst_resource}; + dst.reserve_representation(16); + const auto* const dst_data = dst.representation().data(); + BEMAN_BIG_INT_NAMESPACE::pmr::big_int src{1, &src_resource}; + src <<= 200; + const auto expected = src; + + dst = std::move(src); + + EXPECT_EQ(dst.representation().data(), dst_data); + EXPECT_EQ(dst, expected); + EXPECT_EQ(dst.get_allocator().resource(), &dst_resource); +} diff --git a/tests/beman/big_int/dispatch_contract.test.cpp b/tests/beman/big_int/dispatch_contract.test.cpp new file mode 100644 index 00000000..739c2ed1 --- /dev/null +++ b/tests/beman/big_int/dispatch_contract.test.cpp @@ -0,0 +1,509 @@ +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-License-Identifier: BSL-1.0 + +// Result-span contract of the multiply and divide dispatchers: they must produce the correct product, quotient and +// remainder into spans that are NOT pre-zeroed (callers may hand over uninitialized storage), and must leave limbs +// past the product alone. Every case runs a dispatcher into a poison-filled span and into a zeroed span, and the +// two must agree limb for limb; products are also checked against multiply_long where the work is small. + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace detail = ::BEMAN_BIG_INT_NAMESPACE::detail; + +namespace { + +using uint_t = ::BEMAN_BIG_INT_NAMESPACE::uint_multiprecision_t; +using alloc_t = std::allocator; +using limbs_t = std::vector; +using cspan_t = std::span; +using span_t = std::span; + +constexpr uint_t poison = ~uint_t{0} / 255 * 0xA5; +constexpr uint_t guard_value = ~uint_t{0} / 255 * 0x5A; +constexpr std::size_t guard_limbs = 3; +constexpr std::size_t limb_bits = std::numeric_limits::digits; +constexpr std::size_t max_ref_work = 4'000'000; +constexpr std::size_t max_limbs = 20'000; + +limbs_t random_limbs(const std::size_t n, std::mt19937_64& rng) { + limbs_t v(n); + for (auto& x : v) { + x = static_cast(rng()); + } + if (v.back() == 0) { + v.back() = 1; + } + return v; +} + +// A power of two spanning `n` limbs: the single set bit sits in the top limb at position `bit`. +limbs_t power_of_two(const std::size_t n, const unsigned bit) { + limbs_t v(n, uint_t{0}); + v.back() = uint_t{1} << (bit % limb_bits); + return v; +} + +// Calls `run(result, a, b)` into a poisoned and into a zeroed result of a.size() + b.size() limbs plus guards. +// Checks that the two agree, that guard limbs are untouched, that the returned size is the trimmed size, and +// (when affordable) that the product matches multiply_long. +template +void check_product(const char* what, const cspan_t a, const cspan_t b, Run&& run) { + const std::size_t total = a.size() + b.size(); + limbs_t dirty(total + guard_limbs, poison); + limbs_t clean(total + guard_limbs, uint_t{0}); + std::fill(dirty.begin() + static_cast(total), dirty.end(), guard_value); + std::fill(clean.begin() + static_cast(total), clean.end(), guard_value); + + const std::size_t n_dirty = run(span_t{dirty.data(), total}, a, b); + const std::size_t n_clean = run(span_t{clean.data(), total}, a, b); + + for (std::size_t i = total; i < dirty.size(); ++i) { + EXPECT_EQ(dirty[i], guard_value) + << what << ": guard limb " << (i - total) << " overwritten, la=" << a.size() << " lb=" << b.size(); + } + EXPECT_EQ(n_dirty, n_clean) << what << ": la=" << a.size() << " lb=" << b.size(); + EXPECT_EQ(n_dirty, detail::trimmed_size_span(cspan_t{dirty.data(), total})) + << what << ": returned size is not the trimmed size, la=" << a.size() << " lb=" << b.size(); + const bool same = std::equal(dirty.begin(), dirty.begin() + static_cast(total), clean.begin()); + EXPECT_TRUE(same) << what << ": poisoned result differs from zeroed result, la=" << a.size() << " lb=" << b.size(); + + if (a.size() * b.size() <= max_ref_work) { + limbs_t ref(total, uint_t{0}); + detail::multiply_long(span_t{ref}, a, b); + const bool matches = std::equal(ref.begin(), ref.end(), dirty.begin()); + EXPECT_TRUE(matches) << what << ": differs from multiply_long, la=" << a.size() << " lb=" << b.size(); + } +} + +// Each runner returns the trimmed size and takes (result, a, b); `a` and `b` are trimmed with a.size() >= 2 where +// the runner is a direct tier entry point. +std::size_t run_dispatch(const span_t r, const cspan_t a, const cspan_t b) { + alloc_t alloc; + return detail::multiply_dispatch(r, a, b, alloc); +} + +std::size_t run_square_dispatch(const span_t r, const cspan_t a, const cspan_t) { + alloc_t alloc; + return detail::square_dispatch(r, a, alloc); +} + +std::size_t run_runtime(const span_t r, const cspan_t a, const cspan_t b) { + alloc_t alloc; + const detail::scratch_allocator hooks(alloc); + return detail::multiply_runtime(r, a, b, hooks.heap()); +} + +std::size_t run_sliced(const span_t r, const cspan_t a, const cspan_t b) { + alloc_t alloc; + const detail::scratch_allocator hooks(alloc); + return detail::multiply_runtime_sliced(r, a, b, hooks.heap()); +} + +std::size_t run_unsliced(const span_t r, const cspan_t a, const cspan_t b) { + alloc_t alloc; + const detail::scratch_allocator hooks(alloc); + return detail::multiply_runtime_unsliced(r, a, b, hooks.heap()); +} + +std::size_t run_any(const span_t r, const cspan_t a, const cspan_t b) { + alloc_t alloc; + const detail::scratch_allocator hooks(alloc); + return detail::multiply_runtime_any(r, a, b, hooks.heap()); +} + +std::size_t run_square_runtime(const span_t r, const cspan_t a, const cspan_t) { + alloc_t alloc; + const detail::scratch_allocator hooks(alloc); + return detail::square_runtime(r, a, hooks.heap()); +} + +std::size_t run_basecase(const span_t r, const cspan_t a, const cspan_t b) { + return detail::multiply_basecase_runtime(r, a, b); +} + +// All entry points for one operand pair (a, b). `squaring` means b is the same span as a. +void check_all_entry_points(const cspan_t a, const cspan_t b, const bool squaring) { + check_product("multiply_dispatch", a, b, run_dispatch); + check_product("multiply_runtime_any", a, b, run_any); + if (a.size() >= 2 && b.size() >= 2) { + check_product("multiply_runtime", a, b, run_runtime); + check_product("multiply_runtime_unsliced", a, b, run_unsliced); + check_product("multiply_runtime_sliced", a, b, run_sliced); + if (std::min(a.size(), b.size()) < detail::mul_header_basecase_limbs) { + check_product("multiply_basecase_runtime", a, b, run_basecase); + } + if (squaring) { + check_product("square_dispatch", a, b, run_square_dispatch); + check_product("square_runtime", a, b, run_square_runtime); + } + } +} + +std::size_t sat_sub(const std::size_t x, const std::size_t d) { return x > d ? x - d : 1; } + +// Operand sizes straddling every cutoff and basecase boundary. +std::vector boundary_sizes() { + std::vector v = {1, 2, 3, 4, 5, 8, 16, 31, 32, 33}; + const std::size_t cutoffs[] = {detail::square_long_cutoff, + detail::karatsuba_cutoff, + detail::square_karatsuba_cutoff, + detail::toom_cook_3_cutoff, + detail::square_toom_cook_3_cutoff, + detail::toom_cook_4_cutoff, + detail::square_toom_cook_4_cutoff, + detail::toom_cook_6_5_cutoff, + detail::square_toom_cook_6_5_cutoff, + detail::toom_cook_8_5_cutoff, + detail::square_toom_cook_8_5_cutoff}; + for (const std::size_t c : cutoffs) { + v.push_back(sat_sub(c, 1)); + v.push_back(c); + v.push_back(c + 1); + } + std::ranges::sort(v); + v.erase(std::unique(v.begin(), v.end()), v.end()); + std::erase_if(v, [](const std::size_t n) { return n > max_limbs; }); + return v; +} + +TEST(DispatchContract, BalancedProductsAtEveryCutoff) { + std::mt19937_64 rng{0xd15ca7c4u}; + for (const std::size_t n : boundary_sizes()) { + const limbs_t a = random_limbs(n, rng); + const limbs_t b = random_limbs(n, rng); + check_all_entry_points(a, b, false); + if (n + 1 <= max_limbs) { + const limbs_t c = random_limbs(n + 1, rng); + check_all_entry_points(a, c, false); + check_all_entry_points(c, a, false); + } + } +} + +TEST(DispatchContract, SquaresAtEveryCutoff) { + std::mt19937_64 rng{0x5ca1ab1eu}; + for (const std::size_t n : boundary_sizes()) { + const limbs_t a = random_limbs(n, rng); + check_all_entry_points(a, a, true); + } +} + +TEST(DispatchContract, SingleLimbOperands) { + std::mt19937_64 rng{0x51a61eu}; + for (const std::size_t n : {std::size_t{1}, std::size_t{2}, std::size_t{3}, std::size_t{17}, std::size_t{200}}) { + const limbs_t a = random_limbs(n, rng); + // No carry out of the top limb: the product still has to define limb n. + const limbs_t small(1, uint_t{3}); + const limbs_t carry(1, ~uint_t{0}); + limbs_t low_top = a; + if (!low_top.empty()) { + low_top.back() = 1; + } + check_all_entry_points(low_top, small, false); + check_all_entry_points(small, low_top, false); + check_all_entry_points(a, carry, false); + check_all_entry_points(carry, a, false); + } +} + +TEST(DispatchContract, PowerOfTwoOperands) { + std::mt19937_64 rng{0xb17b17u}; + for (const std::size_t n : boundary_sizes()) { + if (n < 2) { + continue; + } + for (const unsigned bit : {0u, 1u, 17u, static_cast(limb_bits - 1)}) { + const limbs_t p = power_of_two(n, bit); + for (const std::size_t other : {std::size_t{2}, std::size_t{7}, n, n + 5}) { + if (other > max_limbs) { + continue; + } + const limbs_t a = random_limbs(other, rng); + check_all_entry_points(a, p, false); + check_all_entry_points(p, a, false); + } + check_all_entry_points(p, p, true); + const limbs_t q = power_of_two(n + 1, bit); + check_all_entry_points(p, q, false); + } + } +} + +TEST(DispatchContract, PowerOfTwoKernelWritesEveryLimb) { + std::mt19937_64 rng{0x9072u}; + for (const std::size_t pn : {std::size_t{1}, std::size_t{2}, std::size_t{5}}) { + for (const unsigned bit : {0u, 3u, static_cast(limb_bits - 1)}) { + const limbs_t p = power_of_two(pn, bit); + for (const std::size_t an : {std::size_t{1}, std::size_t{2}, std::size_t{9}}) { + limbs_t a = random_limbs(an, rng); + // Force a top limb that does not carry out for small shifts. + a.back() = 1; + check_product("multiply_power_of_two", a, p, [](const span_t r, const cspan_t x, const cspan_t y) { + return detail::multiply_power_of_two(r, x, y); + }); + } + } + } +} + +// Sliced shapes: a long operand against a short one at or above karatsuba_cutoff. +TEST(DispatchContract, SlicedUnbalancedShapes) { + std::mt19937_64 rng{0x511ceu}; + const std::size_t kc = detail::karatsuba_cutoff; + const std::pair shapes[] = {{kc, 3 * kc + 1}, + {kc, 4 * kc + 3}, + {kc + 5, 9 * kc}, + {kc + 1, 2 * kc}, + {2 * kc, 7 * kc + 2}, + {kc - 1, 6 * kc}}; + for (const auto& [s, l] : shapes) { + if (l > max_limbs) { + continue; + } + const limbs_t a = random_limbs(s, rng); + const limbs_t b = random_limbs(l, rng); + check_all_entry_points(a, b, false); + check_all_entry_points(b, a, false); + } +} + +// The first balanced size the FFT gate takes (64-bit limbs), when it is small enough to run quickly. +TEST(DispatchContract, FirstFftSize) { + if constexpr (limb_bits == 64) { + std::mt19937_64 rng{0xff7u}; + for (std::size_t n = detail::karatsuba_cutoff; n <= 6000; ++n) { + if (detail::fft_mul_worthwhile(n, n)) { + const limbs_t a = random_limbs(n, rng); + const limbs_t b = random_limbs(n, rng); + check_all_entry_points(a, b, false); + break; + } + } + for (std::size_t n = detail::karatsuba_cutoff; n <= 6000; ++n) { + if (detail::square_fft_worthwhile(n)) { + const limbs_t a = random_limbs(n, rng); + check_all_entry_points(a, a, true); + break; + } + } + } +} + +// Untrimmed operands: the high zero limbs are dropped before dispatch and limbs past the trimmed sum stay untouched. +TEST(DispatchContract, UntrimmedOperands) { + std::mt19937_64 rng{0x07ca11edu}; + for (const std::size_t n : {std::size_t{2}, std::size_t{5}, std::size_t{40}}) { + limbs_t a = random_limbs(n, rng); + limbs_t b = random_limbs(n + 1, rng); + a.resize(n + 3, uint_t{0}); + b.resize(n + 2, uint_t{0}); + const std::size_t trimmed_total = n + (n + 1); + const std::size_t total = a.size() + b.size(); + limbs_t dirty(total, poison); + limbs_t clean(total, uint_t{0}); + alloc_t alloc; + const std::size_t n_dirty = detail::multiply_dispatch(span_t{dirty}, cspan_t{a}, cspan_t{b}, alloc); + const std::size_t n_clean = detail::multiply_dispatch(span_t{clean}, cspan_t{a}, cspan_t{b}, alloc); + EXPECT_EQ(n_dirty, n_clean); + EXPECT_TRUE( + std::equal(dirty.begin(), dirty.begin() + static_cast(trimmed_total), clean.begin())); + for (std::size_t i = trimmed_total; i < total; ++i) { + EXPECT_EQ(dirty[i], poison) << "limb " << i << " past the trimmed product was written"; + } + } +} + +// --------------------------------------------------------------------------- +// Division +// --------------------------------------------------------------------------- + +// Quotient/remainder spans are fully written by every tier: poison-fill both and compare against a zeroed run. +void check_division(const std::size_t m, const std::size_t s, std::mt19937_64& rng) { + const limbs_t dividend = random_limbs(m, rng); + const limbs_t divisor = random_limbs(s, rng); + const cspan_t a{dividend}; + const cspan_t b{divisor}; + alloc_t alloc; + + const std::size_t qn = m - s + 1; + const std::size_t rn = m + 1; + + auto run_full = [&](const uint_t fill) { + limbs_t q(qn, fill); + limbs_t r(rn, fill); + detail::scratch_allocator scratch(detail::divide_schoolbook_storage_size(m, s, false), alloc); + detail::divide_dispatch(span_t{q}, span_t{r}, a, b, scratch, alloc); + return std::pair{std::move(q), std::move(r)}; + }; + auto run_q = [&](const uint_t fill) { + limbs_t q(qn, fill); + detail::scratch_allocator scratch(detail::divide_schoolbook_storage_size(m, s, true), alloc); + detail::divide_dispatch_q(span_t{q}, a, b, scratch, alloc); + return q; + }; + + const auto [qd, rd] = run_full(poison); + const auto [qc, rc] = run_full(uint_t{0}); + EXPECT_EQ(qd, qc) << "divide_dispatch quotient, m=" << m << " s=" << s; + EXPECT_EQ(rd, rc) << "divide_dispatch remainder, m=" << m << " s=" << s; + EXPECT_EQ(run_q(poison), run_q(uint_t{0})) << "divide_dispatch_q quotient, m=" << m << " s=" << s; + EXPECT_EQ(run_q(poison), qc) << "divide_dispatch_q differs from divide_dispatch, m=" << m << " s=" << s; + + // q * divisor + r == dividend, checked through the (already contract-tested) multiply_long. + limbs_t prod(qn + s, uint_t{0}); + detail::multiply_long(span_t{prod}, cspan_t{qc.data(), detail::trimmed_size_span(cspan_t{qc})}, b); + limbs_t sum(std::max(prod.size(), rd.size()) + 1, uint_t{0}); + std::copy(prod.begin(), prod.end(), sum.begin()); + const bool carry = detail::add_unsigned_spans(span_t{sum}.first(sum.size() - 1), + span_t{sum}.first(sum.size() - 1), + cspan_t{rd.data(), std::min(rd.size(), sum.size() - 1)}); + EXPECT_FALSE(carry); + for (std::size_t i = 0; i < sum.size(); ++i) { + const uint_t want = i < dividend.size() ? dividend[i] : uint_t{0}; + ASSERT_EQ(sum[i], want) << "q*d+r != dividend at limb " << i << ", m=" << m << " s=" << s; + } +} + +std::vector> division_shapes() { + std::vector> v; + const std::size_t bz = detail::burnikel_ziegler_cutoff; + const std::size_t off = detail::burnikel_ziegler_offset; + for (const std::size_t s : {std::size_t{2}, std::size_t{3}, std::size_t{4}, bz - 1, bz, bz + 1}) { + for (const std::size_t m : {s, s + 1, s + off - 1, s + off, s + off + 1, 3 * s + 1}) { + v.emplace_back(m, s); + } + } + v.emplace_back(8, 4); + v.emplace_back(9, 2); + // Barrett march gate: below, at and above the m/16 line for the compiled march cutoff. + const std::size_t c = detail::barrett_march_cutoff; + for (const std::size_t s : {c - 1, c, c + 1}) { + for (const std::size_t m : {16 * s - 1, 16 * s, 16 * s + 5}) { + if (s >= 2 && m <= max_limbs) { + v.emplace_back(m, s); + } + } + } + return v; +} + +TEST(DispatchContract, DivisionAcrossTierGates) { + std::mt19937_64 rng{0xd1f1de5u}; + bool saw_schoolbook = false; + bool saw_barrett = false; + bool saw_other = false; + for (const auto& [m, s] : division_shapes()) { + check_division(m, s, rng); + if (detail::divide_takes_schoolbook(m, s)) { + saw_schoolbook = true; + } else if (detail::divide_takes_barrett(m, s)) { + saw_barrett = true; + } else { + saw_other = true; + } + } + EXPECT_TRUE(saw_schoolbook); + EXPECT_TRUE(saw_barrett); + EXPECT_TRUE(saw_other); +} + +TEST(DispatchContract, DivideTakesSchoolbookMatchesGates) { + for (std::size_t s = 2; s < 80; ++s) { + for (const std::size_t m : {s, s + 1, s + 9, s + 10, s + 64, 4 * s, 16 * s, 16 * s + 5}) { + const bool barrett = (s >= detail::barrett_march_cutoff && m / 16 >= s) || + (s >= detail::barrett_march8_cutoff && m / 8 >= s) || + (m >= detail::barrett_quarter_cutoff && m / 4 >= s) || + (m >= detail::barrett_balanced_cutoff && m - s >= s); + const bool bz = s >= detail::burnikel_ziegler_cutoff && m - s >= detail::burnikel_ziegler_offset; + EXPECT_EQ(detail::divide_takes_schoolbook(m, s), !barrett && !bz) << "m=" << m << " s=" << s; + } + } +} + +// divide_unsigned reads the dividend only while copying it to scratch, so the quotient may start at its first limb. +TEST(DispatchContract, SchoolbookQuotientMayAliasDividend) { + std::mt19937_64 rng{0xa11a5u}; + for (const auto& [m, s] : {std::pair{8, 4}, {9, 2}, {20, 3}, {5, 5}, {33, 17}}) { + const limbs_t dividend = random_limbs(m, rng); + const limbs_t divisor = random_limbs(s, rng); + const std::size_t qn = m - s + 1; + alloc_t alloc; + + limbs_t q_ref(qn, uint_t{0}); + limbs_t r_ref(m + 1, uint_t{0}); + { + detail::scratch_allocator scratch(detail::divide_unsigned_storage_size(m, s), alloc); + detail::divide_unsigned(span_t{q_ref}, span_t{r_ref}, cspan_t{dividend}, cspan_t{divisor}, scratch); + } + + limbs_t buf = dividend; + limbs_t r(m + 1, poison); + { + detail::scratch_allocator scratch(detail::divide_unsigned_storage_size(m, s), alloc); + detail::divide_unsigned( + span_t{buf.data(), qn}, span_t{r}, cspan_t{buf.data(), m}, cspan_t{divisor}, scratch); + } + EXPECT_TRUE(std::equal(q_ref.begin(), q_ref.end(), buf.begin())) << "m=" << m << " s=" << s; + EXPECT_EQ(r, r_ref) << "m=" << m << " s=" << s; + } +} + +// --------------------------------------------------------------------------- +// Constant evaluation: the dispatcher must not rely on a pre-zeroed result there either. +// --------------------------------------------------------------------------- + +constexpr bool constexpr_product_ok(const std::span a, const std::span b) { + std::array dirty{}; + std::array ref{}; + for (auto& x : dirty) { + x = ~uint_t{0}; + } + std::allocator alloc; + const std::size_t total = a.size() + b.size(); + const std::size_t n = detail::multiply_dispatch(std::span{dirty}, a, b, alloc); + detail::multiply_long(std::span{ref}, a, b); + for (std::size_t i = 0; i < total; ++i) { + if (dirty[i] != ref[i]) { + return false; + } + } + return n == detail::trimmed_size_span(std::span{ref.data(), total}); +} + +constexpr bool constexpr_cases() { + constexpr uint_t max = ~uint_t{0}; + const std::array one{7}; + const std::array three{3}; + const std::array wide{1, 2, 4}; + const std::array full{max, max, max}; + const std::array p2{0, 0, 4}; + const std::array p2_unit{0, 0, 1}; + const std::array pair{5, 6}; + const std::array sq{7, 9, 11}; + const std::array top{max}; + return constexpr_product_ok(one, three) && constexpr_product_ok(wide, three) && constexpr_product_ok(full, top) && + constexpr_product_ok(top, full) && constexpr_product_ok(wide, pair) && constexpr_product_ok(p2, pair) && + constexpr_product_ok(pair, p2) && constexpr_product_ok(p2_unit, wide) && constexpr_product_ok(p2, p2) && + constexpr_product_ok(full, full) && constexpr_product_ok(sq, sq); +} + +static_assert(constexpr_cases()); + +} // namespace diff --git a/tests/beman/big_int/front_end_add_sub.test.cpp b/tests/beman/big_int/front_end_add_sub.test.cpp new file mode 100644 index 00000000..ca765286 --- /dev/null +++ b/tests/beman/big_int/front_end_add_sub.test.cpp @@ -0,0 +1,351 @@ +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-License-Identifier: BSL-1.0 + +// Differential tests for the single-pass add/sub front ends (add_in_place, add_into) against a plain +// limb-vector reference, for big_int, basic_big_int<128> and basic_big_int<256>. + +#include +#include +#include +#include +#include + +#include +#include + +#include "front_end_reference.hpp" +#include "testing.hpp" + +namespace bb = BEMAN_BIG_INT_NAMESPACE; + +BEMAN_BIG_INT_BEGIN_NAMESPACE +namespace front_end_test { + +template +void check_add_sub(const ref& a, const ref& b) { + const T ta = make(a); + const T tb = make(b); + const ref sum = ref_add(a, b); + const ref diff = ref_add(a, ref_neg(b)); + const auto describe = [&] { return "a=" + hex(a) + " b=" + hex(b); }; + SCOPED_TRACE(describe()); + + // copy_copy (add_into) + EXPECT_TRUE(matches(ta + tb, sum)); + EXPECT_TRUE(matches(ta - tb, diff)); + EXPECT_TRUE(matches(ta, a)) << "operand modified"; + EXPECT_TRUE(matches(tb, b)) << "operand modified"; + + // rvalue forms (storage reuse) + EXPECT_TRUE(matches(T(ta) + tb, sum)); + EXPECT_TRUE(matches(ta + T(tb), sum)); + EXPECT_TRUE(matches(T(ta) + T(tb), sum)); + EXPECT_TRUE(matches(T(ta) - tb, diff)); + EXPECT_TRUE(matches(ta - T(tb), diff)); + EXPECT_TRUE(matches(T(ta) - T(tb), diff)); + + // compound forms + T c = ta; + c += tb; + EXPECT_TRUE(matches(c, sum)); + T d = ta; + d -= tb; + EXPECT_TRUE(matches(d, diff)); + c -= tb; + EXPECT_TRUE(matches(c, a)) << "(a + b) - b"; + d += tb; + EXPECT_TRUE(matches(d, a)) << "(a - b) + b"; + + // A fresh result from operands that fit inline lands inline exactly when the result fits. + if (std::max(a.mag.size(), b.mag.size()) <= T::inplace_capacity) { + const T r = ta + tb; + EXPECT_EQ(is_inplace(r), sum.mag.size() <= T::inplace_capacity); + const T r2 = ta - tb; + EXPECT_EQ(is_inplace(r2), diff.mag.size() <= T::inplace_capacity); + } +} + +template +void check_integer_forms(const ref& a, long long v, unsigned long long u) { + const T ta = make(a); + const ref rv = + from_u64(v < 0 ? 0 - static_cast(v) : static_cast(v), v < 0); + const ref ru = from_u64(u, false); + SCOPED_TRACE("a=" + hex(a) + " v=" + std::to_string(v)); + EXPECT_TRUE(matches(ta + v, ref_add(a, rv))); + EXPECT_TRUE(matches(v + ta, ref_add(rv, a))); + EXPECT_TRUE(matches(ta - v, ref_add(a, ref_neg(rv)))); + EXPECT_TRUE(matches(v - ta, ref_add(rv, ref_neg(a)))); + EXPECT_TRUE(matches(T(ta) + v, ref_add(a, rv))); + EXPECT_TRUE(matches(v - T(ta), ref_add(rv, ref_neg(a)))); + EXPECT_TRUE(matches(ta + u, ref_add(a, ru))); + EXPECT_TRUE(matches(u - ta, ref_add(ru, ref_neg(a)))); + T c = ta; + c += v; + EXPECT_TRUE(matches(c, ref_add(a, rv))); + c = ta; + c -= u; + EXPECT_TRUE(matches(c, ref_add(a, ref_neg(ru)))); +} + +template +void add_sub_sweep() { + std::mt19937_64 rng(12345); + const auto sizes = sizes_for(T::inplace_capacity); + for (const std::size_t na : sizes) { + for (const std::size_t nb : sizes) { + for (unsigned pa = 0; pa < 6; ++pa) { + for (unsigned pb = 0; pb < 6; ++pb) { + for (unsigned signs = 0; signs < 4; ++signs) { + // Subsample the pattern product for the larger sizes to keep the run short. + if (na + nb > 14 && (pa + pb + signs) % 3 != 0) { + continue; + } + check_add_sub(gen(rng, na, pa, (signs & 1U) != 0), gen(rng, nb, pb, (signs & 2U) != 0)); + } + } + } + } + } +} + +template +void add_sub_ripple() { + // a = 2^(kW) - 1 style operands against +-1 and +-2^(jW): the carry or borrow ripples through every + // limb, including exactly at and one past the inline capacity. + std::mt19937_64 rng(777); + for (std::size_t k = 1; k <= 24; ++k) { + const ref ones = gen(rng, k, 1, false); + const ref pow = gen(rng, k + 1, 2, false); + for (const bool n1 : {false, true}) { + for (const bool n2 : {false, true}) { + ref a = ones; + ref b = ref{n2, {1}}; + a.neg = n1; + check_add_sub(a, b); + check_add_sub(b, a); + ref c = pow; + c.neg = n2; + check_add_sub(a, c); + check_add_sub(c, a); + check_add_sub(c, ref{n1, {1}}); + } + } + } +} + +template +void integer_sweep() { + std::mt19937_64 rng(2468); + const long long vs[] = { + 0, 1, -1, 5, -5, std::numeric_limits::max(), std::numeric_limits::min()}; + for (const std::size_t n : sizes_for(T::inplace_capacity)) { + for (unsigned p = 0; p < 6; ++p) { + for (const bool neg : {false, true}) { + for (const long long v : vs) { + check_integer_forms(gen(rng, n, p, neg), v, static_cast(v)); + } + } + } + } +} + +// A two-limb integer operand whose high limb is zero is not canonical; the front ends must trim it. +template +void wide_integer_operands() { +#ifdef BEMAN_BIG_INT_HAS_INT128_FUNDAMENTAL + if constexpr (W != 64) { + return; // the split below assumes 64-bit limbs + } + std::mt19937_64 rng(4242); + for (const std::size_t n : {0U, 1U, 2U, 3U, 5U}) { + for (const bool neg : {false, true}) { + const ref a = gen(rng, n, 0, neg); + const T ta = make(a); + for (const detail::int128_t v : {static_cast(0), + static_cast(5), + static_cast(-5), + (static_cast(1) << 70) + 3, + -((static_cast(1) << 90) + 1)}) { + const detail::uint128_t m = + v < 0 ? 0 - static_cast(v) : static_cast(v); + ref rv; + rv.neg = v < 0; + rv.mag = {to_limb(m), to_limb(m >> 64)}; + trim(rv.mag); + SCOPED_TRACE("a=" + hex(a)); + EXPECT_TRUE(matches(ta + v, ref_add(a, rv))); + EXPECT_TRUE(matches(v + ta, ref_add(rv, a))); + EXPECT_TRUE(matches(ta - v, ref_add(a, ref_neg(rv)))); + EXPECT_TRUE(matches(v - ta, ref_add(rv, ref_neg(a)))); + EXPECT_TRUE(matches(T(ta) + v, ref_add(a, rv))); + EXPECT_TRUE(matches(v - T(ta), ref_add(rv, ref_neg(a)))); + T c = ta; + c += v; + EXPECT_TRUE(matches(c, ref_add(a, rv))); + c = ta; + c -= v; + EXPECT_TRUE(matches(c, ref_add(a, ref_neg(rv)))); + } + } + } +#endif +} + +// Every sign combination of one-limb operands at the edges of the limb range: carry out of the single limb, +// borrow, cancellation to zero, and results that change the limb count. +template +void single_limb_sweep() { + const limb ones = std::numeric_limits::max(); + const limb half = limb{1} << (W - 1); + const limb mags[] = {0, 1, 2, 3, half - 1, half, half + 1, ones - 1, ones}; + for (const limb ma : mags) { + for (const limb mb : mags) { + for (unsigned signs = 0; signs < 4; ++signs) { + ref a; + ref b; + if (ma != 0) { + a.mag = {ma}; + a.neg = (signs & 1U) != 0; + } + if (mb != 0) { + b.mag = {mb}; + b.neg = (signs & 2U) != 0; + } + check_add_sub(a, b); + } + } + } +} + +template +void aliasing() { + std::mt19937_64 rng(97531); + for (const std::size_t n : sizes_for(T::inplace_capacity)) { + for (unsigned p = 0; p < 6; ++p) { + for (const bool neg : {false, true}) { + const ref a = gen(rng, n, p, neg); + SCOPED_TRACE("a=" + hex(a)); + T x = make(a); + x += x; + EXPECT_TRUE(matches(x, ref_add(a, a))) << "x += x"; + x = make(a); + x -= x; + EXPECT_TRUE(matches(x, ref{})) << "x -= x"; + x = make(a); + x += -x; + EXPECT_TRUE(matches(x, ref{})) << "x += -x"; + x = make(a); + x -= -x; + EXPECT_TRUE(matches(x, ref_add(a, a))) << "x -= -x"; + x = make(a); + x = x + x; + EXPECT_TRUE(matches(x, ref_add(a, a))) << "x = x + x"; + x = make(a); + x = x - x; + EXPECT_TRUE(matches(x, ref{})) << "x = x - x"; + } + } + } +} + +// ----- constant evaluation: multi-limb values (heap storage under consteval) ----- + +template +constexpr bool cx_add_sub(unsigned k) { + const T x = cx_ones(k); + const T y = cx_ones(k + 1); + const unsigned kw = k * W; + + // carry ripples through every limb + if (!cx_pow2(x + 1, kw) || !cx_pow2(1 + x, kw) || !cx_pow2(T(x) + T{1}, kw)) { + return false; + } + if (((x + 1) - 1) != x || (x - (x - 1)) != 1 || ((x + x) != (x << 1))) { + return false; + } + // borrow ripples, and the operand-order / sign-flip paths + const T p = x + 1; + if (!(p - 1 == x) || !(1 - p == -x) || !(x - p == -1) || !(-x - 1 == -p) || !(-x + p == 1)) { + return false; + } + if (!(y - x == (cx_ones(1) << kw)) || !(x - y == -(cx_ones(1) << kw)) || !(y + -x == y - x)) { + return false; + } + // aliasing + T a = x; + a += a; + if (a != (x << 1)) { + return false; + } + a -= a; + if (a != 0 || !tail_is_zero(a)) { + return false; + } + a = x; + a += -a; + if (a != 0) { + return false; + } + a = x; + a = a + a; + if (a != (x << 1)) { + return false; + } + // compound on a growing value + T g{1}; + for (unsigned i = 0; i < k; ++i) { + g += g; + } + g -= 1; + g += g + 1; // 2 * 2^k - 1 + return g == (T{1} << (k + 1)) - 1; +} + +template +constexpr bool cx_single_limb() { + const limb ones = std::numeric_limits::max(); + const T m{ones}; + T two_w{1}; + two_w <<= W; + T c = m; + c += m; // carry out of the single limb, in place + T d = m; + d -= 5; + T e = -m; + e -= m; + return m + 1 == two_w && m + m == c && 1 + m == two_w && (T{0} - m) == -m && (m - m) == 0 && + (m - m).representation().size() == 1 && c - m == m && d + 5 == m && -m + m == 0 && e == -c && + -m - 1 == -two_w; +} + +} // namespace front_end_test +BEMAN_BIG_INT_END_NAMESPACE + +namespace fe = BEMAN_BIG_INT_NAMESPACE::front_end_test; + +#define FRONT_END_TYPES(CASE) \ + CASE(BigInt, bb::big_int) \ + CASE(Inline128, bb::basic_big_int<128>) \ + CASE(Inline256, bb::basic_big_int<256>) + +#define FRONT_END_TEST(SUFFIX, TYPE) \ + TEST(FrontEndAddSub##SUFFIX, Sweep) { fe::add_sub_sweep(); } \ + TEST(FrontEndAddSub##SUFFIX, SingleLimb) { fe::single_limb_sweep(); } \ + TEST(FrontEndAddSub##SUFFIX, Ripple) { fe::add_sub_ripple(); } \ + TEST(FrontEndAddSub##SUFFIX, IntegerOperands) { fe::integer_sweep(); } \ + TEST(FrontEndAddSub##SUFFIX, WideIntegerOperands) { fe::wide_integer_operands(); } \ + TEST(FrontEndAddSub##SUFFIX, Aliasing) { fe::aliasing(); } +FRONT_END_TYPES(FRONT_END_TEST) + +#define FRONT_END_STATIC(TYPE) \ + static_assert(fe::cx_add_sub(1)); \ + static_assert(fe::cx_add_sub(2)); \ + static_assert(fe::cx_add_sub(TYPE::inplace_capacity)); \ + static_assert(fe::cx_add_sub(TYPE::inplace_capacity + 1)); \ + static_assert(fe::cx_add_sub(TYPE::inplace_capacity + 3)) +FRONT_END_STATIC(bb::big_int); +FRONT_END_STATIC(bb::basic_big_int<128>); +FRONT_END_STATIC(bb::basic_big_int<256>); +static_assert(fe::cx_single_limb()); +static_assert(fe::cx_single_limb>()); diff --git a/tests/beman/big_int/front_end_reference.hpp b/tests/beman/big_int/front_end_reference.hpp new file mode 100644 index 00000000..995c545f --- /dev/null +++ b/tests/beman/big_int/front_end_reference.hpp @@ -0,0 +1,313 @@ +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-License-Identifier: BSL-1.0 + +// Shared reference model for the front_end_add_sub and front_end_shift tests: a sign plus trimmed +// limb-vector magnitude with naive add/sub/shift, and helpers to build operands and compare results. + +#ifndef BEMAN_BIG_INT_TESTS_FRONT_END_REFERENCE_HPP +#define BEMAN_BIG_INT_TESTS_FRONT_END_REFERENCE_HPP + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "testing.hpp" + +BEMAN_BIG_INT_BEGIN_NAMESPACE +namespace front_end_test { + +using limb = uint_multiprecision_t; +// Converts without a cast when the types already agree (GCC's -Wuseless-cast), and with one otherwise. +template +constexpr limb to_limb(const U v) noexcept { + if constexpr (std::is_same_v) { + return v; + } else { + return static_cast(v); + } +} + +inline constexpr unsigned W = static_cast(std::numeric_limits::digits); + +// Sign and trimmed little-endian magnitude (empty for zero). +BEMAN_BIG_INT_DIAGNOSTIC_PUSH() +BEMAN_BIG_INT_DIAGNOSTIC_IGNORED_GCC("-Wpadded") +struct ref { + bool neg = false; + std::vector mag; +}; +BEMAN_BIG_INT_DIAGNOSTIC_POP() + +inline void trim(std::vector& v); + +// The magnitude of a 64-bit value split into limbs of any width. +inline ref from_u64(const unsigned long long m, const bool neg) { + ref r; + for (unsigned sh = 0; sh < 64; sh += W) { + r.mag.push_back(to_limb(m >> sh)); + } + trim(r.mag); + r.neg = neg && !r.mag.empty(); + return r; +} + +inline void trim(std::vector& v) { + while (!v.empty() && v.back() == 0) { + v.pop_back(); + } +} + +inline int cmp_mag(const std::vector& a, const std::vector& b) { + if (a.size() != b.size()) { + return a.size() < b.size() ? -1 : 1; + } + for (std::size_t i = a.size(); i-- > 0;) { + if (a[i] != b[i]) { + return a[i] < b[i] ? -1 : 1; + } + } + return 0; +} + +inline std::vector add_mag(const std::vector& a, const std::vector& b) { + std::vector r; + limb carry = 0; + for (std::size_t i = 0; i < std::max(a.size(), b.size()); ++i) { + const limb x = i < a.size() ? a[i] : 0; + const limb y = i < b.size() ? b[i] : 0; + const limb s = x + y; + const limb t = s + carry; + carry = static_cast(s < x) + static_cast(t < s); + r.push_back(t); + } + if (carry != 0) { + r.push_back(carry); + } + trim(r); + return r; +} + +// Requires a >= b. +inline std::vector sub_mag(const std::vector& a, const std::vector& b) { + std::vector r; + limb borrow = 0; + for (std::size_t i = 0; i < a.size(); ++i) { + const limb y = i < b.size() ? b[i] : 0; + const limb d = a[i] - y; + const limb e = d - borrow; + borrow = static_cast(a[i] < y) + static_cast(d < borrow); + r.push_back(e); + } + trim(r); + return r; +} + +inline ref ref_add(const ref& a, const ref& b) { + if (a.neg == b.neg) { + ref r{a.neg, add_mag(a.mag, b.mag)}; + r.neg = r.neg && !r.mag.empty(); + return r; + } + const int c = cmp_mag(a.mag, b.mag); + if (c == 0) { + return {}; + } + return c > 0 ? ref{a.neg, sub_mag(a.mag, b.mag)} : ref{b.neg, sub_mag(b.mag, a.mag)}; +} + +inline ref ref_neg(ref a) { + a.neg = !a.neg && !a.mag.empty(); + return a; +} + +inline bool bit_at(const std::vector& v, std::size_t i) { + const std::size_t l = i / W; + return l < v.size() && ((v[l] >> (i % W)) & 1U) != 0; +} + +inline ref ref_shl(const ref& a, std::size_t s) { + if (a.mag.empty()) { + return {}; + } + ref r; + r.neg = a.neg; + r.mag.assign(a.mag.size() + s / W + 2, 0); + for (std::size_t i = 0; i < a.mag.size() * W; ++i) { + if (bit_at(a.mag, i)) { + const std::size_t j = i + s; + r.mag[j / W] |= limb{1} << (j % W); + } + } + trim(r.mag); + return r; +} + +// Floor division by 2^s. +inline ref ref_shr(const ref& a, std::size_t s) { + ref r; + r.neg = a.neg; + r.mag.assign(a.mag.size(), 0); + bool inexact = false; + for (std::size_t i = 0; i < a.mag.size() * W; ++i) { + if (!bit_at(a.mag, i)) { + continue; + } + if (i < s) { + inexact = true; + } else { + const std::size_t j = i - s; + r.mag[j / W] |= limb{1} << (j % W); + } + } + trim(r.mag); + if (a.neg && inexact) { + r.mag = add_mag(r.mag, {1}); + } + r.neg = a.neg && !r.mag.empty(); + return r; +} + +inline std::string hex(const ref& r) { + if (r.mag.empty()) { + return "0"; + } + static constexpr char digits[] = "0123456789abcdef"; + std::string s; + for (std::size_t i = r.mag.size(); i-- > 0;) { + for (unsigned d = W / 4; d-- > 0;) { + const char c = digits[(r.mag[i] >> (4 * d)) & 0xFU]; + if (!s.empty() || c != '0') { + s.push_back(c); + } + } + } + return (r.neg ? "-" : "") + s; +} + +template +T make(const ref& r) { + T x; + const auto s = hex(r); + const auto res = from_chars(s.data(), s.data() + s.size(), x, 16); + EXPECT_TRUE(res.ec == std::errc{}); + return x; +} + +template +constexpr bool tail_is_zero(const T& x) { + if (!is_inplace(x)) { + return true; + } + const auto* const limbs = x.representation().data(); + for (std::size_t i = x.representation().size(); i < T::inplace_capacity; ++i) { + if (limbs[i] != 0) { + return false; + } + } + return true; +} + +template +::testing::AssertionResult matches(const T& x, const ref& expect) { + const auto rep = x.representation(); + std::vector got(rep.begin(), rep.end()); + trim(got); + const bool neg = x < 0; + if (got != expect.mag || neg != expect.neg) { + return ::testing::AssertionFailure() + << "got " << (neg ? "-" : "") << "[" << got.size() << " limbs] expected " << hex(expect); + } + if (!is_normalized(x)) { + return ::testing::AssertionFailure() << "not normalized, expected " << hex(expect); + } + if (!tail_is_zero(x)) { + return ::testing::AssertionFailure() << "inline tail not zero, expected " << hex(expect); + } + return ::testing::AssertionSuccess(); +} + +// Operand generator: a magnitude of `n` limbs (0 means zero) in one of several carry-stressing shapes. +inline ref gen(std::mt19937_64& rng, std::size_t n, unsigned pattern, bool neg) { + ref r; + if (n == 0) { + return r; + } + r.mag.assign(n, 0); + const limb ones = std::numeric_limits::max(); + switch (pattern % 6) { + case 0: // random + for (auto& l : r.mag) { + l = to_limb(rng()); + } + break; + case 1: // all ones + std::fill(r.mag.begin(), r.mag.end(), ones); + break; + case 2: // 2^(W(n-1)): zeros below a single top bit + r.mag.back() = 1; + break; + case 3: // top limb 1, ones below + std::fill(r.mag.begin(), r.mag.end(), ones); + r.mag.back() = 1; + break; + case 4: // random with zero low limbs + for (std::size_t i = n / 2; i < n; ++i) { + r.mag[i] = to_limb(rng()); + } + break; + default: // single high bit set + r.mag.back() = limb{1} << (W - 1); + break; + } + if (r.mag.back() == 0) { + r.mag.back() = 1; + } + r.neg = neg; + return r; +} + +inline std::vector sizes_for(std::size_t cap) { + std::vector v; + for (std::size_t i = 0; i <= 20; ++i) { + v.push_back(i); + } + for (const std::size_t extra : {cap - 1, cap, cap + 1, cap + 2}) { + if (extra > 20) { + v.push_back(extra); + } + } + return v; +} + +template +constexpr T cx_ones(unsigned k) { + char digits[2048] = {}; + const unsigned n = k * W / 4; + for (unsigned i = 0; i < n; ++i) { + digits[i] = 'f'; + } + T x; + [[maybe_unused]] const auto r = from_chars(digits, digits + n, x, 16); + return x; +} + +template +constexpr bool cx_pow2(const T& x, unsigned e) { // x == 2^e + T p{1}; + p <<= e; + return x == p; +} + +} // namespace front_end_test +BEMAN_BIG_INT_END_NAMESPACE + +#endif // BEMAN_BIG_INT_TESTS_FRONT_END_REFERENCE_HPP diff --git a/tests/beman/big_int/front_end_shift.test.cpp b/tests/beman/big_int/front_end_shift.test.cpp new file mode 100644 index 00000000..bbc9baaa --- /dev/null +++ b/tests/beman/big_int/front_end_shift.test.cpp @@ -0,0 +1,240 @@ +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-License-Identifier: BSL-1.0 + +// Differential tests for the single-pass shift front ends (shift_left / shift_right in place and the +// assign_shifted_* fresh-result paths) against a plain limb-vector reference, for big_int, +// basic_big_int<128> and basic_big_int<256>. + +#include +#include +#include +#include +#include + +#include +#include + +#include "front_end_reference.hpp" +#include "testing.hpp" + +namespace bb = BEMAN_BIG_INT_NAMESPACE; + +BEMAN_BIG_INT_BEGIN_NAMESPACE +namespace front_end_test { + +template +void check_shifts(const ref& a, const std::size_t s) { + const T ta = make(a); + const ref shl = ref_shl(a, s); + const ref shr = ref_shr(a, s); + SCOPED_TRACE("a=" + hex(a) + " s=" + std::to_string(s)); + + EXPECT_TRUE(matches(ta << s, shl)); + EXPECT_TRUE(matches(ta >> s, shr)); + EXPECT_TRUE(matches(ta, a)) << "operand modified"; + EXPECT_TRUE(matches(T(ta) << s, shl)); + EXPECT_TRUE(matches(T(ta) >> s, shr)); + T c = ta; + c <<= s; + EXPECT_TRUE(matches(c, shl)); + T d = ta; + d >>= s; + EXPECT_TRUE(matches(d, shr)); + + // A fresh result lands inline exactly when it fits, even from a heap source. + const T l = ta << s; + EXPECT_EQ(is_inplace(l), shl.mag.size() <= T::inplace_capacity); + const T r = ta >> s; + EXPECT_EQ(is_inplace(r), shr.mag.size() <= T::inplace_capacity); + // An inline value shifted in place stays inline when the result fits. + if (is_inplace(ta) && shl.mag.size() <= T::inplace_capacity) { + EXPECT_TRUE(is_inplace(c)); + } +} + +template +void shift_sweep() { + std::mt19937_64 rng(1357); + const std::size_t cap = T::inplace_capacity; + for (const std::size_t n : sizes_for(cap)) { + if (n > 8 && n != cap && n != cap + 1) { + continue; + } + for (unsigned p = 0; p < 6; ++p) { + for (const bool neg : {false, true}) { + const ref a = gen(rng, n, p, neg); + std::vector shifts = {0, + 1, + W - 1, + W, + W + 1, + 2 * W, + 2 * W + 13, + 3 * W - 1, + n * W - 1, + n * W, + n * W + 1, + n * W + 5, + cap * W, + cap * W + 13, + (cap + 1) * W, + (cap + 2) * W + 1}; + for (const std::size_t s : shifts) { + if (s > 100000) { + continue; // n == 0 underflow of n * W - 1 + } + check_shifts(a, s); + } + } + } + } +} + +// Every small value against every small shift, checked against the builtin (arithmetic shift = floor). +template +void small_exhaustive() { + for (long long x = -70; x <= 70; ++x) { + for (unsigned s = 0; s <= 130; ++s) { + const T tx{x}; + long long expect_r = x >> std::min(s, 62U); + EXPECT_EQ(tx >> s, expect_r) << x << " >> " << s; + T c = tx; + c >>= s; + EXPECT_EQ(c, expect_r) << x << " >>= " << s; + EXPECT_TRUE(tail_is_zero(c)); + if (s <= 40) { + EXPECT_EQ(tx << s, x * (1LL << s)) << x << " << " << s; + T e = tx; + e <<= s; + EXPECT_EQ(e, x * (1LL << s)) << x << " <<= " << s; + EXPECT_TRUE(tail_is_zero(e)); + } + } + } +} + +// Shifting a value that exactly fits the inline capacity must not spill unless the result needs it. +template +void left_shift_exact_sizing() { + constexpr std::size_t cap_bits = T::inplace_capacity * W; + for (const bool neg : {false, true}) { + // 1 << (cap_bits - 1) is the top bit of the inline buffer. + T one{neg ? -1 : 1}; + T top = one << (cap_bits - 1); + EXPECT_TRUE(is_inplace(top)) << "copy form spilled"; + T inplace = one; + inplace <<= (cap_bits - 1); + EXPECT_TRUE(is_inplace(inplace)) << "in-place form spilled"; + EXPECT_EQ(top, inplace); + T next = one << cap_bits; + EXPECT_FALSE(is_inplace(next)); + T moved = std::move(one) << (cap_bits - 1); + EXPECT_TRUE(is_inplace(moved)) << "rvalue form spilled"; + EXPECT_TRUE(tail_is_zero(top) && tail_is_zero(inplace) && tail_is_zero(moved)); + } +} + +// Aliasing and identity shifts: the operand and the destination are the same object. +template +void shift_aliasing() { + std::mt19937_64 rng(97531); + for (const std::size_t n : sizes_for(T::inplace_capacity)) { + for (unsigned p = 0; p < 6; ++p) { + for (const bool neg : {false, true}) { + const ref a = gen(rng, n, p, neg); + SCOPED_TRACE("a=" + hex(a)); + T x = make(a); + x = x << 0; + EXPECT_TRUE(matches(x, a)) << "x = x << 0"; + x = make(a); + x = x >> 0; + EXPECT_TRUE(matches(x, a)) << "x = x >> 0"; + x = make(a); + x <<= 0; + EXPECT_TRUE(matches(x, a)) << "x <<= 0"; + x = make(a); + x <<= W; + EXPECT_TRUE(matches(x, ref_shl(a, W))) << "x <<= W"; + x = make(a); + x >>= W; + EXPECT_TRUE(matches(x, ref_shr(a, W))) << "x >>= W"; + x = make(a); + x = x << W; + EXPECT_TRUE(matches(x, ref_shl(a, W))) << "x = x << W"; + x = make(a); + x = x >> W; + EXPECT_TRUE(matches(x, ref_shr(a, W))) << "x = x >> W"; + } + } + } +} + +// ----- constant evaluation: multi-limb values (heap storage under consteval) ----- + +// 2^(k * W) - 1, parsed from hex so no operation under test builds it. +// ----- constant evaluation: multi-limb values (heap storage under consteval) ----- + +template +constexpr bool cx_shift(unsigned k) { + const T x = cx_ones(k); + const unsigned kw = k * W; + for (const unsigned s : {1U, W - 1, W, W + 1, 2 * W + 13}) { + if (((x << s) >> s) != x || !((x << s) == (x * (T{1} << s)))) { + return false; + } + T c = x; + c <<= s; + const T cc = c; + c >>= s; + if (c != x || !tail_is_zero(c) || (cc >> s) != x) { + return false; + } + // negative floor rounding: low bits of an all-ones value are always discarded when s < kw + if (s < kw) { + const T q = x >> s; + if (((-x) >> s) != -(q + 1) || !tail_is_zero(-x >> s)) { + return false; + } + T n = -x; + n >>= s; + if (n != -(q + 1)) { + return false; + } + } + } + if ((x >> kw) != 0 || ((-x) >> kw) != -1 || ((-x) >> (kw + 5)) != -1) { + return false; + } + T z = x; + z >>= kw; + T nz = -x; + nz >>= kw; + return z == 0 && tail_is_zero(z) && nz == -1 && tail_is_zero(nz); +} + +} // namespace front_end_test +BEMAN_BIG_INT_END_NAMESPACE + +namespace fe = BEMAN_BIG_INT_NAMESPACE::front_end_test; + +#define FRONT_END_TYPES(CASE) \ + CASE(BigInt, bb::big_int) \ + CASE(Inline128, bb::basic_big_int<128>) \ + CASE(Inline256, bb::basic_big_int<256>) + +#define FRONT_END_TEST(SUFFIX, TYPE) \ + TEST(FrontEndShift##SUFFIX, Sweep) { fe::shift_sweep(); } \ + TEST(FrontEndShift##SUFFIX, Aliasing) { fe::shift_aliasing(); } \ + TEST(FrontEndShift##SUFFIX, SmallExhaustive) { fe::small_exhaustive(); } \ + TEST(FrontEndShift##SUFFIX, LeftShiftExactSizing) { fe::left_shift_exact_sizing(); } +FRONT_END_TYPES(FRONT_END_TEST) + +#define FRONT_END_STATIC(TYPE) \ + static_assert(fe::cx_shift(1)); \ + static_assert(fe::cx_shift(2)); \ + static_assert(fe::cx_shift(TYPE::inplace_capacity)); \ + static_assert(fe::cx_shift(TYPE::inplace_capacity + 1)); \ + static_assert(fe::cx_shift(TYPE::inplace_capacity + 3)) +FRONT_END_STATIC(bb::big_int); +FRONT_END_STATIC(bb::basic_big_int<128>); +FRONT_END_STATIC(bb::basic_big_int<256>); diff --git a/tests/beman/big_int/inline_tail.test.cpp b/tests/beman/big_int/inline_tail.test.cpp new file mode 100644 index 00000000..304270b4 --- /dev/null +++ b/tests/beman/big_int/inline_tail.test.cpp @@ -0,0 +1,252 @@ +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-License-Identifier: BSL-1.0 + +// Regression tests for the inline-tail invariant: for an in-place value, the limbs in +// [limb_count, inplace_capacity) must be zero. They only matter when inplace_capacity > 1, +// hence basic_big_int<128> and basic_big_int<256>. Reused heap destinations of base 8/32 +// from_chars are covered as well. + +#include +#include +#include +#include +#include + +#include +#include + +#include "testing.hpp" + +namespace bb = BEMAN_BIG_INT_NAMESPACE; + +BEMAN_BIG_INT_BEGIN_NAMESPACE +namespace inline_tail_test { + +inline constexpr unsigned limb_width = static_cast(std::numeric_limits::digits); + +// True when every limb of an in-place value past its limb count is zero. +template +constexpr bool tail_is_zero(const T& x) { + if (!is_inplace(x)) { + return true; + } + const auto* const limbs = x.representation().data(); + for (std::size_t i = x.representation().size(); i < T::inplace_capacity; ++i) { + if (limbs[i] != 0) { + return false; + } + } + return true; +} + +// 2^(k * limb_width) - 1, i.e. k limbs of all ones. Parsed from hex digits: shifting up to the full +// inline capacity can reserve a spare limb and spill to the heap. +template +constexpr T all_ones(unsigned k) { + char digits[1024] = {}; + for (unsigned i = 0; i < k * limb_width / 4; ++i) { + digits[i] = 'f'; + } + T x; + const auto n = k * limb_width / 4; + [[maybe_unused]] const auto r = from_chars(digits, digits + n, x, 16); + return x; +} + +template +constexpr T two_limbs() { // 2^limb_width + 7 + T x{1}; + x <<= limb_width; + x += 7; + return x; +} + +template +constexpr bool sub_self() { + T x = all_ones(2); + x -= x; + return x == 0U && tail_is_zero(x); +} + +template +constexpr bool xor_self() { + T x = all_ones(2); + x ^= x; + return x == 0U && tail_is_zero(x); +} + +// Shift all_ones(k) (k = whole inline capacity) right by `limbs` whole limbs plus `extra` bits. +template +constexpr bool shr_case(unsigned limbs, unsigned extra, bool negative) { + constexpr unsigned k = static_cast(T::inplace_capacity); + T x = all_ones(k); + if (negative) { + x = -x; + } + const unsigned s = limbs * limb_width + extra; + x >>= s; + const unsigned remaining = k * limb_width - s; + T expected{1}; + expected <<= remaining; + if (negative) { + expected = -expected; // floor(-(2^n - 1) / 2^s) == -2^(n - s) + } else { + expected -= 1; + } + return x == expected && tail_is_zero(x); +} + +template +constexpr bool shr_all() { + constexpr unsigned k = static_cast(T::inplace_capacity); + for (const bool neg : {false, true}) { + for (const unsigned extra : {0U, 3U}) { + for (const unsigned limbs : {1U, 3U, k - 1U}) { + if (limbs >= k || (limbs * limb_width + extra) >= k * limb_width) { + continue; + } + if (!shr_case(limbs, extra, neg)) { + return false; + } + } + } + if (!shr_case(k, 0, neg)) { // shift by every bit + return false; + } + } + return true; +} + +template +constexpr bool shr_by_a_limb_matches_builtin() { + T x = all_ones(2); + x >>= limb_width; + return x == std::numeric_limits::max() && tail_is_zero(x); +} + +// The quotient limbs written in place must not survive above the new limb count. +template +constexpr bool rem_by_small() { + T x = all_ones(2); + x -= 1; // 2^(2W) - 2, which is 2 modulo 3 + x %= 3; + if (!(x == 2U && tail_is_zero(x))) { + return false; + } + T q = all_ones(2); + q -= 1; + q /= 3; + return q * 3 + 2 == all_ones(2) - 1 && tail_is_zero(q); +} + +template +constexpr bool div_rem_small_by_large() { + const T a = two_limbs(); + T b{1}; + b <<= 3 * limb_width; + auto r = div_rem_to_zero(a, b); + if (!(r.quotient == 0U && tail_is_zero(r.quotient) && r.remainder == a && tail_is_zero(r.remainder))) { + return false; + } + T a2 = two_limbs(); + auto r2 = div_rem_to_zero(std::move(a2), b); + return r2.quotient == 0U && tail_is_zero(r2.quotient) && r2.remainder == two_limbs(); +} + +template +constexpr bool from_chars_into_two_limbs(int base, const char* digits, unsigned expected) { + T x = two_limbs(); + std::string_view s{digits}; + const auto r = from_chars(s.data(), s.data() + s.size(), x, base); + return r.ec == std::errc{} && x == expected && tail_is_zero(x); +} + +template +constexpr bool from_chars_small_cases() { + return from_chars_into_two_limbs(16, "5", 5U) && from_chars_into_two_limbs(8, "5", 5U) && + from_chars_into_two_limbs(32, "5", 5U) && from_chars_into_two_limbs(2, "101", 5U) && + from_chars_into_two_limbs(4, "11", 5U) && from_chars_into_two_limbs(10, "5", 5U); +} + +// A heap destination full of ones, parsed into with a multi-block power-of-two base. +template +constexpr bool from_chars_reused_heap(int base, const std::string_view digits, const unsigned expected_shift) { + T x = all_ones(static_cast(T::inplace_capacity) + 10); + if (is_inplace(x)) { + return false; + } + const auto r = from_chars(digits.data(), digits.data() + digits.size(), x, base); + T fresh; + const auto r2 = from_chars(digits.data(), digits.data() + digits.size(), fresh, base); + T expected{1}; + expected <<= expected_shift; + return r.ec == std::errc{} && r2.ec == std::errc{} && x == fresh && x == expected; +} + +template +constexpr bool from_chars_reused_heap_all() { + // base 8: 3 bits per digit; base 32: 5 bits per digit; many blocks. + return from_chars_reused_heap(8, "1000000000000000000000000000000000000000", 39 * 3) && + from_chars_reused_heap(32, "10000000000000000000000000000000000000000000", 43 * 5) && + from_chars_reused_heap(8, "1000000000000000000000", 21 * 3) && + from_chars_reused_heap(32, "1000000000000", 12 * 5); +} + +template +constexpr bool moved_from_after_steal() { + T src{1}; + src <<= static_cast(T::inplace_bits) + 10; + T dst; + dst = std::move(src); + if (is_inplace(dst) || !(src == 0U) || !tail_is_zero(src)) { + return false; + } + // A destination that already holds more room still steals, and must leave a clean source. + T src2{1}; + src2 <<= static_cast(T::inplace_bits) + 10; + T dst2{1}; + dst2 <<= static_cast(T::inplace_bits) + 500; + dst2 = std::move(src2); + return !is_inplace(dst2) && src2 == 0U && tail_is_zero(src2); +} + +} // namespace inline_tail_test +BEMAN_BIG_INT_END_NAMESPACE + +namespace it = BEMAN_BIG_INT_NAMESPACE::inline_tail_test; + +#define INLINE_TAIL_CASES(CASE) \ + CASE(Sub128, bb::basic_big_int<128>, sub_self) \ + CASE(Sub256, bb::basic_big_int<256>, sub_self) \ + CASE(Xor128, bb::basic_big_int<128>, xor_self) \ + CASE(Xor256, bb::basic_big_int<256>, xor_self) \ + CASE(ShrAll128, bb::basic_big_int<128>, shr_all) \ + CASE(ShrAll256, bb::basic_big_int<256>, shr_all) \ + CASE(ShrLimb128, bb::basic_big_int<128>, shr_by_a_limb_matches_builtin) \ + CASE(ShrLimb256, bb::basic_big_int<256>, shr_by_a_limb_matches_builtin) \ + CASE(Rem128, bb::basic_big_int<128>, rem_by_small) \ + CASE(Rem256, bb::basic_big_int<256>, rem_by_small) \ + CASE(DivRem128, bb::basic_big_int<128>, div_rem_small_by_large) \ + CASE(DivRem256, bb::basic_big_int<256>, div_rem_small_by_large) \ + CASE(FromChars128, bb::basic_big_int<128>, from_chars_small_cases) \ + CASE(FromChars256, bb::basic_big_int<256>, from_chars_small_cases) \ + CASE(FromCharsHeap64, bb::big_int, from_chars_reused_heap_all) \ + CASE(FromCharsHeap128, bb::basic_big_int<128>, from_chars_reused_heap_all) \ + CASE(Steal64, bb::big_int, moved_from_after_steal) \ + CASE(Steal128, bb::basic_big_int<128>, moved_from_after_steal) \ + CASE(Steal256, bb::basic_big_int<256>, moved_from_after_steal) + +#define INLINE_TAIL_TEST(NAME, TYPE, FN) \ + TEST(InlineTail, NAME) { EXPECT_TRUE(it::FN()); } +INLINE_TAIL_CASES(INLINE_TAIL_TEST) + +// Constant evaluation of the cases that need no heap-limb reads beyond the span. +static_assert(it::sub_self>()); +static_assert(it::xor_self>()); +static_assert(it::shr_all>()); +static_assert(it::shr_all>()); +static_assert(it::rem_by_small>()); +static_assert(it::div_rem_small_by_large>()); +static_assert(it::from_chars_small_cases>()); +static_assert(it::from_chars_reused_heap_all()); +static_assert(it::moved_from_after_steal>()); diff --git a/tests/beman/big_int/muldiv_frontend.test.cpp b/tests/beman/big_int/muldiv_frontend.test.cpp new file mode 100644 index 00000000..477683f7 --- /dev/null +++ b/tests/beman/big_int/muldiv_frontend.test.cpp @@ -0,0 +1,499 @@ +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-License-Identifier: BSL-1.0 + +// Front ends of multiplication and division (operator*, *=, /, %, /=, %=, div_rem_to_zero) against a +// limb-level reference: aliasing, results at and one past the inline capacity, every operand size from +// 0 to 20 limbs, shapes straddling the schoolbook / divide-and-conquer / Barrett gates, and constant +// evaluation of multi-limb products and divisions. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "testing.hpp" + +BEMAN_BIG_INT_BEGIN_NAMESPACE +namespace muldiv_frontend_test { + +using limb_t = uint_multiprecision_t; +using limbs_t = std::vector; + +// Converts without a cast when the types already agree (-Wuseless-cast), and with one otherwise. +template +constexpr limb_t to_limb(const U v) noexcept { + if constexpr (std::is_same_v) { + return v; + } else { + return static_cast(v); + } +} + +inline constexpr std::size_t limb_bits = static_cast(std::numeric_limits::digits); + +// Random magnitude of exactly `n` limbs with a nonzero top limb; n == 0 gives the empty (zero) magnitude. +inline limbs_t random_mag(const std::size_t n, std::mt19937_64& rng) { + limbs_t v(n); + for (auto& x : v) { + x = to_limb(rng()); + } + if (n != 0 && v.back() == 0) { + v.back() = 1; + } + return v; +} + +template +T make(const limbs_t& mag, const bool neg) { + if (mag.empty()) { + return T{}; + } + static constexpr char digits[] = "0123456789abcdef"; + std::string s; + if (neg) { + s.push_back('-'); + } + for (std::size_t i = mag.size(); i-- > 0;) { + for (std::size_t d = limb_bits / 4; d-- > 0;) { + s.push_back(digits[(mag[i] >> (4 * d)) & 0xFU]); + } + } + T x; + const auto [p, ec] = from_chars(s.data(), s.data() + s.size(), x, 16); + EXPECT_TRUE(ec == std::errc{} && p == s.data() + s.size()); + return x; +} + +template +limbs_t mag_of(const T& x) { + const auto rep = x.representation(); + limbs_t v(rep.begin(), rep.end()); + while (!v.empty() && v.back() == 0) { + v.pop_back(); + } + return v; +} + +inline limbs_t ref_mul(const limbs_t& a, const limbs_t& b) { + if (a.empty() || b.empty()) { + return {}; + } + limbs_t r(a.size() + b.size(), limb_t{0}); + detail::multiply_long(std::span{r}, std::span{a}, std::span{b}); + while (!r.empty() && r.back() == 0) { + r.pop_back(); + } + return r; +} + +inline limbs_t ref_add(const limbs_t& a, const limbs_t& b) { + limbs_t r(std::max(a.size(), b.size()) + 1, limb_t{0}); + bool carry = false; + for (std::size_t i = 0; i < r.size(); ++i) { + const limb_t x = i < a.size() ? a[i] : limb_t{0}; + const limb_t y = i < b.size() ? b[i] : limb_t{0}; + const auto [sum, out] = detail::carrying_add(x, y, carry); + r[i] = sum; + carry = out; + } + while (!r.empty() && r.back() == 0) { + r.pop_back(); + } + return r; +} + +// -1, 0, 1 for |a| <=> |b|. +inline int ref_cmp(const limbs_t& a, const limbs_t& b) { + if (a.size() != b.size()) { + return a.size() < b.size() ? -1 : 1; + } + for (std::size_t i = a.size(); i-- > 0;) { + if (a[i] != b[i]) { + return a[i] < b[i] ? -1 : 1; + } + } + return 0; +} + +// True when every in-place limb past the limb count is zero (the invariant behind inplace_to_bit_uint). +template +bool tail_is_zero(const T& x) { + if (!is_inplace(x)) { + return true; + } + const auto* const limbs = x.representation().data(); + for (std::size_t i = x.representation().size(); i < T::inplace_capacity; ++i) { + if (limbs[i] != 0) { + return false; + } + } + return true; +} + +template +::testing::AssertionResult is_value(const T& x, const limbs_t& mag, const bool neg) { + if (mag_of(x) != mag) { + return ::testing::AssertionFailure() + << "magnitude differs (got " << mag_of(x).size() << " limbs, want " << mag.size() << ")"; + } + if ((x < 0) != (neg && !mag.empty())) { + return ::testing::AssertionFailure() << "sign differs"; + } + if (!is_normalized(x)) { + return ::testing::AssertionFailure() << "not normalized"; + } + if (!tail_is_zero(x)) { + return ::testing::AssertionFailure() << "stale inline tail"; + } + if (x != make(mag, neg)) { + return ::testing::AssertionFailure() << "operator== disagrees with the limbs"; + } + return ::testing::AssertionSuccess(); +} + +// Checks every multiplication form of a * b against the reference. +template +void check_multiply(const limbs_t& am, const bool an, const limbs_t& bm, const bool bn) { + const limbs_t pm = ref_mul(am, bm); + const bool neg = an != bn; + const T a = make(am, an); + const T b = make(bm, bn); + const T a0 = a; + const T b0 = b; + + EXPECT_TRUE(is_value(a * b, pm, neg)) << "a*b la=" << am.size() << " lb=" << bm.size(); + EXPECT_TRUE(is_value(T{a} * b, pm, neg)) << "rvalue a*b la=" << am.size() << " lb=" << bm.size(); + T c = a; + c *= b; + EXPECT_TRUE(is_value(c, pm, neg)) << "a*=b la=" << am.size() << " lb=" << bm.size(); + // Reuse a destination that already holds a different, possibly longer, value. + std::mt19937_64 rng{1}; + T d = make(random_mag(24, rng), true); + d = a; + d *= b; + EXPECT_TRUE(is_value(d, pm, neg)) << "reused a*=b la=" << am.size() << " lb=" << bm.size(); + EXPECT_EQ(a, a0); + EXPECT_EQ(b, b0); +} + +// Checks every division form of a / b, a % b and div_rem_to_zero against the reference identities. +template +void check_divide(const limbs_t& am, const bool an, const limbs_t& bm, const bool bn) { + ASSERT_FALSE(bm.empty()); + const T a = make(am, an); + const T b = make(bm, bn); + const T a0 = a; + const T b0 = b; + + const T q = a / b; + const T r = a % b; + + // |r| < |b|, sign(r) follows a, q * b + r == a. + const limbs_t qm = mag_of(q); + const limbs_t rm = mag_of(r); + EXPECT_LT(ref_cmp(rm, bm), 0) << "remainder not below divisor, la=" << am.size() << " lb=" << bm.size(); + EXPECT_EQ(ref_add(ref_mul(qm, bm), rm), am) << "q*b+r != a, la=" << am.size() << " lb=" << bm.size(); + EXPECT_TRUE(is_value(q, qm, an != bn)); + EXPECT_TRUE(is_value(r, rm, an)); + + const auto both = div_rem_to_zero(a, b); + EXPECT_EQ(both.quotient, q) << "div_rem_to_zero quotient, la=" << am.size() << " lb=" << bm.size(); + EXPECT_EQ(both.remainder, r) << "div_rem_to_zero remainder, la=" << am.size() << " lb=" << bm.size(); + EXPECT_TRUE(is_value(both.quotient, qm, an != bn)); + EXPECT_TRUE(is_value(both.remainder, rm, an)); + + T c = a; + c /= b; + EXPECT_TRUE(is_value(c, qm, an != bn)) << "a/=b la=" << am.size() << " lb=" << bm.size(); + T d = a; + d %= b; + EXPECT_TRUE(is_value(d, rm, an)) << "a%=b la=" << am.size() << " lb=" << bm.size(); + EXPECT_EQ(a, a0); + EXPECT_EQ(b, b0); +} + +template +void differential_all_sizes() { + std::mt19937_64 rng{0x6d756c64u}; + for (std::size_t la = 0; la <= 20; ++la) { + for (std::size_t lb = 0; lb <= 20; ++lb) { + const limbs_t am = random_mag(la, rng); + const limbs_t bm = random_mag(lb, rng); + for (const bool an : {false, true}) { + for (const bool bn : {false, true}) { + check_multiply(am, an, bm, bn); + if (lb != 0) { + check_divide(am, an, bm, bn); + } + } + } + } + } +} + +TEST(MulDivFrontend, DifferentialBigInt) { differential_all_sizes(); } +TEST(MulDivFrontend, DifferentialInline128) { differential_all_sizes>(); } +TEST(MulDivFrontend, DifferentialInline256) { differential_all_sizes>(); } + +// Divisor and dividend of related sizes: below, at, and above each gate of the dispatcher. +template +void gate_shapes() { + std::mt19937_64 rng{0x6761746u}; + std::vector> shapes; + const std::size_t bz = detail::burnikel_ziegler_cutoff; + const std::size_t off = detail::burnikel_ziegler_offset; + for (const std::size_t s : {std::size_t{2}, std::size_t{3}, bz - 1, bz, bz + 1}) { + for (const std::size_t m : {s, s + 1, s + off - 1, s + off, s + off + 1, 3 * s + 1}) { + shapes.emplace_back(m, s); + } + } + const std::size_t c = detail::barrett_march_cutoff; + for (const std::size_t s : {c - 1, c, c + 1}) { + for (const std::size_t m : {16 * s - 1, 16 * s, 16 * s + 5}) { + shapes.emplace_back(m, s); + } + } + for (const auto& [m, s] : shapes) { + const limbs_t am = random_mag(m, rng); + const limbs_t bm = random_mag(s, rng); + check_divide(am, false, bm, false); + check_divide(am, true, bm, false); + check_divide(am, false, bm, true); + check_multiply(am, false, bm, true); + } +} + +TEST(MulDivFrontend, GateShapesBigInt) { gate_shapes(); } +TEST(MulDivFrontend, GateShapesInline128) { gate_shapes>(); } + +// Products and quotients landing exactly at, and one limb past, the inline capacity. +template +void inline_capacity_edges() { + std::mt19937_64 rng{0xed6eu}; + constexpr std::size_t cap = T::inplace_capacity; + for (const std::size_t result_limbs : {cap - 1, cap, cap + 1, cap + 2}) { + if (result_limbs < 2) { + continue; + } + for (const bool full : {false, true}) { + // a (result_limbs - 1 limbs) * b (1 limb): all-ones forces a carry limb, a small b does not. + limbs_t am = random_mag(result_limbs - 1, rng); + limbs_t bm = {full ? ~limb_t{0} : limb_t{3}}; + if (full) { + std::ranges::fill(am, ~limb_t{0}); + } + check_multiply(am, false, bm, false); + check_multiply(bm, true, am, false); + // And the matching division (multi-limb divisor too). + const limbs_t big = random_mag(result_limbs + 1, rng); + check_divide(big, false, bm, false); + check_divide(big, true, am.size() >= 2 ? am : random_mag(2, rng), false); + } + } +} + +TEST(MulDivFrontend, InlineCapacityEdgesBigInt) { inline_capacity_edges(); } +TEST(MulDivFrontend, InlineCapacityEdges128) { inline_capacity_edges>(); } +TEST(MulDivFrontend, InlineCapacityEdges256) { inline_capacity_edges>(); } + +// Aliasing: the destination is also an operand, and divisors that are copies of the dividend. +template +void aliasing() { + std::mt19937_64 rng{0xa11a5u}; + for (const std::size_t n : {std::size_t{1}, + std::size_t{2}, + std::size_t{3}, + std::size_t{9}, + std::size_t{20}, + std::size_t{80}, + std::size_t{300}}) { + for (const bool neg : {false, true}) { + const limbs_t am = random_mag(n, rng); + const limbs_t bm = random_mag(n / 2 + 1, rng); + const limbs_t sqm = ref_mul(am, am); + const limbs_t pm = ref_mul(am, bm); + + T a = make(am, neg); + const T b = make(bm, false); + a = a * b; + EXPECT_TRUE(is_value(a, pm, neg)) << "a = a * b, n=" << n; + + a = make(am, neg); + a = b * a; + EXPECT_TRUE(is_value(a, pm, neg)) << "a = b * a, n=" << n; + + a = make(am, neg); + a *= a; + EXPECT_TRUE(is_value(a, sqm, false)) << "a *= a, n=" << n; + + a = make(am, neg); + a = a * a; + EXPECT_TRUE(is_value(a, sqm, false)) << "a = a * a, n=" << n; + + a = make(am, neg); + a *= b; + EXPECT_TRUE(is_value(a, pm, neg)) << "a *= b, n=" << n; + + a = make(am, neg); + a /= a; + EXPECT_TRUE(is_value(a, limbs_t{1}, false)) << "a /= a, n=" << n; + + a = make(am, neg); + a %= a; + EXPECT_TRUE(is_value(a, limbs_t{}, false)) << "a %= a, n=" << n; + + a = make(am, neg); + const T copy = a; + a /= copy; + EXPECT_TRUE(is_value(a, limbs_t{1}, false)) << "a /= copy of a, n=" << n; + + a = make(am, neg); + a %= copy; + EXPECT_TRUE(is_value(a, limbs_t{}, false)) << "a %= copy of a, n=" << n; + + a = make(am, neg); + a = a / a; + EXPECT_TRUE(is_value(a, limbs_t{1}, false)) << "a = a / a, n=" << n; + + a = make(am, neg); + a = a % a; + EXPECT_TRUE(is_value(a, limbs_t{}, false)) << "a = a % a, n=" << n; + + // a = a / b and a = a % b through the destination. + const auto both = div_rem_to_zero(make(am, neg), b); + a = make(am, neg); + a = a / b; + EXPECT_EQ(a, both.quotient) << "a = a / b, n=" << n; + a = make(am, neg); + a = a % b; + EXPECT_EQ(a, both.remainder) << "a = a % b, n=" << n; + a = make(am, neg); + a /= b; + EXPECT_EQ(a, both.quotient) << "a /= b, n=" << n; + a = make(am, neg); + a %= b; + EXPECT_EQ(a, both.remainder) << "a %= b, n=" << n; + } + } +} + +TEST(MulDivFrontend, AliasingBigInt) { aliasing(); } +TEST(MulDivFrontend, AliasingInline128) { aliasing>(); } + +// Compound ops against built-in integers, one-limb and two-limb. +TEST(MulDivFrontend, CompoundWithIntegers) { + std::mt19937_64 rng{0x1417u}; + for (const std::size_t n : {std::size_t{1}, std::size_t{2}, std::size_t{7}, std::size_t{40}}) { + const limbs_t am = random_mag(n, rng); + for (const std::int64_t v : + {std::int64_t{3}, std::int64_t{-7}, std::int64_t{1} << 40, std::numeric_limits::max()}) { + // The 64-bit magnitude split into limbs of whatever width the build uses. + const std::uint64_t vmag = + v < 0 ? std::uint64_t{0} - static_cast(v) : static_cast(v); + limbs_t vm; + for (std::size_t sh = 0; sh < 64; sh += limb_bits) { + vm.push_back(to_limb(vmag >> sh)); + } + while (vm.size() > 1 && vm.back() == 0) { + vm.pop_back(); + } + big_int a = make(am, false); + a *= v; + EXPECT_TRUE(is_value(a, ref_mul(am, vm), v < 0)) << "a *= int, n=" << n; + a = make(am, false); + a /= v; + EXPECT_EQ(a, make(am, false) / make(vm, v < 0)); + a = make(am, false); + a %= v; + EXPECT_EQ(a, make(am, false) % make(vm, v < 0)); + } + } +} + +// Constant evaluation of multi-limb products and divisions. +consteval bool constexpr_multi_limb() { + big_int a = (big_int{1} << 200) + 12345; + big_int b = (big_int{1} << 70) + 3; + + big_int q = a / b; + big_int r = a % b; + if (q * b + r != a || r >= b || r < 0) { + return false; + } + const auto both = div_rem_to_zero(a, b); + if (both.quotient != q || both.remainder != r) { + return false; + } + + big_int c = a; + c *= b; + if (c != a * b || c / b != a || c % b != 0) { + return false; + } + c = a; + c /= b; + if (c != q) { + return false; + } + c = a; + c %= b; + if (c != r) { + return false; + } + c = a; + c *= c; + if (c != a * a) { + return false; + } + c = a; + c /= c; + if (c != 1) { + return false; + } + c = a; + c %= c; + if (c != 0) { + return false; + } + c = -a; + c *= 3; + if (c != -(a * 3)) { + return false; + } + c = a * 5; + c /= 5; + c %= (big_int{1} << 100); + return c == a % (big_int{1} << 100); +} + +static_assert(constexpr_multi_limb()); + +consteval bool constexpr_inline_variants() { + basic_big_int<256> a = (basic_big_int<256>{1} << 300) + 77; + basic_big_int<256> b = (basic_big_int<256>{1} << 130) + 5; + auto c = a; + c *= b; + if (c != a * b || c / b != a || c % b != 0) { + return false; + } + c = a; + c /= b; + return c == a / b; +} + +static_assert(constexpr_inline_variants()); + +} // namespace muldiv_frontend_test +BEMAN_BIG_INT_END_NAMESPACE diff --git a/tests/beman/big_int/perf/CMakeLists.txt b/tests/beman/big_int/perf/CMakeLists.txt index f69a7e76..fc6cf585 100644 --- a/tests/beman/big_int/perf/CMakeLists.txt +++ b/tests/beman/big_int/perf/CMakeLists.txt @@ -10,6 +10,14 @@ option( OFF ) +# Inline capacity (bits) of the big integer type behind every timed row: sweep_int = basic_big_int. 64 is +# big_int; the item 1 inline-capacity study builds 128, 256 and 512 as well. +set(BEMAN_BIG_INT_SWEEP_INLINE_BITS + 64 + CACHE STRING + "Inline bits of the integer type timed by shape_sweep. Default: 64 (big_int). Values: { 64, 128, 256, 512 }." +) + add_executable(beman.big_int.benchmarks.shape_sweep shape_sweep.cpp) target_link_libraries( beman.big_int.benchmarks.shape_sweep @@ -33,7 +41,9 @@ file( ) target_compile_definitions( beman.big_int.benchmarks.shape_sweep - PRIVATE BEMAN_BIG_INT_SWEEP_VIA_CMAKE + PRIVATE + BEMAN_BIG_INT_SWEEP_VIA_CMAKE + BEMAN_BIG_INT_SWEEP_INLINE_BITS=${BEMAN_BIG_INT_SWEEP_INLINE_BITS} ) target_include_directories( beman.big_int.benchmarks.shape_sweep diff --git a/tests/beman/big_int/perf/gmp_gap_analysis.md b/tests/beman/big_int/perf/gmp_gap_analysis.md index 5b813b50..36932b87 100644 --- a/tests/beman/big_int/perf/gmp_gap_analysis.md +++ b/tests/beman/big_int/perf/gmp_gap_analysis.md @@ -527,6 +527,12 @@ Decision criteria applied (from the plan): | 9 | Make `add_unsigned_spans` stop at the carry tail (1024x1 add is 2.5x GMP in the kernel row) and give the public add the same compiler-independent loop | kernel/gmp 2.5x, auto 1.2x | kernel rows only; public path already fine | S | both | | 10 | Conversions: after #2 and #4, recheck; power-chain reuse looks minor at 10000 limbs (4.5-7%) but is untested above 100k limbs | tochars/fromchars 262144/131072 2.8x | speculative | M | both | +Status of item 1: implemented on branch `opt_1`; results, targets met and not met, and the inline-capacity study are in +[`item1_frontend_results.md`](item1_frontend_results.md) (small `auto` is now 2-5 ns above the allocation floor for +mul/sqr, 0-6 ns for shifts and 3-6 ns for add/sub; x64 add 1000 840 -> 600 ns, meeting the 650 ns goal; open items: x64 +GCC in-place shifts at >= 256 limbs are 5-8% slower than the baseline, and a few sub-nanosecond small in-place rows). Decision +(2026-10-06): the default inline capacity stays at 64 bits (`big_int`, one limb). + Not recommended: asm for cmp (already faster than GMP on x64, parity on M4); asm for mul 32-512 limbs on x64 (IFMA beats GMP by 1.1-1.8x). diff --git a/tests/beman/big_int/perf/item1/final_m4.csv b/tests/beman/big_int/perf/item1/final_m4.csv new file mode 100644 index 00000000..77a44039 --- /dev/null +++ b/tests/beman/big_int/perf/item1/final_m4.csv @@ -0,0 +1,435 @@ +# item 1 before/after medians, m4; ns per operation (vecsort and the study are in study.csv) +# base_ns: cf1cf54 harness (base64); first_9145b4e_ns: first measurement of opt_1; d689f75_ns: re-measurement; +# after_ns: final value (b823c5f for shl/shr, d689f75 for the other ops); empty cells were not measured +# aggregation: median of the runs, divrem up to 32 limbs the mean of seeds 1 and 2; shl/shr base_ns from the +# b823c5f runs; spots (large band, 3-round runs) are medians of two seeds +# base64: sweep_inline_bits=64 sweep_int_sizeof=16 ARCH=aarch64 BMI2_ADX=0 AVX512_IFMA=0 SIMD_MUL=1 limb_bits=64 ndebug=1 compiler=appleclang-21.0.0_(clang-2100.3.34.2) build_type=RelWithDebInfo cxx_flags=-O3__-O2_-g_-DNDEBUG_defs:BEMAN_BIG_INT_SIMD_MUL +# after: sweep_inline_bits=64 sweep_int_sizeof=16 ARCH=aarch64 BMI2_ADX=0 AVX512_IFMA=0 SIMD_MUL=1 limb_bits=64 ndebug=1 compiler=appleclang-21.0.0_(clang-2100.3.34.2) build_type=RelWithDebInfo cxx_flags=-O3__-O2_-g_-DNDEBUG_defs:BEMAN_BIG_INT_SIMD_MUL +# after_shifts: sweep_inline_bits=64 sweep_int_sizeof=16 ARCH=aarch64 BMI2_ADX=0 AVX512_IFMA=0 SIMD_MUL=1 limb_bits=64 ndebug=1 compiler=appleclang-21.0.0_(clang-2100.3.34.2) build_type=RelWithDebInfo cxx_flags=-O3__-O2_-g_-DNDEBUG_defs:BEMAN_BIG_INT_SIMD_MUL +op,shape,row,base_ns,first_9145b4e_ns,d689f75_ns,after_ns,after_commit +add,1x1,auto,6.94,7.68,5.88,5.88,d689f75 +add,1x1,floor,9.43,9.33,9.63,9.63,d689f75 +add,1x1,gmp,1.49,1.49,1.49,1.49,d689f75 +add,1x1,gmpz,3.07,2.98,2.99,2.99,d689f75 +add,1x1,inplace,3.22,3.51,1.87,1.87,d689f75 +add,1x1,kernel,1.14,1.14,1.15,1.15,d689f75 +add,2x2,auto,14.84,15.21,15.91,15.91,d689f75 +add,2x2,floor,9.95,9.62,9.63,9.63,d689f75 +add,2x2,gmp,1.99,1.98,1.86,1.86,d689f75 +add,2x2,gmpz,5.89,3.32,3.25,3.25,d689f75 +add,2x2,inplace,3.77,3.88,4.11,4.11,d689f75 +add,2x2,kernel,1.40,1.39,1.40,1.40,d689f75 +add,4x4,auto,15.36,14.32,14.57,14.57,d689f75 +add,4x4,floor,10.86,10.21,10.36,10.36,d689f75 +add,4x4,gmp,1.74,1.74,1.75,1.75,d689f75 +add,4x4,gmpz,4.82,5.08,4.08,4.08,d689f75 +add,4x4,inplace,3.74,3.50,3.82,3.82,d689f75 +add,4x4,kernel,1.96,1.92,1.94,1.94,d689f75 +add,8x8,auto,15.85,15.14,15.55,15.55,d689f75 +add,8x8,floor,11.97,12.09,11.98,11.98,d689f75 +add,8x8,gmp,2.25,2.23,2.25,2.25,d689f75 +add,8x8,gmpz,5.24,4.47,4.37,4.37,d689f75 +add,8x8,inplace,5.10,4.05,4.27,4.27,d689f75 +add,8x8,kernel,3.51,3.48,3.50,3.50,d689f75 +add,16x16,auto,16.42,16.88,17.69,17.69,d689f75 +add,16x16,floor,14.22,13.65,14.03,14.03,d689f75 +add,16x16,gmp,3.24,3.23,3.24,3.24,d689f75 +add,16x16,gmpz,4.54,4.26,4.47,4.47,d689f75 +add,16x16,inplace,9.78,5.59,5.74,5.74,d689f75 +add,16x16,kernel,7.58,7.56,7.56,7.56,d689f75 +add,64x64,auto,54.66,38.17,37.94,37.94,d689f75 +add,64x64,floor,61.10,60.74,60.91,60.91,d689f75 +add,64x64,gmp,11.20,11.09,11.13,11.13,d689f75 +add,64x64,gmpz,13.00,13.16,13.38,13.38,d689f75 +add,64x64,inplace,51.59,19.96,19.97,19.97,d689f75 +add,64x64,kernel,48.84,49.14,49.17,49.17,d689f75 +add,256x256,auto,148.77,109.92,109.77,109.77,d689f75 +add,256x256,floor,247.92,246.22,246.62,246.62,d689f75 +add,256x256,gmp,56.98,56.98,57.42,57.42,d689f75 +add,256x256,gmpz,57.94,57.78,58.12,58.12,d689f75 +add,256x256,inplace,240.00,100.92,101.21,101.21,d689f75 +add,256x256,kernel,234.01,234.69,236.51,236.51,d689f75 +add,1024x1,auto,172.66,107.13,108.17,108.17,d689f75 +add,1024x1,floor,758.47,754.90,761.81,761.81,d689f75 +add,1024x1,gmp,263.68,263.06,263.80,263.80,d689f75 +add,1024x1,gmpz,266.88,264.77,266.38,266.38,d689f75 +add,1024x1,inplace,980.38,2.98,3.11,3.11,d689f75 +add,1024x1,kernel,735.91,736.75,739.63,739.63,d689f75 +add,1024x1024,auto,564.88,450.12,450.30,450.30,d689f75 +add,1024x1024,floor,988.69,990.03,988.05,988.05,d689f75 +add,1024x1024,gmp,252.47,251.20,252.08,252.08,d689f75 +add,1024x1024,gmpz,253.74,253.31,252.50,252.50,d689f75 +add,1024x1024,inplace,989.67,433.60,437.13,437.13,d689f75 +add,1024x1024,kernel,983.58,970.88,977.76,977.76,d689f75 +add,2000x2000,auto,1132.31,886.66,877.01,877.01,d689f75 +add,2000x2000,floor,1944.43,1933.36,1931.36,1931.36,d689f75 +add,2000x2000,gmp,496.69,495.26,496.15,496.15,d689f75 +add,2000x2000,gmpz,498.79,496.74,497.95,497.95,d689f75 +add,2000x2000,inplace,1948.39,854.16,858.24,858.24,d689f75 +add,2000x2000,kernel,1929.97,1916.90,1927.84,1927.84,d689f75 +add,16384x16384,auto,9473.86,7266.76,7304.57,7304.57,d689f75 +add,16384x16384,floor,16294.38,15957.12,16478.88,16478.88,d689f75 +add,16384x16384,gmp,4112.33,4095.07,4127.24,4127.24,d689f75 +add,16384x16384,gmpz,4097.89,4101.70,4161.40,4161.40,d689f75 +add,16384x16384,inplace,16449.04,7105.27,7216.20,7216.20,d689f75 +add,16384x16384,kernel,16179.17,15812.65,16418.17,16418.17,d689f75 +divrem,2x1,auto,31.08,24.83,25.25,25.25,d689f75 +divrem,2x1,floor,13.80,13.54,13.78,13.78,d689f75 +divrem,2x1,gmp,5.06,5.04,5.04,5.04,d689f75 +divrem,2x1,gmpz,8.79,8.70,8.73,8.73,d689f75 +divrem,2x1,kernel,5.96,6.02,6.08,6.08,d689f75 +divrem,8x4,auto,96.02,75.77,75.40,75.40,d689f75 +divrem,8x4,floor,61.27,60.83,60.78,60.78,d689f75 +divrem,8x4,gmp,40.99,40.79,40.82,40.82,d689f75 +divrem,8x4,gmpz,43.58,43.82,43.90,43.90,d689f75 +divrem,8x4,kernel,46.21,46.73,46.62,46.62,d689f75 +divrem,32x16,auto,481.27,432.18,431.97,431.97,d689f75 +divrem,32x16,floor,417.68,411.09,410.63,410.63,d689f75 +divrem,32x16,gmp,209.16,208.75,207.91,207.91,d689f75 +divrem,32x16,gmpz,211.34,210.27,210.96,210.96,d689f75 +divrem,32x16,kernel,402.14,394.39,392.37,392.37,d689f75 +divrem,128x64,auto,3762.20,3691.20,3732.95,3732.95,d689f75 +divrem,128x64,floor,3668.61,3643.68,3657.77,3657.77,d689f75 +divrem,128x64,gmp,1773.90,1770.88,1774.84,1774.84,d689f75 +divrem,128x64,gmpz,1776.53,1777.14,1779.45,1779.45,d689f75 +divrem,128x64,kernel,3620.32,3601.84,3589.48,3589.48,d689f75 +divrem,512x256,auto,28815.19,28613.14,28675.56,28675.56,d689f75 +divrem,512x256,floor,28397.35,28378.52,28602.75,28602.75,d689f75 +divrem,512x256,gmp,16503.89,16435.91,16500.11,16500.11,d689f75 +divrem,512x256,gmpz,16493.35,16426.06,16474.24,16474.24,d689f75 +divrem,512x256,kernel,28349.18,28376.44,28459.87,28459.87,d689f75 +divrem,2000x100,auto,95716.26,94398.99,94547.32,94547.32,d689f75 +divrem,2000x100,floor,94481.02,93964.10,94287.57,94287.57,d689f75 +divrem,2000x100,gmp,69183.99,69292.76,69226.18,69226.18,d689f75 +divrem,2000x100,gmpz,69295.24,69057.00,69276.42,69276.42,d689f75 +divrem,2000x100,kernel,94148.52,94036.83,94541.41,94541.41,d689f75 +divrem,2000x1000,auto,234126.70,232354.65,232426.25,232426.25,d689f75 +divrem,2000x1000,floor,232151.52,231957.59,232324.23,232324.23,d689f75 +divrem,2000x1000,gmp,136991.99,135583.19,136316.04,136316.04,d689f75 +divrem,2000x1000,gmpz,135710.87,134539.70,134657.60,134657.60,d689f75 +divrem,2000x1000,kernel,232520.20,232348.72,232104.16,232104.16,d689f75 +divrem,4000x2000,auto,695982.04,688042.38,691382.19,691382.19,d689f75 +divrem,4000x2000,floor,692556.04,687476.72,687357.65,687357.65,d689f75 +divrem,4000x2000,gmp,347009.54,342697.39,343287.08,343287.08,d689f75 +divrem,4000x2000,gmpz,348235.03,343219.29,344788.79,344788.79,d689f75 +divrem,4000x2000,kernel,692014.58,686774.31,687024.99,687024.99,d689f75 +divrem,4096x2048,auto,706591.95,708499.26,707065.18,707065.18,d689f75 +divrem,4096x2048,floor,703491.38,701575.43,705680.31,705680.31,d689f75 +divrem,4096x2048,gmp,358878.35,357719.06,358520.68,358520.68,d689f75 +divrem,4096x2048,gmpz,360025.31,358228.93,358386.46,358386.46,d689f75 +divrem,4096x2048,kernel,704150.16,702138.66,706127.22,706127.22,d689f75 +divrem,65536x1,auto,246390.50,229283.22,231899.70,231899.70,d689f75 +divrem,65536x1,floor,228618.15,230561.71,232554.66,232554.66,d689f75 +divrem,65536x1,gmp,172095.96,169897.85,170984.09,170984.09,d689f75 +divrem,65536x1,gmpz,171581.49,170562.26,171285.58,171285.58,d689f75 +divrem,65536x1,kernel,230262.51,229888.18,230179.46,230179.46,d689f75 +fromchars,4x10,auto,91.22,91.64,91.87,91.87,d689f75 +fromchars,4x10,gmp,31.74,31.96,31.29,31.29,d689f75 +fromchars,4x10,gmpz,76.49,77.55,76.33,76.33,d689f75 +fromchars,4x10,kernel,32.62,32.73,33.02,33.02,d689f75 +fromchars,16x10,auto,399.46,397.81,394.31,394.31,d689f75 +fromchars,16x10,gmp,142.18,141.94,143.45,143.45,d689f75 +fromchars,16x10,gmpz,318.10,318.46,315.67,315.67,d689f75 +fromchars,16x10,kernel,193.39,192.01,192.54,192.54,d689f75 +fromchars,64x10,auto,2198.30,2182.15,2187.21,2187.21,d689f75 +fromchars,64x10,gmp,956.50,955.87,953.58,953.58,d689f75 +fromchars,64x10,gmpz,1601.95,1601.26,1603.17,1603.17,d689f75 +fromchars,64x10,kernel,1859.66,1855.87,1856.35,1856.35,d689f75 +fromchars,256x10,auto,13836.44,13791.90,13804.55,13804.55,d689f75 +fromchars,256x10,gmp,7999.72,7964.27,7953.50,7953.50,d689f75 +fromchars,256x10,gmpz,10776.47,10791.69,10737.40,10737.40,d689f75 +fromchars,256x10,kernel,11564.20,11537.28,11547.08,11547.08,d689f75 +fromchars,2000x10,auto,300171.02,300086.76,300851.98,300851.98,d689f75 +fromchars,2000x10,gmp,183952.42,184059.01,184097.48,184097.48,d689f75 +fromchars,2000x10,gmpz,203580.28,203324.28,203515.14,203515.14,d689f75 +fromchars,2000x10,kernel,282789.02,282617.74,283278.17,283278.17,d689f75 +fromchars,10000x10,auto,4198920.80,4171162.60,4189262.50,4189262.50,d689f75 +fromchars,10000x10,gmp,1769836.79,1764921.88,1770253.46,1770253.46,d689f75 +fromchars,10000x10,gmpz,1887806.82,1857816.27,1872965.91,1872965.91,d689f75 +fromchars,10000x10,kernel,4137033.30,4088150.00,4117279.10,4117279.10,d689f75 +mul,1x1,auto,15.57,14.18,14.29,14.29,d689f75 +mul,1x1,gmp,3.75,3.73,3.74,3.74,d689f75 +mul,1x1,gmpz,2.75,2.74,2.74,2.74,d689f75 +mul,2x2,auto,28.10,18.59,18.23,18.23,d689f75 +mul,2x2,floor,14.75,14.57,14.53,14.53,d689f75 +mul,2x2,gmp,4.57,4.71,4.61,4.61,d689f75 +mul,2x2,gmpz,6.55,6.78,6.48,6.48,d689f75 +mul,2x2,kernel,6.56,6.39,6.33,6.33,d689f75 +mul,3x3,auto,30.09,19.52,20.04,20.04,d689f75 +mul,3x3,floor,16.89,16.68,17.00,17.00,d689f75 +mul,3x3,gmp,6.22,6.28,6.08,6.08,d689f75 +mul,3x3,gmpz,8.43,8.60,10.50,10.50,d689f75 +mul,3x3,kernel,8.34,8.23,8.14,8.14,d689f75 +mul,4x4,auto,29.68,21.63,21.65,21.65,d689f75 +mul,4x4,floor,17.12,16.81,16.89,16.89,d689f75 +mul,4x4,gmp,9.12,8.96,8.96,8.96,d689f75 +mul,4x4,gmpz,10.34,10.58,10.99,10.99,d689f75 +mul,4x4,kernel,11.21,11.09,11.09,11.09,d689f75 +mul,6x6,auto,33.04,27.34,27.23,27.23,d689f75 +mul,6x6,floor,23.84,23.70,24.30,24.30,d689f75 +mul,6x6,gmp,14.56,14.71,14.43,14.43,d689f75 +mul,6x6,gmpz,18.75,18.59,18.63,18.63,d689f75 +mul,6x6,kernel,19.10,18.96,19.30,19.30,d689f75 +mul,8x8,auto,41.89,34.83,35.12,35.12,d689f75 +mul,8x8,floor,30.86,30.50,30.38,30.38,d689f75 +mul,8x8,gmp,22.54,23.28,22.72,22.72,d689f75 +mul,8x8,gmpz,26.81,27.04,24.98,24.98,d689f75 +mul,8x8,kernel,23.19,22.96,22.99,22.99,d689f75 +mul,12x12,auto,61.78,54.01,53.80,53.80,d689f75 +mul,12x12,floor,49.80,49.55,49.55,49.55,d689f75 +mul,12x12,gmp,44.67,44.31,44.16,44.16,d689f75 +mul,12x12,gmpz,46.56,46.66,46.69,46.69,d689f75 +mul,12x12,kernel,41.43,41.09,41.41,41.41,d689f75 +mul,16x16,auto,94.78,82.68,82.40,82.40,d689f75 +mul,16x16,floor,79.04,78.52,78.41,78.41,d689f75 +mul,16x16,gmp,77.53,76.82,76.80,76.80,d689f75 +mul,16x16,gmpz,79.14,78.45,78.46,78.46,d689f75 +mul,16x16,kernel,69.88,69.48,69.61,69.61,d689f75 +mul,32x32,auto,298.71,281.84,281.40,281.40,d689f75 +mul,32x32,floor,281.20,278.70,279.12,279.12,d689f75 +mul,32x32,gmp,263.35,264.67,262.85,262.85,d689f75 +mul,32x32,gmpz,265.86,264.68,263.29,263.29,d689f75 +mul,32x32,kernel,259.39,258.45,258.25,258.25,d689f75 +mul,64x64,auto,1039.33,1008.60,1007.77,1007.77,d689f75 +mul,64x64,floor,1015.38,1009.32,1010.36,1010.36,d689f75 +mul,64x64,gmp,776.34,772.60,774.12,774.12,d689f75 +mul,64x64,gmpz,778.35,779.15,775.95,775.95,d689f75 +mul,64x64,kernel,986.07,984.11,984.88,984.88,d689f75 +mul,128x128,auto,3638.80,3579.48,3582.43,3582.43,d689f75 +mul,128x128,floor,3593.87,3594.77,3611.09,3611.09,d689f75 +mul,128x128,gmp,2666.93,2653.89,2656.36,2656.36,d689f75 +mul,128x128,gmpz,2681.14,3008.62,2988.71,2988.71,d689f75 +mul,128x128,kernel,3579.89,3581.45,3587.14,3587.14,d689f75 +mul,256x256,auto,11896.32,11814.01,12024.95,12024.95,d689f75 +mul,256x256,floor,11793.47,11788.01,11786.17,11786.17,d689f75 +mul,256x256,gmp,9113.72,7852.43,7829.56,7829.56,d689f75 +mul,256x256,gmpz,7879.10,7859.24,7855.93,7855.93,d689f75 +mul,256x256,kernel,11758.11,11787.01,11795.64,11795.64,d689f75 +mul,512x512,auto,37400.51,36887.94,36998.75,36998.75,d689f75 +mul,512x512,floor,37078.73,36981.49,36984.97,36984.97,d689f75 +mul,512x512,gmp,21395.58,20147.68,20288.49,20288.49,d689f75 +mul,512x512,gmpz,19080.58,19044.35,19097.76,19097.76,d689f75 +mul,512x512,kernel,37086.22,36932.21,37216.43,37216.43,d689f75 +mul,1000x1000,auto,107250.23,106691.93,106887.81,106887.81,d689f75 +mul,1000x1000,floor,107071.76,106884.70,107171.46,107171.46,d689f75 +mul,1000x1000,gmp,55510.15,55222.24,55272.01,55272.01,d689f75 +mul,1000x1000,gmpz,55204.16,55236.22,55100.44,55100.44,d689f75 +mul,1000x1000,kernel,107169.80,106848.39,106770.83,106770.83,d689f75 +mul,2000x100,auto,47136.27,46319.63,46406.54,46406.54,d689f75 +mul,2000x100,floor,46841.78,46796.49,46697.96,46697.96,d689f75 +mul,2000x100,gmp,35277.32,35067.11,35134.72,35134.72,d689f75 +mul,2000x100,gmpz,34798.98,34707.86,34725.50,34725.50,d689f75 +mul,2000x100,kernel,46855.99,46697.11,46668.32,46668.32,d689f75 +mul,2000x2000,auto,302932.54,301592.05,301689.05,301689.05,d689f75 +mul,2000x2000,floor,303225.36,302273.63,302411.38,302411.38,d689f75 +mul,2000x2000,gmp,149200.50,148395.32,148630.10,148630.10,d689f75 +mul,2000x2000,gmpz,148912.46,148861.11,148856.32,148856.32,d689f75 +mul,2000x2000,kernel,302264.01,301691.55,302389.30,302389.30,d689f75 +mul,16384x16384,auto,5443656.25,5353474.00,5522911.38,5522911.38,d689f75 +mul,16384x16384,floor,5487395.75,5329625.00,5449203.12,5449203.12,d689f75 +mul,16384x16384,gmp,1586908.66,1582892.61,1608006.42,1608006.42,d689f75 +mul,16384x16384,gmpz,1571395.84,1567599.34,1599088.16,1599088.16,d689f75 +mul,16384x16384,kernel,5447302.12,5393000.00,5449135.50,5449135.50,d689f75 +shl,1x13,auto,17.55,12.38,12.29,12.25,b823c5f +shl,1x13,floor,11.01,10.99,10.93,11.39,b823c5f +shl,1x13,gmp,1.24,1.24,1.24,1.24,b823c5f +shl,1x13,gmpz,1.99,1.98,1.99,2.00,b823c5f +shl,1x13,inplace,2.30,2.49,3.42,2.50,b823c5f +shl,1x13,kernel,2.24,2.23,2.49,2.25,b823c5f +shl,1x13,kernelip,1.54,1.77,1.73,1.77,b823c5f +shl,4x13,auto,19.05,13.02,13.09,13.09,b823c5f +shl,4x13,floor,13.14,13.12,13.05,13.16,b823c5f +shl,4x13,gmp,2.00,1.99,1.99,1.99,b823c5f +shl,4x13,gmpz,2.73,2.73,2.74,2.76,b823c5f +shl,4x13,inplace,3.56,3.77,5.45,3.77,b823c5f +shl,4x13,kernel,3.72,3.73,3.73,3.73,b823c5f +shl,4x13,kernelip,3.37,3.38,3.37,3.38,b823c5f +shl,16x13,auto,21.48,16.19,15.46,16.07,b823c5f +shl,16x13,floor,12.07,12.29,12.15,12.13,b823c5f +shl,16x13,gmp,3.48,3.48,3.49,3.50,b823c5f +shl,16x13,gmpz,5.04,5.14,5.17,5.21,b823c5f +shl,16x13,inplace,5.79,5.30,6.88,5.36,b823c5f +shl,16x13,kernel,4.09,4.14,4.10,4.10,b823c5f +shl,16x13,kernelip,6.81,6.81,6.81,6.85,b823c5f +shl,256x13,auto,87.47,41.28,41.22,41.45,b823c5f +shl,256x13,floor,57.80,57.39,57.12,58.03,b823c5f +shl,256x13,gmp,37.00,36.92,37.03,37.16,b823c5f +shl,256x13,gmpz,38.09,37.88,37.95,38.44,b823c5f +shl,256x13,inplace,36.52,29.80,45.09,29.77,b823c5f +shl,256x13,kernel,49.44,49.11,49.26,49.70,b823c5f +shl,256x13,kernelip,49.17,48.67,48.81,49.01,b823c5f +shl,2000x13,auto,653.59,262.42,260.96,260.41,b823c5f +shl,2000x13,floor,401.41,400.48,402.14,402.17,b823c5f +shl,2000x13,gmp,1895.46,1801.38,1523.52,1979.30,b823c5f +shl,2000x13,gmpz,2186.34,1949.73,2261.95,1711.39,b823c5f +shl,2000x13,inplace,283.65,196.53,347.83,195.87,b823c5f +shl,2000x13,kernel,385.88,383.49,386.24,385.21,b823c5f +shl,2000x13,kernelip,376.98,376.62,377.03,378.19,b823c5f +shl,2000x77,auto,796.26,458.64,450.31,452.34,b823c5f +shl,2000x77,floor,542.67,539.62,546.92,544.42,b823c5f +shl,2000x77,gmp,294.19,290.15,290.44,290.08,b823c5f +shl,2000x77,gmpz,290.46,289.05,289.58,292.62,b823c5f +shl,2000x77,inplace,425.80,228.79,233.41,228.42,b823c5f +shl,2000x77,kernel,527.34,529.59,530.09,529.20,b823c5f +shl,2000x77,kernelip,520.55,520.14,521.24,522.77,b823c5f +shl,131072x13,auto,48969.00,,,18081.05,b823c5f +shl,131072x13,inplace,19832.04,,,18591.43,b823c5f +shl,131072x13,kernel,29296.36,,,29310.71,b823c5f +shr,1x13,auto,9.49,6.61,6.61,6.59,b823c5f +shr,1x13,floor,2.49,2.48,2.49,2.49,b823c5f +shr,1x13,gmp,0.99,0.99,0.99,0.99,b823c5f +shr,1x13,gmpz,1.99,1.99,1.99,1.99,b823c5f +shr,1x13,inplace,2.36,3.33,3.08,3.32,b823c5f +shr,1x13,kernel,2.23,2.24,2.24,2.24,b823c5f +shr,4x13,auto,17.76,14.00,14.09,15.20,b823c5f +shr,4x13,floor,12.36,11.96,12.29,12.28,b823c5f +shr,4x13,gmp,1.74,1.74,1.75,1.74,b823c5f +shr,4x13,gmpz,2.77,2.73,2.74,2.74,b823c5f +shr,4x13,inplace,3.26,3.70,4.74,3.58,b823c5f +shr,4x13,kernel,3.72,3.73,3.74,3.73,b823c5f +shr,16x13,auto,21.61,18.36,17.99,18.55,b823c5f +shr,16x13,floor,13.27,12.98,13.15,13.30,b823c5f +shr,16x13,gmp,3.24,3.23,3.23,3.23,b823c5f +shr,16x13,gmpz,4.26,4.24,4.24,4.26,b823c5f +shr,16x13,inplace,5.07,5.24,6.12,5.20,b823c5f +shr,16x13,kernel,4.17,4.16,4.13,4.17,b823c5f +shr,256x13,auto,87.58,42.59,42.55,42.86,b823c5f +shr,256x13,floor,75.62,75.25,75.39,75.89,b823c5f +shr,256x13,gmp,36.91,36.77,36.75,36.77,b823c5f +shr,256x13,gmpz,38.52,38.09,38.11,38.08,b823c5f +shr,256x13,inplace,35.83,30.38,44.76,30.39,b823c5f +shr,256x13,kernel,66.39,66.53,66.79,66.52,b823c5f +shr,2000x13,auto,692.17,255.11,271.16,273.83,b823c5f +shr,2000x13,floor,528.17,525.08,526.91,526.23,b823c5f +shr,2000x13,gmp,313.17,298.89,294.86,310.40,b823c5f +shr,2000x13,gmpz,298.43,298.33,302.23,309.36,b823c5f +shr,2000x13,inplace,282.46,196.34,347.46,195.99,b823c5f +shr,2000x13,kernel,512.63,513.76,512.73,513.74,b823c5f +shr,2000x77,auto,1064.38,276.44,280.69,276.01,b823c5f +shr,2000x77,floor,528.64,526.33,528.25,530.58,b823c5f +shr,2000x77,gmp,295.63,290.11,291.92,293.15,b823c5f +shr,2000x77,gmpz,289.86,290.30,287.30,286.56,b823c5f +shr,2000x77,inplace,420.02,195.85,195.86,196.12,b823c5f +shr,2000x77,kernel,518.28,516.77,517.40,521.51,b823c5f +shr,131072x13,auto,50436.86,,,18186.25,b823c5f +shr,131072x13,inplace,19857.28,,,18632.11,b823c5f +shr,131072x13,kernel,37566.04,,,37657.43,b823c5f +sqr,4x4,auto,29.58,21.62,21.59,21.59,d689f75 +sqr,4x4,floor,16.41,16.42,16.24,16.24,d689f75 +sqr,4x4,gmp,6.97,6.95,6.96,6.96,d689f75 +sqr,4x4,gmpz,9.42,9.54,9.71,9.71,d689f75 +sqr,4x4,kernel,11.34,11.25,11.31,11.31,d689f75 +sqr,16x16,auto,88.87,75.34,75.85,75.85,d689f75 +sqr,16x16,floor,70.56,72.47,71.81,71.81,d689f75 +sqr,16x16,gmp,62.15,62.83,62.06,62.06,d689f75 +sqr,16x16,gmpz,65.88,66.06,66.19,66.19,d689f75 +sqr,16x16,kernel,62.16,63.80,61.91,61.91,d689f75 +sqr,64x64,auto,626.38,600.00,600.47,600.47,d689f75 +sqr,64x64,floor,602.00,598.38,599.75,599.75,d689f75 +sqr,64x64,gmp,671.80,669.88,673.18,673.18,d689f75 +sqr,64x64,gmpz,675.05,672.30,677.12,677.12,d689f75 +sqr,64x64,kernel,585.11,580.66,582.15,582.15,d689f75 +sqr,512x512,auto,24695.35,24374.42,24317.03,24317.03,d689f75 +sqr,512x512,floor,24419.32,24348.04,24389.00,24389.00,d689f75 +sqr,512x512,gmp,17412.99,18323.93,16499.51,16499.51,d689f75 +sqr,512x512,gmpz,19776.65,19676.06,20575.62,20575.62,d689f75 +sqr,512x512,kernel,24368.15,24356.54,24344.64,24344.64,d689f75 +sqr,2000x2000,auto,224588.01,223342.36,223255.60,223255.60,d689f75 +sqr,2000x2000,floor,223620.80,223261.40,223801.08,223801.08,d689f75 +sqr,2000x2000,gmp,121110.38,120575.21,120788.51,120788.51,d689f75 +sqr,2000x2000,gmpz,121249.65,120909.93,121153.62,121153.62,d689f75 +sqr,2000x2000,kernel,223692.57,223746.91,223224.30,223224.30,d689f75 +sub,1x1,auto,7.57,8.00,5.71,5.71,d689f75 +sub,1x1,floor,1.74,1.75,1.74,1.74,d689f75 +sub,1x1,gmp,1.49,1.49,1.50,1.50,d689f75 +sub,1x1,gmpz,2.74,2.48,2.49,2.49,d689f75 +sub,1x1,inplace,3.41,3.52,1.99,1.99,d689f75 +sub,1x1,kernel,1.03,1.05,1.05,1.05,d689f75 +sub,2x2,auto,15.00,14.72,14.73,14.73,d689f75 +sub,2x2,floor,9.48,9.46,9.29,9.29,d689f75 +sub,2x2,gmp,1.93,1.92,1.99,1.99,d689f75 +sub,2x2,gmpz,3.24,2.87,2.87,2.87,d689f75 +sub,2x2,inplace,3.26,3.48,3.83,3.83,d689f75 +sub,2x2,kernel,1.40,1.42,1.40,1.40,d689f75 +sub,4x4,auto,15.84,14.72,15.13,15.13,d689f75 +sub,4x4,floor,10.04,10.38,9.98,9.98,d689f75 +sub,4x4,gmp,1.74,1.74,1.74,1.74,d689f75 +sub,4x4,gmpz,2.99,2.74,2.74,2.74,d689f75 +sub,4x4,inplace,3.90,3.61,4.05,4.05,d689f75 +sub,4x4,kernel,1.96,1.94,1.93,1.93,d689f75 +sub,8x8,auto,16.25,15.59,15.73,15.73,d689f75 +sub,8x8,floor,11.68,11.80,11.47,11.47,d689f75 +sub,8x8,gmp,2.25,2.23,2.24,2.24,d689f75 +sub,8x8,gmpz,3.50,3.23,3.24,3.24,d689f75 +sub,8x8,inplace,5.04,4.12,4.38,4.38,d689f75 +sub,8x8,kernel,3.51,3.49,3.50,3.50,d689f75 +sub,16x16,auto,18.38,17.50,18.05,18.05,d689f75 +sub,16x16,floor,15.34,15.12,15.06,15.06,d689f75 +sub,16x16,gmp,3.25,3.17,3.23,3.23,d689f75 +sub,16x16,gmpz,4.36,4.29,4.37,4.37,d689f75 +sub,16x16,inplace,10.92,5.23,5.45,5.45,d689f75 +sub,16x16,kernel,7.70,7.71,7.71,7.71,d689f75 +sub,64x64,auto,41.20,35.94,36.13,36.13,d689f75 +sub,64x64,floor,60.20,60.22,61.20,61.20,d689f75 +sub,64x64,gmp,11.47,11.21,11.10,11.10,d689f75 +sub,64x64,gmpz,13.89,13.86,13.47,13.47,d689f75 +sub,64x64,inplace,51.16,19.90,19.82,19.82,d689f75 +sub,64x64,kernel,49.81,49.49,53.45,53.45,d689f75 +sub,256x256,auto,132.79,110.27,111.16,111.16,d689f75 +sub,256x256,floor,245.86,245.50,246.62,246.62,d689f75 +sub,256x256,gmp,57.34,57.09,57.34,57.34,d689f75 +sub,256x256,gmpz,58.10,57.89,57.96,57.96,d689f75 +sub,256x256,inplace,239.86,101.05,101.20,101.20,d689f75 +sub,256x256,kernel,236.89,236.64,237.82,237.82,d689f75 +sub,1024x1,auto,189.95,129.92,128.09,128.09,d689f75 +sub,1024x1,floor,758.74,754.80,760.56,760.56,d689f75 +sub,1024x1,gmp,262.62,262.28,263.20,263.20,d689f75 +sub,1024x1,gmpz,266.63,265.53,267.36,267.36,d689f75 +sub,1024x1,inplace,976.43,2.98,3.26,3.26,d689f75 +sub,1024x1,kernel,736.32,729.88,736.08,736.08,d689f75 +sub,1024x1024,auto,562.27,453.14,454.62,454.62,d689f75 +sub,1024x1024,floor,986.75,977.93,986.71,986.71,d689f75 +sub,1024x1024,gmp,252.00,251.46,252.12,252.12,d689f75 +sub,1024x1024,gmpz,254.09,253.48,252.03,252.03,d689f75 +sub,1024x1024,inplace,988.36,434.54,435.00,435.00,d689f75 +sub,1024x1024,kernel,978.48,975.80,979.40,979.40,d689f75 +sub,2000x2000,auto,1115.55,876.50,884.12,884.12,d689f75 +sub,2000x2000,floor,1921.93,1925.27,1946.38,1946.38,d689f75 +sub,2000x2000,gmp,495.37,494.18,495.57,495.57,d689f75 +sub,2000x2000,gmpz,497.61,497.19,497.37,497.37,d689f75 +sub,2000x2000,inplace,1952.06,849.67,852.99,852.99,d689f75 +sub,2000x2000,kernel,1929.80,1919.83,1926.73,1926.73,d689f75 +tochars,1x10,auto,6.01,,6.00,6.00,d689f75 +tochars,1x10,gmp,16.49,,16.69,16.69,d689f75 +tochars,1x10,gmpz,24.96,,24.96,24.96,d689f75 +tochars,1x10,kernel,34.33,,34.39,34.39,d689f75 +tochars,4x10,auto,84.03,,84.37,84.37,d689f75 +tochars,4x10,gmp,74.78,,74.97,74.97,d689f75 +tochars,4x10,gmpz,97.29,,95.43,95.43,d689f75 +tochars,4x10,kernel,103.03,,102.16,102.16,d689f75 +tochars,16x10,auto,828.75,,841.67,841.67,d689f75 +tochars,16x10,gmp,507.89,,510.80,510.80,d689f75 +tochars,16x10,gmpz,604.03,,601.88,601.88,d689f75 +tochars,16x10,kernel,740.73,,738.61,738.61,d689f75 +tochars,16x16,auto,85.74,,85.03,85.03,d689f75 +tochars,16x16,gmp,79.56,,79.34,79.34,d689f75 +tochars,16x16,gmpz,160.03,,159.40,159.40,d689f75 +tochars,64x10,auto,4746.03,,4744.51,4744.51,d689f75 +tochars,64x10,gmp,2838.95,,2833.98,2833.98,d689f75 +tochars,64x10,gmpz,3167.69,,3155.32,3155.32,d689f75 +tochars,64x10,kernel,4495.84,,4524.18,4524.18,d689f75 +tochars,256x10,auto,33794.60,,33818.88,33818.88,d689f75 +tochars,256x10,gmp,19012.54,,19142.78,19142.78,d689f75 +tochars,256x10,gmpz,20403.92,,20421.38,20421.38,d689f75 +tochars,256x10,kernel,32935.90,,32743.63,32743.63,d689f75 +tochars,2000x10,auto,679612.50,,676793.05,676793.05,d689f75 +tochars,2000x10,gmp,403089.59,,403513.75,403513.75,d689f75 +tochars,2000x10,gmpz,415929.71,,415982.99,415982.99,d689f75 +tochars,2000x10,kernel,668488.88,,668851.67,668851.67,d689f75 +tochars,2000x16,auto,9936.17,,9680.80,9680.80,d689f75 +tochars,2000x16,gmp,9103.53,,9141.08,9141.08,d689f75 +tochars,2000x16,gmpz,16845.17,,17045.62,17045.62,d689f75 +tochars,10000x10,auto,8278333.33,8240937.50,8342500.00,8342500.00,d689f75 +tochars,10000x10,gmp,4115158.20,4108583.40,4165145.80,4165145.80,d689f75 +tochars,10000x10,gmpz,4173420.80,4180900.10,4195345.80,4195345.80,d689f75 +tochars,10000x10,kernel,8238222.33,8216750.00,8247499.83,8247499.83,d689f75 diff --git a/tests/beman/big_int/perf/item1/final_x64.csv b/tests/beman/big_int/perf/item1/final_x64.csv new file mode 100644 index 00000000..2703de5a --- /dev/null +++ b/tests/beman/big_int/perf/item1/final_x64.csv @@ -0,0 +1,435 @@ +# item 1 before/after medians, x64; ns per operation (vecsort and the study are in study.csv) +# base_ns: cf1cf54 harness (base64); first_9145b4e_ns: first measurement of opt_1; d689f75_ns: re-measurement; +# after_ns: final value (b823c5f for shl/shr, d689f75 for the other ops); empty cells were not measured +# aggregation: median of the runs, divrem up to 32 limbs the mean of seeds 1 and 2; shl/shr base_ns from the +# b823c5f runs; spots (large band, 3-round runs) are medians of two seeds +# base64: sweep_inline_bits=64 sweep_int_sizeof=16 ARCH=x86_64 BMI2_ADX=1 AVX512_IFMA=1 SIMD_MUL=1 limb_bits=64 ndebug=1 compiler=gcc-14.3.0 build_type=RelWithDebInfo cxx_flags=-march=native_-O3__-O2_-g_-DNDEBUG_defs:BEMAN_BIG_INT_SIMD_MUL +# after: sweep_inline_bits=64 sweep_int_sizeof=16 ARCH=x86_64 BMI2_ADX=1 AVX512_IFMA=1 SIMD_MUL=1 limb_bits=64 ndebug=1 compiler=gcc-14.3.0 build_type=RelWithDebInfo cxx_flags=-march=native_-O3__-O2_-g_-DNDEBUG_defs:BEMAN_BIG_INT_SIMD_MUL +# after_shifts: sweep_inline_bits=64 sweep_int_sizeof=16 ARCH=x86_64 BMI2_ADX=1 AVX512_IFMA=1 SIMD_MUL=1 limb_bits=64 ndebug=1 compiler=gcc-14.3.0 build_type=RelWithDebInfo cxx_flags=-march=native_-O3__-O2_-g_-DNDEBUG_defs:BEMAN_BIG_INT_SIMD_MUL +op,shape,row,base_ns,first_9145b4e_ns,d689f75_ns,after_ns,after_commit +add,1x1,auto,8.23,6.71,5.38,5.38,d689f75 +add,1x1,floor,9.86,9.82,9.82,9.82,d689f75 +add,1x1,gmp,2.27,2.27,2.27,2.27,d689f75 +add,1x1,gmpz,4.16,4.16,4.16,4.16,d689f75 +add,1x1,inplace,4.13,4.91,2.56,2.56,d689f75 +add,1x1,kernel,1.89,1.92,1.94,1.94,d689f75 +add,2x2,auto,17.86,15.91,15.88,15.88,d689f75 +add,2x2,floor,10.21,10.27,10.20,10.20,d689f75 +add,2x2,gmp,2.27,2.27,2.27,2.27,d689f75 +add,2x2,gmpz,4.16,4.19,4.16,4.16,d689f75 +add,2x2,inplace,5.89,6.14,6.24,6.24,d689f75 +add,2x2,kernel,2.37,2.40,2.38,2.38,d689f75 +add,4x4,auto,18.05,15.41,15.42,15.42,d689f75 +add,4x4,floor,10.97,11.10,10.95,10.95,d689f75 +add,4x4,gmp,2.50,2.50,2.50,2.50,d689f75 +add,4x4,gmpz,4.38,4.40,4.37,4.37,d689f75 +add,4x4,inplace,5.62,5.71,5.71,5.71,d689f75 +add,4x4,kernel,3.07,3.06,3.05,3.05,d689f75 +add,8x8,auto,19.28,16.75,16.94,16.94,d689f75 +add,8x8,floor,12.50,12.59,12.51,12.51,d689f75 +add,8x8,gmp,3.69,3.68,3.68,3.68,d689f75 +add,8x8,gmpz,4.68,4.69,4.68,4.68,d689f75 +add,8x8,inplace,7.35,7.09,7.32,7.32,d689f75 +add,8x8,kernel,4.63,4.55,4.53,4.53,d689f75 +add,16x16,auto,25.30,20.94,21.06,21.06,d689f75 +add,16x16,floor,15.50,15.61,15.50,15.50,d689f75 +add,16x16,gmp,5.57,5.50,5.50,5.50,d689f75 +add,16x16,gmpz,6.54,6.53,6.53,6.53,d689f75 +add,16x16,inplace,11.84,11.86,11.20,11.20,d689f75 +add,16x16,kernel,7.56,7.56,7.56,7.56,d689f75 +add,64x64,auto,63.69,57.59,46.24,46.24,d689f75 +add,64x64,floor,41.28,41.75,41.14,41.14,d689f75 +add,64x64,gmp,15.45,15.45,15.45,15.45,d689f75 +add,64x64,gmpz,16.49,16.49,16.49,16.49,d689f75 +add,64x64,inplace,38.88,48.06,35.75,35.75,d689f75 +add,64x64,kernel,33.25,32.87,32.87,32.87,d689f75 +add,256x256,auto,223.88,213.52,162.91,162.91,d689f75 +add,256x256,floor,158.78,159.25,158.53,158.53,d689f75 +add,256x256,gmp,78.23,78.22,78.22,78.22,d689f75 +add,256x256,gmpz,79.47,79.48,79.50,79.50,d689f75 +add,256x256,inplace,145.83,196.09,144.70,144.70,d689f75 +add,256x256,kernel,142.56,142.38,142.30,142.30,d689f75 +add,1024x1,auto,280.90,252.46,251.73,251.73,d689f75 +add,1024x1,floor,593.57,593.91,593.48,593.48,d689f75 +add,1024x1,gmp,232.64,233.00,232.99,232.99,d689f75 +add,1024x1,gmpz,233.99,234.06,234.15,234.15,d689f75 +add,1024x1,inplace,579.77,5.11,4.84,4.84,d689f75 +add,1024x1,kernel,577.70,577.40,577.41,577.41,d689f75 +add,1024x1024,auto,841.90,800.14,599.06,599.06,d689f75 +add,1024x1024,floor,594.26,594.67,594.00,594.00,d689f75 +add,1024x1024,gmp,390.76,390.51,390.51,390.51,d689f75 +add,1024x1024,gmpz,391.71,391.44,391.40,391.40,d689f75 +add,1024x1024,inplace,583.05,790.12,581.02,581.02,d689f75 +add,1024x1024,kernel,578.46,577.78,577.78,577.78,d689f75 +add,2000x2000,auto,1721.62,1535.37,1153.34,1153.34,d689f75 +add,2000x2000,floor,1150.49,1150.65,1149.44,1149.44,d689f75 +add,2000x2000,gmp,471.81,468.92,474.00,474.00,d689f75 +add,2000x2000,gmpz,482.93,481.20,481.35,481.35,d689f75 +add,2000x2000,inplace,1134.76,1520.53,1132.45,1132.45,d689f75 +add,2000x2000,kernel,1133.06,1132.11,1131.68,1131.68,d689f75 +add,16384x16384,auto,14810.19,12435.36,9329.74,9329.74,d689f75 +add,16384x16384,floor,9323.76,9320.83,9315.54,9315.54,d689f75 +add,16384x16384,gmp,6917.77,7411.56,6936.92,6936.92,d689f75 +add,16384x16384,gmpz,7364.76,7409.74,6898.03,6898.03,d689f75 +add,16384x16384,inplace,9315.09,12455.62,9291.74,9291.74,d689f75 +add,16384x16384,kernel,9304.25,9298.06,9293.57,9293.57,d689f75 +divrem,2x1,auto,41.66,35.42,36.20,36.20,d689f75 +divrem,2x1,floor,12.77,12.82,12.80,12.80,d689f75 +divrem,2x1,gmp,7.69,7.73,7.69,7.69,d689f75 +divrem,2x1,gmpz,12.46,12.46,12.39,12.39,d689f75 +divrem,2x1,kernel,5.47,5.76,5.68,5.68,d689f75 +divrem,8x4,auto,112.71,94.69,94.06,94.06,d689f75 +divrem,8x4,floor,59.81,67.60,67.72,67.72,d689f75 +divrem,8x4,gmp,41.89,41.83,41.92,41.92,d689f75 +divrem,8x4,gmpz,43.78,45.66,44.19,44.19,d689f75 +divrem,8x4,kernel,45.71,50.93,50.30,50.30,d689f75 +divrem,32x16,auto,370.94,340.25,342.22,342.22,d689f75 +divrem,32x16,floor,303.53,306.00,306.14,306.14,d689f75 +divrem,32x16,gmp,192.02,191.71,190.56,190.56,d689f75 +divrem,32x16,gmpz,193.96,196.41,193.53,193.53,d689f75 +divrem,32x16,kernel,288.55,290.56,288.27,288.27,d689f75 +divrem,128x64,auto,3342.14,3311.61,3446.34,3446.34,d689f75 +divrem,128x64,floor,3275.18,3264.14,3364.22,3364.22,d689f75 +divrem,128x64,gmp,1719.43,1723.24,1719.74,1719.74,d689f75 +divrem,128x64,gmpz,1722.79,1722.86,1725.11,1725.11,d689f75 +divrem,128x64,kernel,3201.06,3226.98,3343.18,3343.18,d689f75 +divrem,512x256,auto,29558.60,29508.37,29817.40,29817.40,d689f75 +divrem,512x256,floor,29265.99,29221.33,29639.40,29639.40,d689f75 +divrem,512x256,gmp,16254.41,16233.17,16260.22,16260.22,d689f75 +divrem,512x256,gmpz,16295.88,16239.44,16287.39,16287.39,d689f75 +divrem,512x256,kernel,29393.89,29166.29,29590.92,29590.92,d689f75 +divrem,2000x100,auto,144653.98,144972.40,144971.35,144971.35,d689f75 +divrem,2000x100,floor,142725.61,142574.95,143817.21,143817.21,d689f75 +divrem,2000x100,gmp,72563.71,72315.51,72331.67,72331.67,d689f75 +divrem,2000x100,gmpz,72269.21,72301.43,72348.47,72348.47,d689f75 +divrem,2000x100,kernel,143841.42,144126.99,144084.80,144084.80,d689f75 +divrem,2000x1000,auto,185318.81,185240.67,186774.10,186774.10,d689f75 +divrem,2000x1000,floor,184152.70,185470.49,185955.12,185955.12,d689f75 +divrem,2000x1000,gmp,137925.83,138206.38,137968.62,137968.62,d689f75 +divrem,2000x1000,gmpz,137926.68,138095.54,137906.30,137906.30,d689f75 +divrem,2000x1000,kernel,184253.69,185220.27,185733.07,185733.07,d689f75 +divrem,4000x2000,auto,478481.80,474978.93,477627.12,477627.12,d689f75 +divrem,4000x2000,floor,474608.67,475195.26,477727.83,477727.83,d689f75 +divrem,4000x2000,gmp,336143.68,336658.24,335369.40,335369.40,d689f75 +divrem,4000x2000,gmpz,334840.80,335770.58,334295.77,334295.77,d689f75 +divrem,4000x2000,kernel,475155.38,475542.15,478191.69,478191.69,d689f75 +divrem,4096x2048,auto,499033.82,498766.78,498476.96,498476.96,d689f75 +divrem,4096x2048,floor,497096.70,498373.57,495500.93,495500.93,d689f75 +divrem,4096x2048,gmp,370374.41,369407.30,369340.38,369340.38,d689f75 +divrem,4096x2048,gmpz,369214.30,367635.09,368706.87,368706.87,d689f75 +divrem,4096x2048,kernel,497946.41,498831.91,495568.67,495568.67,d689f75 +divrem,65536x1,auto,193721.33,174417.12,168835.73,168835.73,d689f75 +divrem,65536x1,floor,174747.77,168074.78,167435.50,167435.50,d689f75 +divrem,65536x1,gmp,148101.10,148101.80,148097.74,148097.74,d689f75 +divrem,65536x1,gmpz,148112.91,148112.78,148110.39,148110.39,d689f75 +divrem,65536x1,kernel,167123.32,167325.38,166946.54,166946.54,d689f75 +fromchars,4x10,auto,194.06,116.13,115.63,115.63,d689f75 +fromchars,4x10,gmp,31.66,31.68,31.65,31.65,d689f75 +fromchars,4x10,gmpz,68.17,68.94,67.88,67.88,d689f75 +fromchars,4x10,kernel,38.15,38.80,38.01,38.01,d689f75 +fromchars,16x10,auto,421.77,433.46,425.20,425.20,d689f75 +fromchars,16x10,gmp,124.14,123.86,124.02,124.02,d689f75 +fromchars,16x10,gmpz,258.17,258.12,256.65,256.65,d689f75 +fromchars,16x10,kernel,217.35,215.86,199.96,199.96,d689f75 +fromchars,64x10,auto,2790.47,3094.72,2852.27,2852.27,d689f75 +fromchars,64x10,gmp,889.61,905.83,890.67,890.67,d689f75 +fromchars,64x10,gmpz,1368.07,1393.03,1369.80,1369.80,d689f75 +fromchars,64x10,kernel,2122.82,2433.77,1807.23,1807.23,d689f75 +fromchars,256x10,auto,28563.94,30825.86,27920.01,27920.01,d689f75 +fromchars,256x10,gmp,8046.36,8017.06,8013.43,8013.43,d689f75 +fromchars,256x10,gmpz,9955.38,9958.36,9970.85,9970.85,d689f75 +fromchars,256x10,kernel,25689.38,29560.08,24430.63,24430.63,d689f75 +fromchars,2000x10,auto,429183.11,427566.80,385669.07,385669.07,d689f75 +fromchars,2000x10,gmp,183073.35,182858.83,182726.82,182726.82,d689f75 +fromchars,2000x10,gmpz,197928.45,197722.46,197822.32,197822.32,d689f75 +fromchars,2000x10,kernel,408671.72,405710.69,362563.51,362563.51,d689f75 +fromchars,10000x10,auto,3670158.92,3651386.33,3475893.00,3475893.00,d689f75 +fromchars,10000x10,gmp,1932681.96,1933053.19,1932903.68,1932903.68,d689f75 +fromchars,10000x10,gmpz,2008734.00,2006859.20,2005996.60,2005996.60,d689f75 +fromchars,10000x10,kernel,3588344.33,3567768.42,3362265.17,3362265.17,d689f75 +mul,1x1,auto,19.41,16.34,16.61,16.61,d689f75 +mul,1x1,gmp,3.39,3.37,3.39,3.39,d689f75 +mul,1x1,gmpz,4.00,4.02,3.98,3.98,d689f75 +mul,2x2,auto,28.25,18.88,18.88,18.88,d689f75 +mul,2x2,floor,15.18,14.95,14.96,14.96,d689f75 +mul,2x2,gmp,3.83,3.84,3.83,3.83,d689f75 +mul,2x2,gmpz,6.13,6.12,6.12,6.12,d689f75 +mul,2x2,kernel,7.59,7.58,7.56,7.56,d689f75 +mul,3x3,auto,29.69,20.25,20.29,20.29,d689f75 +mul,3x3,floor,16.63,16.42,16.39,16.39,d689f75 +mul,3x3,gmp,7.84,7.80,7.76,7.76,d689f75 +mul,3x3,gmpz,10.00,10.00,9.99,9.99,d689f75 +mul,3x3,kernel,9.02,8.95,8.93,8.93,d689f75 +mul,4x4,auto,31.98,22.34,22.43,22.43,d689f75 +mul,4x4,floor,18.58,18.20,18.25,18.25,d689f75 +mul,4x4,gmp,9.70,9.68,9.67,9.67,d689f75 +mul,4x4,gmpz,12.18,12.18,12.18,12.18,d689f75 +mul,4x4,kernel,11.11,10.91,10.91,10.91,d689f75 +mul,6x6,auto,42.06,28.02,27.98,27.98,d689f75 +mul,6x6,floor,24.71,24.38,24.45,24.45,d689f75 +mul,6x6,gmp,15.81,15.81,16.48,16.48,d689f75 +mul,6x6,gmpz,18.21,18.13,18.13,18.13,d689f75 +mul,6x6,kernel,17.49,17.16,17.19,17.19,d689f75 +mul,8x8,auto,47.50,35.87,35.96,35.96,d689f75 +mul,8x8,floor,32.62,32.34,32.56,32.56,d689f75 +mul,8x8,gmp,27.19,27.12,27.04,27.04,d689f75 +mul,8x8,gmpz,28.86,28.92,28.96,28.96,d689f75 +mul,8x8,kernel,25.54,25.41,25.45,25.45,d689f75 +mul,12x12,auto,70.25,57.88,57.39,57.39,d689f75 +mul,12x12,floor,54.98,54.75,54.92,54.92,d689f75 +mul,12x12,gmp,48.95,49.79,48.56,48.56,d689f75 +mul,12x12,gmpz,50.76,50.82,51.05,51.05,d689f75 +mul,12x12,kernel,47.82,47.76,47.89,47.89,d689f75 +mul,16x16,auto,101.81,88.48,88.63,88.63,d689f75 +mul,16x16,floor,86.43,86.25,86.81,86.81,d689f75 +mul,16x16,gmp,81.88,81.90,81.91,81.91,d689f75 +mul,16x16,gmpz,84.06,84.26,84.07,84.07,d689f75 +mul,16x16,kernel,79.33,79.09,79.25,79.25,d689f75 +mul,32x32,auto,212.30,200.50,205.77,205.77,d689f75 +mul,32x32,floor,200.22,200.31,196.46,196.46,d689f75 +mul,32x32,gmp,270.61,271.01,271.53,271.53,d689f75 +mul,32x32,gmpz,270.94,270.99,270.52,270.52,d689f75 +mul,32x32,kernel,187.59,194.34,186.84,186.84,d689f75 +mul,64x64,auto,524.45,498.34,496.22,496.22,d689f75 +mul,64x64,floor,493.43,494.74,492.51,492.51,d689f75 +mul,64x64,gmp,876.33,873.70,873.09,873.09,d689f75 +mul,64x64,gmpz,867.81,867.01,862.63,862.63,d689f75 +mul,64x64,kernel,482.98,485.58,486.68,486.68,d689f75 +mul,128x128,auto,1723.64,1724.84,1717.97,1717.97,d689f75 +mul,128x128,floor,1717.14,1712.62,1704.41,1704.41,d689f75 +mul,128x128,gmp,2763.76,2799.06,2763.24,2763.24,d689f75 +mul,128x128,gmpz,2728.55,2766.93,2733.08,2733.08,d689f75 +mul,128x128,kernel,1714.19,1700.09,1705.90,1705.90,d689f75 +mul,256x256,auto,6231.47,6185.07,6176.43,6176.43,d689f75 +mul,256x256,floor,6203.21,6180.12,6173.99,6173.99,d689f75 +mul,256x256,gmp,7896.46,7899.05,7873.70,7873.70,d689f75 +mul,256x256,gmpz,7788.68,7801.42,7853.82,7853.82,d689f75 +mul,256x256,kernel,6168.92,6169.68,6178.80,6178.80,d689f75 +mul,512x512,auto,19313.37,19192.86,19277.51,19277.51,d689f75 +mul,512x512,floor,19221.82,19253.55,19202.35,19202.35,d689f75 +mul,512x512,gmp,20954.72,20879.58,20826.25,20826.25,d689f75 +mul,512x512,gmpz,20899.69,20869.46,20884.90,20884.90,d689f75 +mul,512x512,kernel,19167.44,19182.64,19192.06,19192.06,d689f75 +mul,1000x1000,auto,59904.34,59737.02,59861.79,59861.79,d689f75 +mul,1000x1000,floor,59560.76,59569.40,59549.42,59549.42,d689f75 +mul,1000x1000,gmp,53843.90,53872.06,53342.56,53342.56,d689f75 +mul,1000x1000,gmpz,53508.52,53654.40,53437.63,53437.63,d689f75 +mul,1000x1000,kernel,59515.28,59486.68,59630.90,59630.90,d689f75 +mul,2000x100,auto,17852.92,17640.00,17643.55,17643.55,d689f75 +mul,2000x100,floor,17698.32,17695.88,17687.17,17687.17,d689f75 +mul,2000x100,gmp,36291.90,36417.26,36260.57,36260.57,d689f75 +mul,2000x100,gmpz,35918.14,36135.70,35939.70,35939.70,d689f75 +mul,2000x100,kernel,17659.32,17660.10,17696.42,17696.42,d689f75 +mul,2000x2000,auto,177072.62,176669.06,176923.73,176923.73,d689f75 +mul,2000x2000,floor,176349.76,176449.62,176724.55,176724.55,d689f75 +mul,2000x2000,gmp,144213.65,144242.57,144332.72,144332.72,d689f75 +mul,2000x2000,gmpz,144205.99,144151.67,144253.61,144253.61,d689f75 +mul,2000x2000,kernel,176380.44,176415.29,177061.13,177061.13,d689f75 +mul,16384x16384,auto,3778230.83,3774169.25,3779465.92,3779465.92,d689f75 +mul,16384x16384,floor,3782802.33,3784154.58,3793417.33,3793417.33,d689f75 +mul,16384x16384,gmp,1772153.46,1797922.67,1768834.29,1768834.29,d689f75 +mul,16384x16384,gmpz,1799441.25,1783926.00,1789186.25,1789186.25,d689f75 +mul,16384x16384,kernel,3778094.42,3781385.08,3788185.67,3788185.67,d689f75 +shl,1x13,auto,16.85,12.64,12.52,12.69,b823c5f +shl,1x13,floor,9.82,9.82,9.83,9.82,b823c5f +shl,1x13,gmp,1.93,2.08,2.08,2.09,b823c5f +shl,1x13,gmpz,3.47,3.27,3.27,3.27,b823c5f +shl,1x13,inplace,4.16,3.69,4.20,3.48,b823c5f +shl,1x13,kernel,2.08,2.08,2.08,2.08,b823c5f +shl,1x13,kernelip,1.77,1.75,1.75,1.78,b823c5f +shl,4x13,auto,18.69,13.49,13.48,13.43,b823c5f +shl,4x13,floor,11.20,11.20,11.20,11.21,b823c5f +shl,4x13,gmp,2.31,2.50,2.50,2.50,b823c5f +shl,4x13,gmpz,4.04,4.04,4.04,4.04,b823c5f +shl,4x13,inplace,5.26,4.62,5.27,4.39,b823c5f +shl,4x13,kernel,3.46,3.50,3.49,3.51,b823c5f +shl,4x13,kernelip,3.01,2.96,2.92,2.97,b823c5f +shl,16x13,auto,24.57,17.09,17.12,16.76,b823c5f +shl,16x13,floor,15.23,15.23,15.21,15.23,b823c5f +shl,16x13,gmp,4.05,4.16,4.28,4.17,b823c5f +shl,16x13,gmpz,5.82,5.82,5.82,5.81,b823c5f +shl,16x13,inplace,9.34,9.12,9.25,8.58,b823c5f +shl,16x13,kernel,7.47,7.45,7.57,7.46,b823c5f +shl,16x13,kernelip,7.20,7.13,7.17,7.11,b823c5f +shl,256x13,auto,136.95,122.03,121.97,121.81,b823c5f +shl,256x13,floor,117.22,116.52,114.62,117.80,b823c5f +shl,256x13,gmp,47.73,47.27,47.20,47.55,b823c5f +shl,256x13,gmpz,50.83,49.84,50.22,50.07,b823c5f +shl,256x13,inplace,93.09,102.25,99.30,98.67,b823c5f +shl,256x13,kernel,95.23,93.85,93.89,95.19,b823c5f +shl,256x13,kernelip,97.74,97.43,96.91,97.66,b823c5f +shl,2000x13,auto,916.70,772.77,772.71,773.42,b823c5f +shl,2000x13,floor,738.86,739.14,739.27,739.42,b823c5f +shl,2000x13,gmp,338.00,335.39,335.47,336.09,b823c5f +shl,2000x13,gmpz,340.98,338.09,338.10,339.41,b823c5f +shl,2000x13,inplace,657.71,740.91,716.88,712.53,b823c5f +shl,2000x13,kernel,719.48,719.70,719.66,719.43,b823c5f +shl,2000x13,kernelip,715.35,715.21,714.43,715.31,b823c5f +shl,2000x77,auto,1066.11,774.80,774.60,774.36,b823c5f +shl,2000x77,floor,871.86,871.91,872.60,872.78,b823c5f +shl,2000x77,gmp,342.82,337.61,337.60,342.52,b823c5f +shl,2000x77,gmpz,344.62,338.50,338.24,344.44,b823c5f +shl,2000x77,inplace,764.85,741.47,741.55,712.63,b823c5f +shl,2000x77,kernel,850.08,850.43,852.17,849.89,b823c5f +shl,2000x77,kernelip,822.53,822.03,822.03,822.04,b823c5f +shl,131072x13,auto,98078.30,,,49870.58,b823c5f +shl,131072x13,inplace,43539.38,,,46566.96,b823c5f +shl,131072x13,kernel,72255.74,,,72220.57,b823c5f +shr,1x13,auto,10.27,8.19,8.20,8.37,b823c5f +shr,1x13,floor,2.28,2.27,2.27,2.27,b823c5f +shr,1x13,gmp,1.92,1.90,1.89,1.92,b823c5f +shr,1x13,gmpz,3.27,3.47,3.47,3.27,b823c5f +shr,1x13,inplace,3.65,3.30,3.88,3.08,b823c5f +shr,1x13,kernel,1.70,1.70,1.70,1.71,b823c5f +shr,4x13,auto,20.98,16.10,16.04,15.89,b823c5f +shr,4x13,floor,11.97,11.12,11.98,11.20,b823c5f +shr,4x13,gmp,2.33,2.33,2.34,2.34,b823c5f +shr,4x13,gmpz,3.97,4.22,3.98,3.96,b823c5f +shr,4x13,inplace,4.79,4.56,5.01,4.17,b823c5f +shr,4x13,kernel,3.48,3.50,3.47,3.48,b823c5f +shr,16x13,auto,25.93,19.97,19.96,20.24,b823c5f +shr,16x13,floor,17.07,15.60,17.09,15.52,b823c5f +shr,16x13,gmp,4.47,4.42,4.31,4.29,b823c5f +shr,16x13,gmpz,6.54,6.32,6.27,6.20,b823c5f +shr,16x13,inplace,8.97,8.68,8.96,8.17,b823c5f +shr,16x13,kernel,8.21,8.86,8.68,8.49,b823c5f +shr,256x13,auto,139.67,125.46,125.03,133.67,b823c5f +shr,256x13,floor,132.19,130.39,131.00,130.77,b823c5f +shr,256x13,gmp,48.97,47.84,48.11,48.14,b823c5f +shr,256x13,gmpz,51.61,50.70,50.77,51.22,b823c5f +shr,256x13,inplace,93.66,101.68,99.09,98.26,b823c5f +shr,256x13,kernel,111.61,110.50,110.05,110.53,b823c5f +shr,2000x13,auto,950.68,750.27,749.40,787.92,b823c5f +shr,2000x13,floor,832.56,832.01,832.82,832.07,b823c5f +shr,2000x13,gmp,339.02,335.77,336.01,337.50,b823c5f +shr,2000x13,gmpz,342.53,339.01,339.01,340.95,b823c5f +shr,2000x13,inplace,657.20,738.65,716.51,711.55,b823c5f +shr,2000x13,kernel,811.82,812.04,812.30,812.03,b823c5f +shr,2000x77,auto,1084.30,750.62,749.37,789.79,b823c5f +shr,2000x77,floor,850.69,852.61,851.92,850.65,b823c5f +shr,2000x77,gmp,338.57,336.33,335.94,337.20,b823c5f +shr,2000x77,gmpz,342.97,339.08,339.06,341.49,b823c5f +shr,2000x77,inplace,764.95,740.12,740.15,712.12,b823c5f +shr,2000x77,kernel,828.38,830.24,830.10,829.29,b823c5f +shr,131072x13,auto,98907.04,,,50326.83,b823c5f +shr,131072x13,inplace,43366.77,,,46564.96,b823c5f +shr,131072x13,kernel,76162.11,,,76197.05,b823c5f +sqr,4x4,auto,33.30,22.16,22.23,22.23,d689f75 +sqr,4x4,floor,16.91,16.79,16.77,16.77,d689f75 +sqr,4x4,gmp,5.81,5.79,5.83,5.83,d689f75 +sqr,4x4,gmpz,8.83,8.84,8.84,8.84,d689f75 +sqr,4x4,kernel,9.08,9.96,9.93,9.93,d689f75 +sqr,16x16,auto,80.08,66.36,67.31,67.31,d689f75 +sqr,16x16,floor,62.62,62.63,63.09,63.09,d689f75 +sqr,16x16,gmp,50.18,50.06,50.11,50.11,d689f75 +sqr,16x16,gmpz,52.53,52.70,52.09,52.09,d689f75 +sqr,16x16,kernel,56.22,57.22,57.84,57.84,d689f75 +sqr,64x64,auto,301.50,276.11,284.36,284.36,d689f75 +sqr,64x64,floor,281.26,285.86,280.56,280.56,d689f75 +sqr,64x64,gmp,560.45,559.34,557.57,557.57,d689f75 +sqr,64x64,gmpz,557.54,556.08,555.55,555.55,d689f75 +sqr,64x64,kernel,270.75,280.22,275.25,275.25,d689f75 +sqr,512x512,auto,10627.56,10582.06,10594.10,10594.10,d689f75 +sqr,512x512,floor,10570.73,10595.58,10609.64,10609.64,d689f75 +sqr,512x512,gmp,15100.15,15082.50,15093.97,15093.97,d689f75 +sqr,512x512,gmpz,15099.66,15072.50,15071.22,15071.22,d689f75 +sqr,512x512,kernel,10562.75,10559.16,10567.09,10567.09,d689f75 +sqr,2000x2000,auto,99937.58,98205.58,99323.21,99323.21,d689f75 +sqr,2000x2000,floor,99286.24,98126.13,98255.54,98255.54,d689f75 +sqr,2000x2000,gmp,100704.68,100652.01,100608.50,100608.50,d689f75 +sqr,2000x2000,gmpz,100650.48,100656.14,100598.48,100598.48,d689f75 +sqr,2000x2000,kernel,99291.30,98188.82,98528.98,98528.98,d689f75 +sub,1x1,auto,9.02,7.76,5.47,5.47,d689f75 +sub,1x1,floor,1.90,1.89,2.05,2.05,d689f75 +sub,1x1,gmp,2.27,2.27,2.27,2.27,d689f75 +sub,1x1,gmpz,4.34,4.34,4.34,4.34,d689f75 +sub,1x1,inplace,3.98,4.68,2.46,2.46,d689f75 +sub,1x1,kernel,1.51,1.51,1.51,1.51,d689f75 +sub,2x2,auto,19.12,15.79,15.66,15.66,d689f75 +sub,2x2,floor,10.00,10.01,9.84,9.84,d689f75 +sub,2x2,gmp,2.27,2.27,2.27,2.27,d689f75 +sub,2x2,gmpz,4.36,4.35,4.34,4.34,d689f75 +sub,2x2,inplace,4.63,5.11,4.83,4.83,d689f75 +sub,2x2,kernel,2.09,2.11,2.12,2.12,d689f75 +sub,4x4,auto,19.04,16.57,16.51,16.51,d689f75 +sub,4x4,floor,10.77,10.77,10.64,10.64,d689f75 +sub,4x4,gmp,2.49,2.57,2.55,2.55,d689f75 +sub,4x4,gmpz,4.62,4.60,4.58,4.58,d689f75 +sub,4x4,inplace,5.46,5.59,5.64,5.64,d689f75 +sub,4x4,kernel,2.87,2.84,2.84,2.84,d689f75 +sub,8x8,auto,20.61,17.96,18.10,18.10,d689f75 +sub,8x8,floor,12.38,12.40,12.21,12.21,d689f75 +sub,8x8,gmp,3.68,3.69,3.69,3.69,d689f75 +sub,8x8,gmpz,4.98,5.12,4.97,4.97,d689f75 +sub,8x8,inplace,7.22,6.96,7.32,7.32,d689f75 +sub,8x8,kernel,4.37,4.38,4.36,4.36,d689f75 +sub,16x16,auto,25.70,21.06,21.05,21.05,d689f75 +sub,16x16,floor,15.44,15.39,15.32,15.32,d689f75 +sub,16x16,gmp,5.73,5.50,5.50,5.50,d689f75 +sub,16x16,gmpz,6.79,6.81,6.80,6.80,d689f75 +sub,16x16,inplace,10.60,11.18,10.04,10.04,d689f75 +sub,16x16,kernel,8.03,8.02,7.46,7.46,d689f75 +sub,64x64,auto,65.89,58.77,45.33,45.33,d689f75 +sub,64x64,floor,42.14,42.22,42.27,42.27,d689f75 +sub,64x64,gmp,15.45,15.45,15.45,15.45,d689f75 +sub,64x64,gmpz,16.74,16.74,16.78,16.78,d689f75 +sub,64x64,inplace,37.66,47.14,34.58,34.58,d689f75 +sub,64x64,kernel,32.91,32.70,32.70,32.70,d689f75 +sub,256x256,auto,223.88,215.89,162.57,162.57,d689f75 +sub,256x256,floor,158.45,158.14,158.41,158.41,d689f75 +sub,256x256,gmp,78.23,78.24,78.24,78.24,d689f75 +sub,256x256,gmpz,79.87,79.84,79.86,79.86,d689f75 +sub,256x256,inplace,145.41,195.91,144.37,144.37,d689f75 +sub,256x256,kernel,142.84,142.45,142.45,142.45,d689f75 +sub,1024x1,auto,283.39,252.84,252.26,252.26,d689f75 +sub,1024x1,floor,593.16,592.65,593.23,593.23,d689f75 +sub,1024x1,gmp,233.64,233.22,233.46,233.46,d689f75 +sub,1024x1,gmpz,234.07,234.20,234.06,234.06,d689f75 +sub,1024x1,inplace,579.78,4.77,4.54,4.54,d689f75 +sub,1024x1,kernel,577.26,576.96,576.50,576.50,d689f75 +sub,1024x1024,auto,842.25,808.58,598.27,598.27,d689f75 +sub,1024x1024,floor,594.39,593.87,594.12,594.12,d689f75 +sub,1024x1024,gmp,390.09,389.76,389.83,389.83,d689f75 +sub,1024x1024,gmpz,392.17,391.22,391.20,391.20,d689f75 +sub,1024x1024,inplace,581.54,791.55,579.87,579.87,d689f75 +sub,1024x1024,kernel,578.22,577.78,577.76,577.76,d689f75 +sub,2000x2000,auto,1741.80,1549.24,1154.12,1154.12,d689f75 +sub,2000x2000,floor,1150.28,1149.14,1149.33,1149.33,d689f75 +sub,2000x2000,gmp,466.64,473.97,477.67,477.67,d689f75 +sub,2000x2000,gmpz,489.71,482.96,486.71,486.71,d689f75 +sub,2000x2000,inplace,1134.18,1519.97,1132.27,1132.27,d689f75 +sub,2000x2000,kernel,1132.39,1132.12,1131.77,1131.77,d689f75 +tochars,1x10,auto,14.68,,14.87,14.87,d689f75 +tochars,1x10,gmp,20.36,,20.16,20.16,d689f75 +tochars,1x10,gmpz,33.83,,33.86,33.86,d689f75 +tochars,1x10,kernel,63.20,,62.88,62.88,d689f75 +tochars,4x10,auto,134.06,,132.03,132.03,d689f75 +tochars,4x10,gmp,70.12,,70.23,70.23,d689f75 +tochars,4x10,gmpz,97.09,,97.60,97.60,d689f75 +tochars,4x10,kernel,180.98,,180.91,180.91,d689f75 +tochars,16x10,auto,1171.44,,1168.06,1168.06,d689f75 +tochars,16x10,gmp,503.67,,503.70,503.70,d689f75 +tochars,16x10,gmpz,590.00,,590.42,590.42,d689f75 +tochars,16x10,kernel,1039.05,,1032.73,1032.73,d689f75 +tochars,16x16,auto,120.09,,104.75,104.75,d689f75 +tochars,16x16,gmp,97.91,,98.80,98.80,d689f75 +tochars,16x16,gmpz,157.27,,157.75,157.75,d689f75 +tochars,64x10,auto,5622.48,,5695.35,5695.35,d689f75 +tochars,64x10,gmp,2665.88,,2665.86,2665.86,d689f75 +tochars,64x10,gmpz,2955.70,,2955.16,2955.16,d689f75 +tochars,64x10,kernel,5245.07,,5332.91,5332.91,d689f75 +tochars,256x10,auto,35873.38,,36990.15,36990.15,d689f75 +tochars,256x10,gmp,18584.43,,18578.85,18578.85,d689f75 +tochars,256x10,gmpz,19672.97,,19669.58,19669.58,d689f75 +tochars,256x10,kernel,34589.72,,35528.75,35528.75,d689f75 +tochars,2000x10,auto,608668.24,,620642.68,620642.68,d689f75 +tochars,2000x10,gmp,400290.87,,400888.47,400888.47,d689f75 +tochars,2000x10,gmpz,409111.46,,409231.54,409231.54,d689f75 +tochars,2000x10,kernel,584312.52,,597544.51,597544.51,d689f75 +tochars,2000x16,auto,13619.85,,11932.17,11932.17,d689f75 +tochars,2000x16,gmp,11111.97,,11111.39,11111.39,d689f75 +tochars,2000x16,gmpz,17926.81,,17928.44,17928.44,d689f75 +tochars,10000x10,auto,5864746.12,5879015.75,5960832.12,5960832.12,d689f75 +tochars,10000x10,gmp,4186340.50,4182821.50,4185719.20,4185719.20,d689f75 +tochars,10000x10,gmpz,4226138.30,4221163.20,4223942.60,4223942.60,d689f75 +tochars,10000x10,kernel,5844017.50,5846314.75,5920813.38,5920813.38,d689f75 diff --git a/tests/beman/big_int/perf/item1/shift_ab.csv b/tests/beman/big_int/perf/item1/shift_ab.csv new file mode 100644 index 00000000..4526d973 --- /dev/null +++ b/tests/beman/big_int/perf/item1/shift_ab.csv @@ -0,0 +1,424 @@ +# in-place shift A/B (variants built from scratch copies of d689f75; new64 = d689f75, v1 = revert of 2f9150b, +# v2 = v1 + register-carried lshift_copy/rshift_copy loops, v3 = v2 with funnel_shl/funnel_shr); ns per operation, +# median of three interleaved runs; same compiler and flags as new64 (see final_*.csv) +machine,variant,op,shape,row,ns +x64,base64,shl,1x13,auto,16.74 +x64,base64,shl,1x13,inplace,4.07 +x64,base64,shl,1x13,kernel,2.08 +x64,base64,shl,4x13,auto,18.62 +x64,base64,shl,4x13,inplace,5.16 +x64,base64,shl,4x13,kernel,3.39 +x64,base64,shl,16x13,auto,24.54 +x64,base64,shl,16x13,inplace,9.16 +x64,base64,shl,16x13,kernel,7.47 +x64,base64,shl,256x13,auto,133.92 +x64,base64,shl,256x13,inplace,93.05 +x64,base64,shl,256x13,kernel,93.47 +x64,base64,shl,2000x13,auto,917.53 +x64,base64,shl,2000x13,inplace,656.70 +x64,base64,shl,2000x13,kernel,718.92 +x64,base64,shl,2000x77,auto,1063.41 +x64,base64,shl,2000x77,inplace,765.66 +x64,base64,shl,2000x77,kernel,852.05 +x64,base64,shl,131072x13,auto,97977.28 +x64,base64,shl,131072x13,inplace,43553.83 +x64,base64,shl,131072x13,kernel,72261.65 +x64,base64,shr,1x13,auto,10.12 +x64,base64,shr,1x13,inplace,3.65 +x64,base64,shr,1x13,kernel,1.70 +x64,base64,shr,4x13,auto,20.94 +x64,base64,shr,4x13,inplace,4.71 +x64,base64,shr,4x13,kernel,3.45 +x64,base64,shr,16x13,auto,25.78 +x64,base64,shr,16x13,inplace,8.72 +x64,base64,shr,16x13,kernel,8.11 +x64,base64,shr,256x13,auto,136.61 +x64,base64,shr,256x13,inplace,93.71 +x64,base64,shr,256x13,kernel,110.21 +x64,base64,shr,2000x13,auto,949.92 +x64,base64,shr,2000x13,inplace,657.15 +x64,base64,shr,2000x13,kernel,811.90 +x64,base64,shr,2000x77,auto,1086.85 +x64,base64,shr,2000x77,inplace,765.54 +x64,base64,shr,2000x77,kernel,831.59 +x64,base64,shr,131072x13,auto,98933.21 +x64,base64,shr,131072x13,inplace,43515.12 +x64,base64,shr,131072x13,kernel,76088.66 +x64,new64,shl,1x13,auto,12.52 +x64,new64,shl,1x13,inplace,4.19 +x64,new64,shl,1x13,kernel,2.08 +x64,new64,shl,4x13,auto,13.42 +x64,new64,shl,4x13,inplace,5.24 +x64,new64,shl,4x13,kernel,3.44 +x64,new64,shl,16x13,auto,17.07 +x64,new64,shl,16x13,inplace,9.24 +x64,new64,shl,16x13,kernel,7.59 +x64,new64,shl,256x13,auto,122.03 +x64,new64,shl,256x13,inplace,99.32 +x64,new64,shl,256x13,kernel,93.56 +x64,new64,shl,2000x13,auto,772.75 +x64,new64,shl,2000x13,inplace,716.89 +x64,new64,shl,2000x13,kernel,718.69 +x64,new64,shl,2000x77,auto,774.60 +x64,new64,shl,2000x77,inplace,741.66 +x64,new64,shl,2000x77,kernel,851.94 +x64,new64,shl,131072x13,auto,49725.22 +x64,new64,shl,131072x13,inplace,46688.97 +x64,new64,shl,131072x13,kernel,72200.14 +x64,new64,shr,1x13,auto,8.19 +x64,new64,shr,1x13,inplace,3.89 +x64,new64,shr,1x13,kernel,1.70 +x64,new64,shr,4x13,auto,16.06 +x64,new64,shr,4x13,inplace,4.94 +x64,new64,shr,4x13,kernel,3.46 +x64,new64,shr,16x13,auto,19.99 +x64,new64,shr,16x13,inplace,8.89 +x64,new64,shr,16x13,kernel,8.33 +x64,new64,shr,256x13,auto,124.95 +x64,new64,shr,256x13,inplace,99.08 +x64,new64,shr,256x13,kernel,110.03 +x64,new64,shr,2000x13,auto,748.99 +x64,new64,shr,2000x13,inplace,716.48 +x64,new64,shr,2000x13,kernel,812.11 +x64,new64,shr,2000x77,auto,749.55 +x64,new64,shr,2000x77,inplace,740.83 +x64,new64,shr,2000x77,kernel,830.57 +x64,new64,shr,131072x13,auto,48304.43 +x64,new64,shr,131072x13,inplace,46701.23 +x64,new64,shr,131072x13,kernel,76237.49 +x64,v1,shl,1x13,auto,12.56 +x64,v1,shl,1x13,inplace,3.63 +x64,v1,shl,1x13,kernel,2.08 +x64,v1,shl,4x13,auto,13.44 +x64,v1,shl,4x13,inplace,4.55 +x64,v1,shl,4x13,kernel,3.45 +x64,v1,shl,16x13,auto,17.00 +x64,v1,shl,16x13,inplace,9.09 +x64,v1,shl,16x13,kernel,7.46 +x64,v1,shl,256x13,auto,122.25 +x64,v1,shl,256x13,inplace,102.16 +x64,v1,shl,256x13,kernel,93.44 +x64,v1,shl,2000x13,auto,773.42 +x64,v1,shl,2000x13,inplace,738.99 +x64,v1,shl,2000x13,kernel,718.92 +x64,v1,shl,2000x77,auto,775.00 +x64,v1,shl,2000x77,inplace,739.01 +x64,v1,shl,2000x77,kernel,853.38 +x64,v1,shl,131072x13,auto,49751.88 +x64,v1,shl,131072x13,inplace,48260.24 +x64,v1,shl,131072x13,kernel,72243.32 +x64,v1,shr,1x13,auto,8.20 +x64,v1,shr,1x13,inplace,3.28 +x64,v1,shr,1x13,kernel,1.70 +x64,v1,shr,4x13,auto,16.04 +x64,v1,shr,4x13,inplace,4.33 +x64,v1,shr,4x13,kernel,3.47 +x64,v1,shr,16x13,auto,20.03 +x64,v1,shr,16x13,inplace,8.66 +x64,v1,shr,16x13,kernel,8.37 +x64,v1,shr,256x13,auto,125.18 +x64,v1,shr,256x13,inplace,101.65 +x64,v1,shr,256x13,kernel,110.51 +x64,v1,shr,2000x13,auto,749.58 +x64,v1,shr,2000x13,inplace,737.36 +x64,v1,shr,2000x13,kernel,812.13 +x64,v1,shr,2000x77,auto,750.01 +x64,v1,shr,2000x77,inplace,739.27 +x64,v1,shr,2000x77,kernel,829.41 +x64,v1,shr,131072x13,auto,48328.01 +x64,v1,shr,131072x13,inplace,48481.13 +x64,v1,shr,131072x13,kernel,76186.38 +x64,v2,shl,1x13,auto,12.56 +x64,v2,shl,1x13,inplace,3.44 +x64,v2,shl,1x13,kernel,2.08 +x64,v2,shl,4x13,auto,13.39 +x64,v2,shl,4x13,inplace,4.32 +x64,v2,shl,4x13,kernel,3.45 +x64,v2,shl,16x13,auto,16.72 +x64,v2,shl,16x13,inplace,8.41 +x64,v2,shl,16x13,kernel,7.46 +x64,v2,shl,256x13,auto,122.03 +x64,v2,shl,256x13,inplace,98.73 +x64,v2,shl,256x13,kernel,93.44 +x64,v2,shl,2000x13,auto,773.57 +x64,v2,shl,2000x13,inplace,712.55 +x64,v2,shl,2000x13,kernel,718.04 +x64,v2,shl,2000x77,auto,774.54 +x64,v2,shl,2000x77,inplace,712.51 +x64,v2,shl,2000x77,kernel,852.72 +x64,v2,shl,131072x13,auto,49864.19 +x64,v2,shl,131072x13,inplace,46565.61 +x64,v2,shl,131072x13,kernel,72309.63 +x64,v2,shr,1x13,auto,8.35 +x64,v2,shr,1x13,inplace,3.02 +x64,v2,shr,1x13,kernel,1.70 +x64,v2,shr,4x13,auto,15.85 +x64,v2,shr,4x13,inplace,4.11 +x64,v2,shr,4x13,kernel,3.47 +x64,v2,shr,16x13,auto,20.24 +x64,v2,shr,16x13,inplace,8.00 +x64,v2,shr,16x13,kernel,8.39 +x64,v2,shr,256x13,auto,133.62 +x64,v2,shr,256x13,inplace,98.31 +x64,v2,shr,256x13,kernel,110.39 +x64,v2,shr,2000x13,auto,787.97 +x64,v2,shr,2000x13,inplace,711.55 +x64,v2,shr,2000x13,kernel,811.93 +x64,v2,shr,2000x77,auto,790.02 +x64,v2,shr,2000x77,inplace,712.12 +x64,v2,shr,2000x77,kernel,830.58 +x64,v2,shr,131072x13,auto,50322.43 +x64,v2,shr,131072x13,inplace,46567.50 +x64,v2,shr,131072x13,kernel,76120.43 +x64,v3,shl,1x13,auto,12.63 +x64,v3,shl,1x13,inplace,3.44 +x64,v3,shl,1x13,kernel,2.08 +x64,v3,shl,4x13,auto,13.41 +x64,v3,shl,4x13,inplace,4.33 +x64,v3,shl,4x13,kernel,3.46 +x64,v3,shl,16x13,auto,16.75 +x64,v3,shl,16x13,inplace,8.42 +x64,v3,shl,16x13,kernel,7.47 +x64,v3,shl,256x13,auto,121.89 +x64,v3,shl,256x13,inplace,98.69 +x64,v3,shl,256x13,kernel,93.42 +x64,v3,shl,2000x13,auto,773.53 +x64,v3,shl,2000x13,inplace,712.63 +x64,v3,shl,2000x13,kernel,718.95 +x64,v3,shl,2000x77,auto,774.47 +x64,v3,shl,2000x77,inplace,712.49 +x64,v3,shl,2000x77,kernel,851.80 +x64,v3,shl,131072x13,auto,49877.81 +x64,v3,shl,131072x13,inplace,46554.86 +x64,v3,shl,131072x13,kernel,72281.85 +x64,v3,shr,1x13,auto,8.35 +x64,v3,shr,1x13,inplace,3.03 +x64,v3,shr,1x13,kernel,1.70 +x64,v3,shr,4x13,auto,15.84 +x64,v3,shr,4x13,inplace,4.11 +x64,v3,shr,4x13,kernel,3.47 +x64,v3,shr,16x13,auto,20.25 +x64,v3,shr,16x13,inplace,8.00 +x64,v3,shr,16x13,kernel,8.69 +x64,v3,shr,256x13,auto,133.70 +x64,v3,shr,256x13,inplace,98.34 +x64,v3,shr,256x13,kernel,110.38 +x64,v3,shr,2000x13,auto,788.00 +x64,v3,shr,2000x13,inplace,711.67 +x64,v3,shr,2000x13,kernel,812.03 +x64,v3,shr,2000x77,auto,790.04 +x64,v3,shr,2000x77,inplace,712.13 +x64,v3,shr,2000x77,kernel,830.63 +x64,v3,shr,131072x13,auto,50313.51 +x64,v3,shr,131072x13,inplace,46580.59 +x64,v3,shr,131072x13,kernel,76160.78 +m4,base64,shl,1x13,auto,18.53 +m4,base64,shl,1x13,inplace,2.29 +m4,base64,shl,1x13,kernel,2.23 +m4,base64,shl,4x13,auto,19.92 +m4,base64,shl,4x13,inplace,3.57 +m4,base64,shl,4x13,kernel,3.72 +m4,base64,shl,16x13,auto,21.44 +m4,base64,shl,16x13,inplace,5.80 +m4,base64,shl,16x13,kernel,4.45 +m4,base64,shl,256x13,auto,85.63 +m4,base64,shl,256x13,inplace,36.24 +m4,base64,shl,256x13,kernel,48.88 +m4,base64,shl,2000x13,auto,638.95 +m4,base64,shl,2000x13,inplace,282.28 +m4,base64,shl,2000x13,kernel,382.89 +m4,base64,shl,2000x77,auto,806.23 +m4,base64,shl,2000x77,inplace,424.68 +m4,base64,shl,2000x77,kernel,528.20 +m4,base64,shl,131072x13,auto,48761.66 +m4,base64,shl,131072x13,inplace,19766.79 +m4,base64,shl,131072x13,kernel,29280.08 +m4,base64,shr,1x13,auto,9.18 +m4,base64,shr,1x13,inplace,2.36 +m4,base64,shr,1x13,kernel,2.23 +m4,base64,shr,4x13,auto,18.41 +m4,base64,shr,4x13,inplace,3.27 +m4,base64,shr,4x13,kernel,3.72 +m4,base64,shr,16x13,auto,21.49 +m4,base64,shr,16x13,inplace,5.12 +m4,base64,shr,16x13,kernel,4.46 +m4,base64,shr,256x13,auto,87.11 +m4,base64,shr,256x13,inplace,35.79 +m4,base64,shr,256x13,kernel,66.65 +m4,base64,shr,2000x13,auto,657.71 +m4,base64,shr,2000x13,inplace,282.03 +m4,base64,shr,2000x13,kernel,512.16 +m4,base64,shr,2000x77,auto,1015.75 +m4,base64,shr,2000x77,inplace,420.07 +m4,base64,shr,2000x77,kernel,517.38 +m4,base64,shr,131072x13,auto,50334.06 +m4,base64,shr,131072x13,inplace,19886.37 +m4,base64,shr,131072x13,kernel,37482.91 +m4,new64,shl,1x13,auto,12.23 +m4,new64,shl,1x13,inplace,3.38 +m4,new64,shl,1x13,kernel,2.48 +m4,new64,shl,4x13,auto,12.69 +m4,new64,shl,4x13,inplace,5.20 +m4,new64,shl,4x13,kernel,3.72 +m4,new64,shl,16x13,auto,15.73 +m4,new64,shl,16x13,inplace,6.83 +m4,new64,shl,16x13,kernel,4.44 +m4,new64,shl,256x13,auto,41.28 +m4,new64,shl,256x13,inplace,44.92 +m4,new64,shl,256x13,kernel,48.66 +m4,new64,shl,2000x13,auto,263.97 +m4,new64,shl,2000x13,inplace,345.72 +m4,new64,shl,2000x13,kernel,383.95 +m4,new64,shl,2000x77,auto,342.48 +m4,new64,shl,2000x77,inplace,228.58 +m4,new64,shl,2000x77,kernel,526.33 +m4,new64,shl,131072x13,auto,18032.30 +m4,new64,shl,131072x13,inplace,22824.97 +m4,new64,shl,131072x13,kernel,29301.55 +m4,new64,shr,1x13,auto,6.73 +m4,new64,shr,1x13,inplace,3.06 +m4,new64,shr,1x13,kernel,2.23 +m4,new64,shr,4x13,auto,14.32 +m4,new64,shr,4x13,inplace,4.47 +m4,new64,shr,4x13,kernel,3.72 +m4,new64,shr,16x13,auto,18.06 +m4,new64,shr,16x13,inplace,6.11 +m4,new64,shr,16x13,kernel,4.19 +m4,new64,shr,256x13,auto,42.47 +m4,new64,shr,256x13,inplace,44.60 +m4,new64,shr,256x13,kernel,66.93 +m4,new64,shr,2000x13,auto,262.02 +m4,new64,shr,2000x13,inplace,345.02 +m4,new64,shr,2000x13,kernel,511.78 +m4,new64,shr,2000x77,auto,251.83 +m4,new64,shr,2000x77,inplace,195.76 +m4,new64,shr,2000x77,kernel,516.14 +m4,new64,shr,131072x13,auto,18098.33 +m4,new64,shr,131072x13,inplace,22564.14 +m4,new64,shr,131072x13,kernel,37480.22 +m4,v1,shl,1x13,auto,12.53 +m4,v1,shl,1x13,inplace,2.48 +m4,v1,shl,1x13,kernel,2.23 +m4,v1,shl,4x13,auto,12.97 +m4,v1,shl,4x13,inplace,3.76 +m4,v1,shl,4x13,kernel,3.72 +m4,v1,shl,16x13,auto,15.49 +m4,v1,shl,16x13,inplace,5.28 +m4,v1,shl,16x13,kernel,4.43 +m4,v1,shl,256x13,auto,41.91 +m4,v1,shl,256x13,inplace,29.68 +m4,v1,shl,256x13,kernel,48.78 +m4,v1,shl,2000x13,auto,266.98 +m4,v1,shl,2000x13,inplace,196.09 +m4,v1,shl,2000x13,kernel,382.87 +m4,v1,shl,2000x77,auto,477.96 +m4,v1,shl,2000x77,inplace,230.90 +m4,v1,shl,2000x77,kernel,528.46 +m4,v1,shl,131072x13,auto,18150.93 +m4,v1,shl,131072x13,inplace,18524.16 +m4,v1,shl,131072x13,kernel,29306.12 +m4,v1,shr,1x13,auto,6.88 +m4,v1,shr,1x13,inplace,3.31 +m4,v1,shr,1x13,kernel,2.23 +m4,v1,shr,4x13,auto,15.08 +m4,v1,shr,4x13,inplace,3.69 +m4,v1,shr,4x13,kernel,3.72 +m4,v1,shr,16x13,auto,18.73 +m4,v1,shr,16x13,inplace,5.19 +m4,v1,shr,16x13,kernel,4.50 +m4,v1,shr,256x13,auto,43.13 +m4,v1,shr,256x13,inplace,30.37 +m4,v1,shr,256x13,kernel,67.43 +m4,v1,shr,2000x13,auto,258.68 +m4,v1,shr,2000x13,inplace,197.75 +m4,v1,shr,2000x13,kernel,511.46 +m4,v1,shr,2000x77,auto,266.65 +m4,v1,shr,2000x77,inplace,195.66 +m4,v1,shr,2000x77,kernel,519.34 +m4,v1,shr,131072x13,auto,18178.09 +m4,v1,shr,131072x13,inplace,18551.56 +m4,v1,shr,131072x13,kernel,37569.55 +m4,v2,shl,1x13,auto,12.26 +m4,v2,shl,1x13,inplace,2.48 +m4,v2,shl,1x13,kernel,2.23 +m4,v2,shl,4x13,auto,13.25 +m4,v2,shl,4x13,inplace,3.71 +m4,v2,shl,4x13,kernel,3.72 +m4,v2,shl,16x13,auto,15.53 +m4,v2,shl,16x13,inplace,6.34 +m4,v2,shl,16x13,kernel,4.21 +m4,v2,shl,256x13,auto,56.30 +m4,v2,shl,256x13,inplace,40.11 +m4,v2,shl,256x13,kernel,48.74 +m4,v2,shl,2000x13,auto,395.31 +m4,v2,shl,2000x13,inplace,315.71 +m4,v2,shl,2000x13,kernel,383.72 +m4,v2,shl,2000x77,auto,394.79 +m4,v2,shl,2000x77,inplace,315.62 +m4,v2,shl,2000x77,kernel,529.17 +m4,v2,shl,131072x13,auto,25035.87 +m4,v2,shl,131072x13,inplace,20843.90 +m4,v2,shl,131072x13,kernel,29489.87 +m4,v2,shr,1x13,auto,6.78 +m4,v2,shr,1x13,inplace,2.75 +m4,v2,shr,1x13,kernel,2.23 +m4,v2,shr,4x13,auto,14.01 +m4,v2,shr,4x13,inplace,3.31 +m4,v2,shr,4x13,kernel,3.72 +m4,v2,shr,16x13,auto,18.09 +m4,v2,shr,16x13,inplace,5.51 +m4,v2,shr,16x13,kernel,4.47 +m4,v2,shr,256x13,auto,43.56 +m4,v2,shr,256x13,inplace,39.66 +m4,v2,shr,256x13,kernel,67.35 +m4,v2,shr,2000x13,auto,289.09 +m4,v2,shr,2000x13,inplace,313.60 +m4,v2,shr,2000x13,kernel,512.05 +m4,v2,shr,2000x77,auto,299.85 +m4,v2,shr,2000x77,inplace,314.75 +m4,v2,shr,2000x77,kernel,518.82 +m4,v2,shr,131072x13,auto,18015.52 +m4,v2,shr,131072x13,inplace,20782.31 +m4,v2,shr,131072x13,kernel,37504.61 +m4,v3,shl,1x13,auto,12.09 +m4,v3,shl,1x13,inplace,2.48 +m4,v3,shl,1x13,kernel,2.23 +m4,v3,shl,4x13,auto,13.12 +m4,v3,shl,4x13,inplace,3.83 +m4,v3,shl,4x13,kernel,3.72 +m4,v3,shl,16x13,auto,16.26 +m4,v3,shl,16x13,inplace,6.49 +m4,v3,shl,16x13,kernel,4.08 +m4,v3,shl,256x13,auto,63.05 +m4,v3,shl,256x13,inplace,47.97 +m4,v3,shl,256x13,kernel,48.37 +m4,v3,shl,2000x13,auto,453.04 +m4,v3,shl,2000x13,inplace,375.19 +m4,v3,shl,2000x13,kernel,382.49 +m4,v3,shl,2000x77,auto,452.87 +m4,v3,shl,2000x77,inplace,377.11 +m4,v3,shl,2000x77,kernel,526.90 +m4,v3,shl,131072x13,auto,28931.40 +m4,v3,shl,131072x13,inplace,24609.09 +m4,v3,shl,131072x13,kernel,29322.43 +m4,v3,shr,1x13,auto,6.68 +m4,v3,shr,1x13,inplace,2.78 +m4,v3,shr,1x13,kernel,2.23 +m4,v3,shr,4x13,auto,14.27 +m4,v3,shr,4x13,inplace,3.54 +m4,v3,shr,4x13,kernel,3.72 +m4,v3,shr,16x13,auto,18.23 +m4,v3,shr,16x13,inplace,5.66 +m4,v3,shr,16x13,kernel,4.17 +m4,v3,shr,256x13,auto,48.31 +m4,v3,shr,256x13,inplace,47.30 +m4,v3,shr,256x13,kernel,67.19 +m4,v3,shr,2000x13,auto,340.82 +m4,v3,shr,2000x13,inplace,374.14 +m4,v3,shr,2000x13,kernel,510.56 +m4,v3,shr,2000x77,auto,339.25 +m4,v3,shr,2000x77,inplace,377.89 +m4,v3,shr,2000x77,kernel,517.15 +m4,v3,shr,131072x13,auto,20633.50 +m4,v3,shr,131072x13,inplace,24608.06 +m4,v3,shr,131072x13,kernel,37720.75 diff --git a/tests/beman/big_int/perf/item1/shortcut.csv b/tests/beman/big_int/perf/item1/shortcut.csv new file mode 100644 index 00000000..587cdb81 --- /dev/null +++ b/tests/beman/big_int/perf/item1/shortcut.csv @@ -0,0 +1,39 @@ +# mul_header_basecase_enabled A/B (opt_1 at 9145b4e; the 'off' build is a scratch copy with the constant +# set to false, not committed); ns per operation, median of three interleaved runs +machine,op,shape,row,on_ns,off_ns +x64,mul,2x2,auto,18.94,22.72 +x64,mul,2x2,kernel,7.64,7.61 +x64,mul,3x3,auto,20.29,24.60 +x64,mul,3x3,kernel,8.94,8.93 +x64,mul,4x4,auto,22.34,26.33 +x64,mul,4x4,kernel,10.95,10.93 +x64,mul,6x6,auto,27.99,32.77 +x64,mul,6x6,kernel,17.15,17.17 +x64,mul,8x8,auto,35.68,41.15 +x64,mul,8x8,kernel,25.65,25.43 +x64,mul,12x12,auto,57.71,62.58 +x64,mul,12x12,kernel,47.56,47.55 +x64,mul,16x16,auto,95.03,115.51 +x64,mul,16x16,kernel,88.15,78.91 +x64,sqr,4x4,auto,22.03,27.32 +x64,sqr,4x4,kernel,9.85,9.89 +x64,sqr,16x16,auto,68.76,72.77 +x64,sqr,16x16,kernel,57.25,57.44 +m4,mul,2x2,auto,19.88,19.41 +m4,mul,2x2,kernel,6.31,6.50 +m4,mul,3x3,auto,20.27,21.23 +m4,mul,3x3,kernel,7.95,8.20 +m4,mul,4x4,auto,21.49,22.31 +m4,mul,4x4,kernel,10.84,10.79 +m4,mul,6x6,auto,27.33,28.07 +m4,mul,6x6,kernel,19.45,19.19 +m4,mul,8x8,auto,34.75,35.82 +m4,mul,8x8,kernel,22.97,23.49 +m4,mul,12x12,auto,54.33,54.46 +m4,mul,12x12,kernel,41.05,41.63 +m4,mul,16x16,auto,82.77,83.43 +m4,mul,16x16,kernel,69.72,70.12 +m4,sqr,4x4,auto,21.47,23.16 +m4,sqr,4x4,kernel,11.26,11.27 +m4,sqr,16x16,auto,76.07,77.95 +m4,sqr,16x16,kernel,63.71,60.90 diff --git a/tests/beman/big_int/perf/item1/study.csv b/tests/beman/big_int/perf/item1/study.csv new file mode 100644 index 00000000..477debe9 --- /dev/null +++ b/tests/beman/big_int/perf/item1/study.csv @@ -0,0 +1,1332 @@ +# inline-capacity study (opt_1 at 9145b4e, before the add/sub/shift/single-limb follow-ups); ns per operation +# (vecsort: ns per element for 100000 one-limb values, rows auto/builtin/copy); median of two runs (seeds 1 and 2, +# divrem the mean of seeds); small band, add/sub/shl/shr/mul/sqr/divrem +# x64 N=64: sweep_inline_bits=64 sweep_int_sizeof=16 ARCH=x86_64 BMI2_ADX=1 AVX512_IFMA=1 SIMD_MUL=1 limb_bits=64 ndebug=1 compiler=gcc-14.3.0 build_type=RelWithDebInfo cxx_flags=-march=native_-O3__-O2_-g_-DNDEBUG_defs:BEMAN_BIG_INT_SIMD_MUL +# x64 N=128: sweep_inline_bits=128 sweep_int_sizeof=24 ARCH=x86_64 BMI2_ADX=1 AVX512_IFMA=1 SIMD_MUL=1 limb_bits=64 ndebug=1 compiler=gcc-14.3.0 build_type=RelWithDebInfo cxx_flags=-march=native_-O3__-O2_-g_-DNDEBUG_defs:BEMAN_BIG_INT_SIMD_MUL +# x64 N=256: sweep_inline_bits=256 sweep_int_sizeof=40 ARCH=x86_64 BMI2_ADX=1 AVX512_IFMA=1 SIMD_MUL=1 limb_bits=64 ndebug=1 compiler=gcc-14.3.0 build_type=RelWithDebInfo cxx_flags=-march=native_-O3__-O2_-g_-DNDEBUG_defs:BEMAN_BIG_INT_SIMD_MUL +# x64 N=512: sweep_inline_bits=512 sweep_int_sizeof=72 ARCH=x86_64 BMI2_ADX=1 AVX512_IFMA=1 SIMD_MUL=1 limb_bits=64 ndebug=1 compiler=gcc-14.3.0 build_type=RelWithDebInfo cxx_flags=-march=native_-O3__-O2_-g_-DNDEBUG_defs:BEMAN_BIG_INT_SIMD_MUL +# m4 N=64: sweep_inline_bits=64 sweep_int_sizeof=16 ARCH=aarch64 BMI2_ADX=0 AVX512_IFMA=0 SIMD_MUL=1 limb_bits=64 ndebug=1 compiler=appleclang-21.0.0_(clang-2100.3.34.2) build_type=RelWithDebInfo cxx_flags=-O3__-O2_-g_-DNDEBUG_defs:BEMAN_BIG_INT_SIMD_MUL +# m4 N=128: sweep_inline_bits=128 sweep_int_sizeof=24 ARCH=aarch64 BMI2_ADX=0 AVX512_IFMA=0 SIMD_MUL=1 limb_bits=64 ndebug=1 compiler=appleclang-21.0.0_(clang-2100.3.34.2) build_type=RelWithDebInfo cxx_flags=-O3__-O2_-g_-DNDEBUG_defs:BEMAN_BIG_INT_SIMD_MUL +# m4 N=256: sweep_inline_bits=256 sweep_int_sizeof=40 ARCH=aarch64 BMI2_ADX=0 AVX512_IFMA=0 SIMD_MUL=1 limb_bits=64 ndebug=1 compiler=appleclang-21.0.0_(clang-2100.3.34.2) build_type=RelWithDebInfo cxx_flags=-O3__-O2_-g_-DNDEBUG_defs:BEMAN_BIG_INT_SIMD_MUL +# m4 N=512: sweep_inline_bits=512 sweep_int_sizeof=72 ARCH=aarch64 BMI2_ADX=0 AVX512_IFMA=0 SIMD_MUL=1 limb_bits=64 ndebug=1 compiler=appleclang-21.0.0_(clang-2100.3.34.2) build_type=RelWithDebInfo cxx_flags=-O3__-O2_-g_-DNDEBUG_defs:BEMAN_BIG_INT_SIMD_MUL +machine,inline_bits,op,shape,row,ns +x64,64,add,1x1,auto,6.66 +x64,64,add,1x1,floor,9.80 +x64,64,add,1x1,gmp,2.27 +x64,64,add,1x1,gmpz,4.16 +x64,64,add,1x1,inplace,4.91 +x64,64,add,1x1,kernel,1.92 +x64,64,add,2x2,auto,15.89 +x64,64,add,2x2,floor,10.23 +x64,64,add,2x2,gmp,2.27 +x64,64,add,2x2,gmpz,4.16 +x64,64,add,2x2,inplace,6.12 +x64,64,add,2x2,kernel,2.40 +x64,64,add,4x4,auto,15.41 +x64,64,add,4x4,floor,11.12 +x64,64,add,4x4,gmp,2.50 +x64,64,add,4x4,gmpz,4.36 +x64,64,add,4x4,inplace,5.72 +x64,64,add,4x4,kernel,3.08 +x64,64,add,8x8,auto,16.75 +x64,64,add,8x8,floor,12.51 +x64,64,add,8x8,gmp,3.68 +x64,64,add,8x8,gmpz,4.66 +x64,64,add,8x8,inplace,7.09 +x64,64,add,8x8,kernel,4.53 +x64,64,add,16x16,auto,20.97 +x64,64,add,16x16,floor,15.45 +x64,64,add,16x16,gmp,5.58 +x64,64,add,16x16,gmpz,7.13 +x64,64,add,16x16,inplace,11.75 +x64,64,add,16x16,kernel,7.56 +x64,64,divrem,2x1,auto,35.38 +x64,64,divrem,2x1,floor,12.80 +x64,64,divrem,2x1,gmp,7.69 +x64,64,divrem,2x1,gmpz,12.61 +x64,64,divrem,2x1,kernel,5.71 +x64,64,divrem,8x4,auto,91.89 +x64,64,divrem,8x4,floor,67.21 +x64,64,divrem,8x4,gmp,42.30 +x64,64,divrem,8x4,gmpz,44.27 +x64,64,divrem,8x4,kernel,49.53 +x64,64,divrem,32x16,auto,339.57 +x64,64,divrem,32x16,floor,305.70 +x64,64,divrem,32x16,gmp,190.93 +x64,64,divrem,32x16,gmpz,193.06 +x64,64,divrem,32x16,kernel,290.44 +x64,64,mul,1x1,auto,16.48 +x64,64,mul,1x1,gmp,3.37 +x64,64,mul,1x1,gmpz,4.00 +x64,64,mul,2x2,auto,18.93 +x64,64,mul,2x2,floor,15.00 +x64,64,mul,2x2,gmp,3.91 +x64,64,mul,2x2,gmpz,6.12 +x64,64,mul,2x2,kernel,7.61 +x64,64,mul,3x3,auto,20.24 +x64,64,mul,3x3,floor,16.34 +x64,64,mul,3x3,gmp,7.73 +x64,64,mul,3x3,gmpz,9.98 +x64,64,mul,3x3,kernel,8.93 +x64,64,mul,4x4,auto,22.30 +x64,64,mul,4x4,floor,18.23 +x64,64,mul,4x4,gmp,9.94 +x64,64,mul,4x4,gmpz,12.46 +x64,64,mul,4x4,kernel,10.94 +x64,64,mul,6x6,auto,28.04 +x64,64,mul,6x6,floor,24.35 +x64,64,mul,6x6,gmp,15.69 +x64,64,mul,6x6,gmpz,18.13 +x64,64,mul,6x6,kernel,17.38 +x64,64,mul,8x8,auto,35.57 +x64,64,mul,8x8,floor,32.41 +x64,64,mul,8x8,gmp,27.08 +x64,64,mul,8x8,gmpz,28.93 +x64,64,mul,8x8,kernel,25.75 +x64,64,mul,12x12,auto,57.61 +x64,64,mul,12x12,floor,54.41 +x64,64,mul,12x12,gmp,48.56 +x64,64,mul,12x12,gmpz,51.28 +x64,64,mul,12x12,kernel,47.33 +x64,64,mul,16x16,auto,88.47 +x64,64,mul,16x16,floor,85.41 +x64,64,mul,16x16,gmp,81.97 +x64,64,mul,16x16,gmpz,84.07 +x64,64,mul,16x16,kernel,78.60 +x64,64,shl,1x13,auto,12.62 +x64,64,shl,1x13,floor,9.82 +x64,64,shl,1x13,gmp,2.08 +x64,64,shl,1x13,gmpz,3.27 +x64,64,shl,1x13,inplace,3.67 +x64,64,shl,1x13,kernel,2.08 +x64,64,shl,1x13,kernelip,1.74 +x64,64,shl,4x13,auto,13.45 +x64,64,shl,4x13,floor,11.20 +x64,64,shl,4x13,gmp,2.50 +x64,64,shl,4x13,gmpz,4.04 +x64,64,shl,4x13,inplace,4.62 +x64,64,shl,4x13,kernel,3.50 +x64,64,shl,4x13,kernelip,2.95 +x64,64,shl,16x13,auto,17.07 +x64,64,shl,16x13,floor,15.26 +x64,64,shl,16x13,gmp,4.18 +x64,64,shl,16x13,gmpz,5.82 +x64,64,shl,16x13,inplace,9.13 +x64,64,shl,16x13,kernel,7.45 +x64,64,shl,16x13,kernelip,7.12 +x64,64,shr,1x13,auto,8.19 +x64,64,shr,1x13,floor,2.27 +x64,64,shr,1x13,gmp,1.89 +x64,64,shr,1x13,gmpz,3.46 +x64,64,shr,1x13,inplace,3.29 +x64,64,shr,1x13,kernel,1.70 +x64,64,shr,4x13,auto,16.02 +x64,64,shr,4x13,floor,11.16 +x64,64,shr,4x13,gmp,2.33 +x64,64,shr,4x13,gmpz,3.98 +x64,64,shr,4x13,inplace,4.35 +x64,64,shr,4x13,kernel,3.50 +x64,64,shr,16x13,auto,19.93 +x64,64,shr,16x13,floor,15.46 +x64,64,shr,16x13,gmp,4.36 +x64,64,shr,16x13,gmpz,6.21 +x64,64,shr,16x13,inplace,8.65 +x64,64,shr,16x13,kernel,8.38 +x64,64,sqr,4x4,auto,22.06 +x64,64,sqr,4x4,floor,16.78 +x64,64,sqr,4x4,gmp,5.82 +x64,64,sqr,4x4,gmpz,8.80 +x64,64,sqr,4x4,kernel,9.88 +x64,64,sqr,16x16,auto,67.22 +x64,64,sqr,16x16,floor,63.28 +x64,64,sqr,16x16,gmp,49.83 +x64,64,sqr,16x16,gmpz,52.09 +x64,64,sqr,16x16,kernel,57.57 +x64,64,sub,1x1,auto,7.75 +x64,64,sub,1x1,floor,1.89 +x64,64,sub,1x1,gmp,2.27 +x64,64,sub,1x1,gmpz,4.34 +x64,64,sub,1x1,inplace,4.67 +x64,64,sub,1x1,kernel,1.51 +x64,64,sub,2x2,auto,15.72 +x64,64,sub,2x2,floor,9.96 +x64,64,sub,2x2,gmp,2.27 +x64,64,sub,2x2,gmpz,4.35 +x64,64,sub,2x2,inplace,5.12 +x64,64,sub,2x2,kernel,2.10 +x64,64,sub,4x4,auto,16.57 +x64,64,sub,4x4,floor,10.78 +x64,64,sub,4x4,gmp,2.50 +x64,64,sub,4x4,gmpz,4.56 +x64,64,sub,4x4,inplace,5.59 +x64,64,sub,4x4,kernel,2.84 +x64,64,sub,8x8,auto,17.94 +x64,64,sub,8x8,floor,12.36 +x64,64,sub,8x8,gmp,3.69 +x64,64,sub,8x8,gmpz,4.96 +x64,64,sub,8x8,inplace,6.94 +x64,64,sub,8x8,kernel,4.36 +x64,64,sub,16x16,auto,21.08 +x64,64,sub,16x16,floor,15.38 +x64,64,sub,16x16,gmp,5.50 +x64,64,sub,16x16,gmpz,6.91 +x64,64,sub,16x16,inplace,11.07 +x64,64,sub,16x16,kernel,8.02 +x64,64,vecsort,100000x1,auto,86.13 +x64,64,vecsort,100000x1,builtin,39.44 +x64,64,vecsort,100000x1,copy,1.13 +x64,128,add,1x1,auto,8.32 +x64,128,add,1x1,floor,2.59 +x64,128,add,1x1,gmp,2.31 +x64,128,add,1x1,gmpz,4.27 +x64,128,add,1x1,inplace,4.91 +x64,128,add,1x1,kernel,1.93 +x64,128,add,2x2,auto,19.38 +x64,128,add,2x2,floor,10.29 +x64,128,add,2x2,gmp,2.27 +x64,128,add,2x2,gmpz,4.16 +x64,128,add,2x2,inplace,6.94 +x64,128,add,2x2,kernel,2.34 +x64,128,add,4x4,auto,17.56 +x64,128,add,4x4,floor,11.34 +x64,128,add,4x4,gmp,2.49 +x64,128,add,4x4,gmpz,4.36 +x64,128,add,4x4,inplace,6.14 +x64,128,add,4x4,kernel,3.03 +x64,128,add,8x8,auto,18.79 +x64,128,add,8x8,floor,12.54 +x64,128,add,8x8,gmp,3.68 +x64,128,add,8x8,gmpz,4.67 +x64,128,add,8x8,inplace,7.36 +x64,128,add,8x8,kernel,4.61 +x64,128,add,16x16,auto,24.19 +x64,128,add,16x16,floor,15.41 +x64,128,add,16x16,gmp,5.50 +x64,128,add,16x16,gmpz,6.52 +x64,128,add,16x16,inplace,13.12 +x64,128,add,16x16,kernel,7.56 +x64,128,divrem,2x1,auto,34.45 +x64,128,divrem,2x1,floor,6.04 +x64,128,divrem,2x1,gmp,7.69 +x64,128,divrem,2x1,gmpz,12.61 +x64,128,divrem,2x1,kernel,5.72 +x64,128,divrem,8x4,auto,94.86 +x64,128,divrem,8x4,floor,65.36 +x64,128,divrem,8x4,gmp,42.01 +x64,128,divrem,8x4,gmpz,43.87 +x64,128,divrem,8x4,kernel,54.47 +x64,128,divrem,32x16,auto,344.57 +x64,128,divrem,32x16,floor,303.09 +x64,128,divrem,32x16,gmp,191.27 +x64,128,divrem,32x16,gmpz,194.19 +x64,128,divrem,32x16,kernel,289.05 +x64,128,mul,1x1,auto,10.50 +x64,128,mul,1x1,gmp,3.38 +x64,128,mul,1x1,gmpz,3.98 +x64,128,mul,2x2,auto,20.30 +x64,128,mul,2x2,floor,15.12 +x64,128,mul,2x2,gmp,3.83 +x64,128,mul,2x2,gmpz,6.17 +x64,128,mul,2x2,kernel,7.56 +x64,128,mul,3x3,auto,21.90 +x64,128,mul,3x3,floor,16.57 +x64,128,mul,3x3,gmp,7.76 +x64,128,mul,3x3,gmpz,9.97 +x64,128,mul,3x3,kernel,8.96 +x64,128,mul,4x4,auto,23.99 +x64,128,mul,4x4,floor,18.38 +x64,128,mul,4x4,gmp,9.67 +x64,128,mul,4x4,gmpz,12.16 +x64,128,mul,4x4,kernel,10.96 +x64,128,mul,6x6,auto,29.59 +x64,128,mul,6x6,floor,24.89 +x64,128,mul,6x6,gmp,15.68 +x64,128,mul,6x6,gmpz,18.02 +x64,128,mul,6x6,kernel,17.16 +x64,128,mul,8x8,auto,37.36 +x64,128,mul,8x8,floor,32.53 +x64,128,mul,8x8,gmp,27.07 +x64,128,mul,8x8,gmpz,29.04 +x64,128,mul,8x8,kernel,25.34 +x64,128,mul,12x12,auto,58.64 +x64,128,mul,12x12,floor,55.00 +x64,128,mul,12x12,gmp,49.87 +x64,128,mul,12x12,gmpz,51.30 +x64,128,mul,12x12,kernel,47.79 +x64,128,mul,16x16,auto,90.39 +x64,128,mul,16x16,floor,86.33 +x64,128,mul,16x16,gmp,81.91 +x64,128,mul,16x16,gmpz,84.06 +x64,128,mul,16x16,kernel,78.55 +x64,128,shl,1x13,auto,7.16 +x64,128,shl,1x13,floor,2.67 +x64,128,shl,1x13,gmp,1.90 +x64,128,shl,1x13,gmpz,3.27 +x64,128,shl,1x13,inplace,4.28 +x64,128,shl,1x13,kernel,2.07 +x64,128,shl,1x13,kernelip,1.77 +x64,128,shl,4x13,auto,14.89 +x64,128,shl,4x13,floor,11.43 +x64,128,shl,4x13,gmp,2.31 +x64,128,shl,4x13,gmpz,4.04 +x64,128,shl,4x13,inplace,4.75 +x64,128,shl,4x13,kernel,3.51 +x64,128,shl,4x13,kernelip,2.95 +x64,128,shl,16x13,auto,18.23 +x64,128,shl,16x13,floor,15.21 +x64,128,shl,16x13,gmp,4.12 +x64,128,shl,16x13,gmpz,5.81 +x64,128,shl,16x13,inplace,8.89 +x64,128,shl,16x13,kernel,7.45 +x64,128,shl,16x13,kernelip,7.13 +x64,128,shr,1x13,auto,8.90 +x64,128,shr,1x13,floor,2.44 +x64,128,shr,1x13,gmp,1.89 +x64,128,shr,1x13,gmpz,3.47 +x64,128,shr,1x13,inplace,3.45 +x64,128,shr,1x13,kernel,1.70 +x64,128,shr,4x13,auto,16.83 +x64,128,shr,4x13,floor,11.19 +x64,128,shr,4x13,gmp,2.34 +x64,128,shr,4x13,gmpz,3.98 +x64,128,shr,4x13,inplace,4.45 +x64,128,shr,4x13,kernel,3.47 +x64,128,shr,16x13,auto,20.91 +x64,128,shr,16x13,floor,15.50 +x64,128,shr,16x13,gmp,4.30 +x64,128,shr,16x13,gmpz,6.33 +x64,128,shr,16x13,inplace,8.48 +x64,128,shr,16x13,kernel,8.39 +x64,128,sqr,4x4,auto,23.82 +x64,128,sqr,4x4,floor,17.23 +x64,128,sqr,4x4,gmp,5.81 +x64,128,sqr,4x4,gmpz,8.79 +x64,128,sqr,4x4,kernel,10.37 +x64,128,sqr,16x16,auto,67.98 +x64,128,sqr,16x16,floor,62.87 +x64,128,sqr,16x16,gmp,49.83 +x64,128,sqr,16x16,gmpz,52.09 +x64,128,sqr,16x16,kernel,55.88 +x64,128,sub,1x1,auto,9.64 +x64,128,sub,1x1,floor,1.89 +x64,128,sub,1x1,gmp,2.27 +x64,128,sub,1x1,gmpz,4.34 +x64,128,sub,1x1,inplace,4.65 +x64,128,sub,1x1,kernel,1.51 +x64,128,sub,2x2,auto,10.82 +x64,128,sub,2x2,floor,2.48 +x64,128,sub,2x2,gmp,2.27 +x64,128,sub,2x2,gmpz,4.34 +x64,128,sub,2x2,inplace,5.45 +x64,128,sub,2x2,kernel,2.12 +x64,128,sub,4x4,auto,19.50 +x64,128,sub,4x4,floor,10.71 +x64,128,sub,4x4,gmp,2.50 +x64,128,sub,4x4,gmpz,4.56 +x64,128,sub,4x4,inplace,5.96 +x64,128,sub,4x4,kernel,2.84 +x64,128,sub,8x8,auto,21.02 +x64,128,sub,8x8,floor,12.29 +x64,128,sub,8x8,gmp,3.80 +x64,128,sub,8x8,gmpz,4.97 +x64,128,sub,8x8,inplace,6.98 +x64,128,sub,8x8,kernel,4.39 +x64,128,sub,16x16,auto,24.45 +x64,128,sub,16x16,floor,15.26 +x64,128,sub,16x16,gmp,5.50 +x64,128,sub,16x16,gmpz,6.85 +x64,128,sub,16x16,inplace,11.10 +x64,128,sub,16x16,kernel,7.74 +x64,128,vecsort,100000x1,auto,92.31 +x64,128,vecsort,100000x1,builtin,39.39 +x64,128,vecsort,100000x1,copy,2.50 +x64,256,add,1x1,auto,8.62 +x64,256,add,1x1,floor,2.57 +x64,256,add,1x1,gmp,2.27 +x64,256,add,1x1,gmpz,4.16 +x64,256,add,1x1,inplace,4.91 +x64,256,add,1x1,kernel,1.92 +x64,256,add,2x2,auto,10.87 +x64,256,add,2x2,floor,2.89 +x64,256,add,2x2,gmp,2.27 +x64,256,add,2x2,gmpz,4.16 +x64,256,add,2x2,inplace,6.38 +x64,256,add,2x2,kernel,2.33 +x64,256,add,4x4,auto,10.61 +x64,256,add,4x4,floor,11.16 +x64,256,add,4x4,gmp,2.55 +x64,256,add,4x4,gmpz,4.38 +x64,256,add,4x4,inplace,5.72 +x64,256,add,4x4,kernel,3.04 +x64,256,add,8x8,auto,18.89 +x64,256,add,8x8,floor,12.50 +x64,256,add,8x8,gmp,3.68 +x64,256,add,8x8,gmpz,5.19 +x64,256,add,8x8,inplace,7.19 +x64,256,add,8x8,kernel,4.53 +x64,256,add,16x16,auto,23.51 +x64,256,add,16x16,floor,15.55 +x64,256,add,16x16,gmp,6.03 +x64,256,add,16x16,gmpz,6.51 +x64,256,add,16x16,inplace,12.77 +x64,256,add,16x16,kernel,7.70 +x64,256,divrem,2x1,auto,36.72 +x64,256,divrem,2x1,floor,6.00 +x64,256,divrem,2x1,gmp,8.13 +x64,256,divrem,2x1,gmpz,12.45 +x64,256,divrem,2x1,kernel,5.90 +x64,256,divrem,8x4,auto,89.11 +x64,256,divrem,8x4,floor,55.64 +x64,256,divrem,8x4,gmp,42.63 +x64,256,divrem,8x4,gmpz,43.81 +x64,256,divrem,8x4,kernel,49.22 +x64,256,divrem,32x16,auto,343.75 +x64,256,divrem,32x16,floor,303.77 +x64,256,divrem,32x16,gmp,191.40 +x64,256,divrem,32x16,gmpz,193.96 +x64,256,divrem,32x16,kernel,288.84 +x64,256,mul,1x1,auto,10.76 +x64,256,mul,1x1,gmp,3.37 +x64,256,mul,1x1,gmpz,3.99 +x64,256,mul,2x2,auto,14.25 +x64,256,mul,2x2,floor,7.80 +x64,256,mul,2x2,gmp,3.82 +x64,256,mul,2x2,gmpz,6.13 +x64,256,mul,2x2,kernel,7.55 +x64,256,mul,3x3,auto,23.45 +x64,256,mul,3x3,floor,16.38 +x64,256,mul,3x3,gmp,7.77 +x64,256,mul,3x3,gmpz,9.97 +x64,256,mul,3x3,kernel,8.93 +x64,256,mul,4x4,auto,24.34 +x64,256,mul,4x4,floor,18.15 +x64,256,mul,4x4,gmp,9.68 +x64,256,mul,4x4,gmpz,12.16 +x64,256,mul,4x4,kernel,10.91 +x64,256,mul,6x6,auto,30.24 +x64,256,mul,6x6,floor,24.38 +x64,256,mul,6x6,gmp,15.71 +x64,256,mul,6x6,gmpz,18.02 +x64,256,mul,6x6,kernel,17.20 +x64,256,mul,8x8,auto,37.98 +x64,256,mul,8x8,floor,32.35 +x64,256,mul,8x8,gmp,27.04 +x64,256,mul,8x8,gmpz,28.98 +x64,256,mul,8x8,kernel,25.43 +x64,256,mul,12x12,auto,59.90 +x64,256,mul,12x12,floor,55.01 +x64,256,mul,12x12,gmp,49.14 +x64,256,mul,12x12,gmpz,51.80 +x64,256,mul,12x12,kernel,47.98 +x64,256,mul,16x16,auto,91.90 +x64,256,mul,16x16,floor,85.73 +x64,256,mul,16x16,gmp,82.15 +x64,256,mul,16x16,gmpz,84.08 +x64,256,mul,16x16,kernel,78.56 +x64,256,shl,1x13,auto,8.19 +x64,256,shl,1x13,floor,2.65 +x64,256,shl,1x13,gmp,1.89 +x64,256,shl,1x13,gmpz,3.30 +x64,256,shl,1x13,inplace,4.28 +x64,256,shl,1x13,kernel,2.07 +x64,256,shl,1x13,kernelip,1.75 +x64,256,shl,4x13,auto,16.33 +x64,256,shl,4x13,floor,11.27 +x64,256,shl,4x13,gmp,2.32 +x64,256,shl,4x13,gmpz,4.04 +x64,256,shl,4x13,inplace,4.75 +x64,256,shl,4x13,kernel,3.54 +x64,256,shl,4x13,kernelip,2.95 +x64,256,shl,16x13,auto,20.49 +x64,256,shl,16x13,floor,15.23 +x64,256,shl,16x13,gmp,4.04 +x64,256,shl,16x13,gmpz,6.40 +x64,256,shl,16x13,inplace,9.58 +x64,256,shl,16x13,kernel,7.45 +x64,256,shl,16x13,kernelip,7.11 +x64,256,shr,1x13,auto,9.11 +x64,256,shr,1x13,floor,2.37 +x64,256,shr,1x13,gmp,1.89 +x64,256,shr,1x13,gmpz,3.47 +x64,256,shr,1x13,inplace,3.46 +x64,256,shr,1x13,kernel,1.92 +x64,256,shr,4x13,auto,11.93 +x64,256,shr,4x13,floor,4.04 +x64,256,shr,4x13,gmp,2.34 +x64,256,shr,4x13,gmpz,3.96 +x64,256,shr,4x13,inplace,4.46 +x64,256,shr,4x13,kernel,3.47 +x64,256,shr,16x13,auto,21.66 +x64,256,shr,16x13,floor,15.45 +x64,256,shr,16x13,gmp,4.35 +x64,256,shr,16x13,gmpz,6.30 +x64,256,shr,16x13,inplace,8.48 +x64,256,shr,16x13,kernel,8.39 +x64,256,sqr,4x4,auto,24.43 +x64,256,sqr,4x4,floor,16.78 +x64,256,sqr,4x4,gmp,6.06 +x64,256,sqr,4x4,gmpz,8.80 +x64,256,sqr,4x4,kernel,9.81 +x64,256,sqr,16x16,auto,69.28 +x64,256,sqr,16x16,floor,63.09 +x64,256,sqr,16x16,gmp,49.84 +x64,256,sqr,16x16,gmpz,52.09 +x64,256,sqr,16x16,kernel,59.19 +x64,256,sub,1x1,auto,10.04 +x64,256,sub,1x1,floor,1.89 +x64,256,sub,1x1,gmp,2.27 +x64,256,sub,1x1,gmpz,4.35 +x64,256,sub,1x1,inplace,4.63 +x64,256,sub,1x1,kernel,1.67 +x64,256,sub,2x2,auto,11.30 +x64,256,sub,2x2,floor,2.48 +x64,256,sub,2x2,gmp,2.27 +x64,256,sub,2x2,gmpz,4.34 +x64,256,sub,2x2,inplace,5.33 +x64,256,sub,2x2,kernel,2.12 +x64,256,sub,4x4,auto,12.10 +x64,256,sub,4x4,floor,3.21 +x64,256,sub,4x4,gmp,2.50 +x64,256,sub,4x4,gmpz,4.58 +x64,256,sub,4x4,inplace,5.60 +x64,256,sub,4x4,kernel,2.84 +x64,256,sub,8x8,auto,20.26 +x64,256,sub,8x8,floor,12.26 +x64,256,sub,8x8,gmp,3.69 +x64,256,sub,8x8,gmpz,4.96 +x64,256,sub,8x8,inplace,7.02 +x64,256,sub,8x8,kernel,4.36 +x64,256,sub,16x16,auto,23.45 +x64,256,sub,16x16,floor,15.29 +x64,256,sub,16x16,gmp,5.54 +x64,256,sub,16x16,gmpz,6.80 +x64,256,sub,16x16,inplace,11.18 +x64,256,sub,16x16,kernel,7.74 +x64,256,vecsort,100000x1,auto,98.59 +x64,256,vecsort,100000x1,builtin,39.41 +x64,256,vecsort,100000x1,copy,2.62 +x64,512,add,1x1,auto,8.66 +x64,512,add,1x1,floor,2.57 +x64,512,add,1x1,gmp,2.27 +x64,512,add,1x1,gmpz,4.16 +x64,512,add,1x1,inplace,4.91 +x64,512,add,1x1,kernel,1.92 +x64,512,add,2x2,auto,11.00 +x64,512,add,2x2,floor,2.88 +x64,512,add,2x2,gmp,2.27 +x64,512,add,2x2,gmpz,4.16 +x64,512,add,2x2,inplace,6.20 +x64,512,add,2x2,kernel,2.35 +x64,512,add,4x4,auto,10.64 +x64,512,add,4x4,floor,3.64 +x64,512,add,4x4,gmp,2.63 +x64,512,add,4x4,gmpz,4.44 +x64,512,add,4x4,inplace,5.71 +x64,512,add,4x4,kernel,3.04 +x64,512,add,8x8,auto,11.99 +x64,512,add,8x8,floor,12.55 +x64,512,add,8x8,gmp,3.68 +x64,512,add,8x8,gmpz,4.73 +x64,512,add,8x8,inplace,7.04 +x64,512,add,8x8,kernel,4.53 +x64,512,add,16x16,auto,23.50 +x64,512,add,16x16,floor,15.41 +x64,512,add,16x16,gmp,5.50 +x64,512,add,16x16,gmpz,6.54 +x64,512,add,16x16,inplace,11.92 +x64,512,add,16x16,kernel,7.56 +x64,512,divrem,2x1,auto,36.64 +x64,512,divrem,2x1,floor,6.15 +x64,512,divrem,2x1,gmp,7.70 +x64,512,divrem,2x1,gmpz,12.38 +x64,512,divrem,2x1,kernel,5.72 +x64,512,divrem,8x4,auto,80.16 +x64,512,divrem,8x4,floor,45.38 +x64,512,divrem,8x4,gmp,41.91 +x64,512,divrem,8x4,gmpz,43.89 +x64,512,divrem,8x4,kernel,44.95 +x64,512,divrem,32x16,auto,342.32 +x64,512,divrem,32x16,floor,304.12 +x64,512,divrem,32x16,gmp,191.14 +x64,512,divrem,32x16,gmpz,195.05 +x64,512,divrem,32x16,kernel,294.90 +x64,512,mul,1x1,auto,10.80 +x64,512,mul,1x1,gmp,3.45 +x64,512,mul,1x1,gmpz,3.99 +x64,512,mul,2x2,auto,14.27 +x64,512,mul,2x2,floor,7.95 +x64,512,mul,2x2,gmp,3.95 +x64,512,mul,2x2,gmpz,6.13 +x64,512,mul,2x2,kernel,7.66 +x64,512,mul,3x3,auto,15.63 +x64,512,mul,3x3,floor,9.23 +x64,512,mul,3x3,gmp,7.81 +x64,512,mul,3x3,gmpz,9.95 +x64,512,mul,3x3,kernel,9.04 +x64,512,mul,4x4,auto,17.77 +x64,512,mul,4x4,floor,11.57 +x64,512,mul,4x4,gmp,9.80 +x64,512,mul,4x4,gmpz,12.18 +x64,512,mul,4x4,kernel,11.15 +x64,512,mul,6x6,auto,30.14 +x64,512,mul,6x6,floor,25.26 +x64,512,mul,6x6,gmp,16.54 +x64,512,mul,6x6,gmpz,18.03 +x64,512,mul,6x6,kernel,17.30 +x64,512,mul,8x8,auto,37.86 +x64,512,mul,8x8,floor,32.79 +x64,512,mul,8x8,gmp,27.05 +x64,512,mul,8x8,gmpz,28.68 +x64,512,mul,8x8,kernel,25.56 +x64,512,mul,12x12,auto,59.89 +x64,512,mul,12x12,floor,54.98 +x64,512,mul,12x12,gmp,48.56 +x64,512,mul,12x12,gmpz,51.34 +x64,512,mul,12x12,kernel,47.55 +x64,512,mul,16x16,auto,91.28 +x64,512,mul,16x16,floor,85.89 +x64,512,mul,16x16,gmp,82.19 +x64,512,mul,16x16,gmpz,84.09 +x64,512,mul,16x16,kernel,78.66 +x64,512,shl,1x13,auto,8.27 +x64,512,shl,1x13,floor,2.66 +x64,512,shl,1x13,gmp,2.08 +x64,512,shl,1x13,gmpz,3.33 +x64,512,shl,1x13,inplace,4.26 +x64,512,shl,1x13,kernel,2.07 +x64,512,shl,1x13,kernelip,1.75 +x64,512,shl,4x13,auto,9.14 +x64,512,shl,4x13,floor,4.14 +x64,512,shl,4x13,gmp,2.52 +x64,512,shl,4x13,gmpz,4.04 +x64,512,shl,4x13,inplace,5.18 +x64,512,shl,4x13,kernel,3.50 +x64,512,shl,4x13,kernelip,2.96 +x64,512,shl,16x13,auto,19.46 +x64,512,shl,16x13,floor,15.21 +x64,512,shl,16x13,gmp,4.17 +x64,512,shl,16x13,gmpz,5.83 +x64,512,shl,16x13,inplace,8.87 +x64,512,shl,16x13,kernel,7.45 +x64,512,shl,16x13,kernelip,7.19 +x64,512,shr,1x13,auto,9.30 +x64,512,shr,1x13,floor,2.27 +x64,512,shr,1x13,gmp,1.89 +x64,512,shr,1x13,gmpz,3.46 +x64,512,shr,1x13,inplace,3.42 +x64,512,shr,1x13,kernel,1.70 +x64,512,shr,4x13,auto,11.89 +x64,512,shr,4x13,floor,4.02 +x64,512,shr,4x13,gmp,2.34 +x64,512,shr,4x13,gmpz,3.96 +x64,512,shr,4x13,inplace,4.49 +x64,512,shr,4x13,kernel,3.95 +x64,512,shr,16x13,auto,22.01 +x64,512,shr,16x13,floor,15.64 +x64,512,shr,16x13,gmp,4.35 +x64,512,shr,16x13,gmpz,6.26 +x64,512,shr,16x13,inplace,8.97 +x64,512,shr,16x13,kernel,8.50 +x64,512,sqr,4x4,auto,17.45 +x64,512,sqr,4x4,floor,9.61 +x64,512,sqr,4x4,gmp,5.81 +x64,512,sqr,4x4,gmpz,8.80 +x64,512,sqr,4x4,kernel,9.87 +x64,512,sqr,16x16,auto,70.15 +x64,512,sqr,16x16,floor,64.25 +x64,512,sqr,16x16,gmp,49.83 +x64,512,sqr,16x16,gmpz,52.10 +x64,512,sqr,16x16,kernel,57.06 +x64,512,sub,1x1,auto,10.21 +x64,512,sub,1x1,floor,1.89 +x64,512,sub,1x1,gmp,2.27 +x64,512,sub,1x1,gmpz,4.34 +x64,512,sub,1x1,inplace,4.63 +x64,512,sub,1x1,kernel,1.51 +x64,512,sub,2x2,auto,11.32 +x64,512,sub,2x2,floor,2.47 +x64,512,sub,2x2,gmp,2.27 +x64,512,sub,2x2,gmpz,4.34 +x64,512,sub,2x2,inplace,5.38 +x64,512,sub,2x2,kernel,2.11 +x64,512,sub,4x4,auto,12.11 +x64,512,sub,4x4,floor,3.23 +x64,512,sub,4x4,gmp,2.50 +x64,512,sub,4x4,gmpz,4.58 +x64,512,sub,4x4,inplace,5.59 +x64,512,sub,4x4,kernel,2.93 +x64,512,sub,8x8,auto,13.48 +x64,512,sub,8x8,floor,4.79 +x64,512,sub,8x8,gmp,3.78 +x64,512,sub,8x8,gmpz,4.97 +x64,512,sub,8x8,inplace,6.95 +x64,512,sub,8x8,kernel,4.39 +x64,512,sub,16x16,auto,25.84 +x64,512,sub,16x16,floor,15.34 +x64,512,sub,16x16,gmp,5.85 +x64,512,sub,16x16,gmpz,6.80 +x64,512,sub,16x16,inplace,11.04 +x64,512,sub,16x16,kernel,7.46 +x64,512,vecsort,100000x1,auto,103.44 +x64,512,vecsort,100000x1,builtin,39.33 +x64,512,vecsort,100000x1,copy,3.14 +m4,64,add,1x1,auto,7.50 +m4,64,add,1x1,floor,9.45 +m4,64,add,1x1,gmp,1.49 +m4,64,add,1x1,gmpz,2.98 +m4,64,add,1x1,inplace,3.52 +m4,64,add,1x1,kernel,1.14 +m4,64,add,2x2,auto,15.68 +m4,64,add,2x2,floor,9.75 +m4,64,add,2x2,gmp,1.98 +m4,64,add,2x2,gmpz,3.24 +m4,64,add,2x2,inplace,3.93 +m4,64,add,2x2,kernel,1.40 +m4,64,add,4x4,auto,14.14 +m4,64,add,4x4,floor,10.43 +m4,64,add,4x4,gmp,1.74 +m4,64,add,4x4,gmpz,5.99 +m4,64,add,4x4,inplace,3.55 +m4,64,add,4x4,kernel,1.92 +m4,64,add,8x8,auto,15.29 +m4,64,add,8x8,floor,12.04 +m4,64,add,8x8,gmp,2.25 +m4,64,add,8x8,gmpz,4.17 +m4,64,add,8x8,inplace,3.87 +m4,64,add,8x8,kernel,3.49 +m4,64,add,16x16,auto,16.99 +m4,64,add,16x16,floor,13.89 +m4,64,add,16x16,gmp,3.23 +m4,64,add,16x16,gmpz,4.49 +m4,64,add,16x16,inplace,5.58 +m4,64,add,16x16,kernel,7.57 +m4,64,divrem,2x1,auto,24.63 +m4,64,divrem,2x1,floor,13.62 +m4,64,divrem,2x1,gmp,5.04 +m4,64,divrem,2x1,gmpz,8.71 +m4,64,divrem,2x1,kernel,6.02 +m4,64,divrem,8x4,auto,74.98 +m4,64,divrem,8x4,floor,60.59 +m4,64,divrem,8x4,gmp,40.84 +m4,64,divrem,8x4,gmpz,43.63 +m4,64,divrem,8x4,kernel,46.08 +m4,64,divrem,32x16,auto,429.70 +m4,64,divrem,32x16,floor,414.18 +m4,64,divrem,32x16,gmp,207.56 +m4,64,divrem,32x16,gmpz,209.34 +m4,64,divrem,32x16,kernel,393.05 +m4,64,mul,1x1,auto,14.50 +m4,64,mul,1x1,gmp,3.72 +m4,64,mul,1x1,gmpz,2.74 +m4,64,mul,2x2,auto,18.58 +m4,64,mul,2x2,floor,14.55 +m4,64,mul,2x2,gmp,4.48 +m4,64,mul,2x2,gmpz,6.54 +m4,64,mul,2x2,kernel,6.28 +m4,64,mul,3x3,auto,19.34 +m4,64,mul,3x3,floor,17.12 +m4,64,mul,3x3,gmp,6.11 +m4,64,mul,3x3,gmpz,8.51 +m4,64,mul,3x3,kernel,8.09 +m4,64,mul,4x4,auto,21.60 +m4,64,mul,4x4,floor,17.05 +m4,64,mul,4x4,gmp,9.32 +m4,64,mul,4x4,gmpz,11.30 +m4,64,mul,4x4,kernel,11.14 +m4,64,mul,6x6,auto,33.52 +m4,64,mul,6x6,floor,24.21 +m4,64,mul,6x6,gmp,14.51 +m4,64,mul,6x6,gmpz,18.41 +m4,64,mul,6x6,kernel,19.34 +m4,64,mul,8x8,auto,35.30 +m4,64,mul,8x8,floor,30.38 +m4,64,mul,8x8,gmp,22.36 +m4,64,mul,8x8,gmpz,25.79 +m4,64,mul,8x8,kernel,22.92 +m4,64,mul,12x12,auto,53.86 +m4,64,mul,12x12,floor,49.28 +m4,64,mul,12x12,gmp,77.97 +m4,64,mul,12x12,gmpz,46.70 +m4,64,mul,12x12,kernel,41.20 +m4,64,mul,16x16,auto,82.81 +m4,64,mul,16x16,floor,80.12 +m4,64,mul,16x16,gmp,76.95 +m4,64,mul,16x16,gmpz,78.12 +m4,64,mul,16x16,kernel,69.86 +m4,64,shl,1x13,auto,11.91 +m4,64,shl,1x13,floor,10.80 +m4,64,shl,1x13,gmp,1.24 +m4,64,shl,1x13,gmpz,2.00 +m4,64,shl,1x13,inplace,2.49 +m4,64,shl,1x13,kernel,2.24 +m4,64,shl,1x13,kernelip,1.46 +m4,64,shl,4x13,auto,12.69 +m4,64,shl,4x13,floor,13.03 +m4,64,shl,4x13,gmp,1.99 +m4,64,shl,4x13,gmpz,2.74 +m4,64,shl,4x13,inplace,3.83 +m4,64,shl,4x13,kernel,3.73 +m4,64,shl,4x13,kernelip,3.37 +m4,64,shl,16x13,auto,15.47 +m4,64,shl,16x13,floor,12.25 +m4,64,shl,16x13,gmp,3.50 +m4,64,shl,16x13,gmpz,5.22 +m4,64,shl,16x13,inplace,5.30 +m4,64,shl,16x13,kernel,4.10 +m4,64,shl,16x13,kernelip,6.81 +m4,64,shr,1x13,auto,6.96 +m4,64,shr,1x13,floor,2.49 +m4,64,shr,1x13,gmp,0.99 +m4,64,shr,1x13,gmpz,1.99 +m4,64,shr,1x13,inplace,3.30 +m4,64,shr,1x13,kernel,2.24 +m4,64,shr,4x13,auto,13.89 +m4,64,shr,4x13,floor,13.00 +m4,64,shr,4x13,gmp,1.74 +m4,64,shr,4x13,gmpz,2.74 +m4,64,shr,4x13,inplace,3.62 +m4,64,shr,4x13,kernel,3.75 +m4,64,shr,16x13,auto,18.17 +m4,64,shr,16x13,floor,13.32 +m4,64,shr,16x13,gmp,3.23 +m4,64,shr,16x13,gmpz,4.23 +m4,64,shr,16x13,inplace,5.23 +m4,64,shr,16x13,kernel,4.19 +m4,64,sqr,4x4,auto,22.78 +m4,64,sqr,4x4,floor,18.20 +m4,64,sqr,4x4,gmp,6.95 +m4,64,sqr,4x4,gmpz,9.60 +m4,64,sqr,4x4,kernel,11.24 +m4,64,sqr,16x16,auto,75.62 +m4,64,sqr,16x16,floor,72.28 +m4,64,sqr,16x16,gmp,63.25 +m4,64,sqr,16x16,gmpz,66.22 +m4,64,sqr,16x16,kernel,63.74 +m4,64,sub,1x1,auto,7.95 +m4,64,sub,1x1,floor,1.74 +m4,64,sub,1x1,gmp,1.49 +m4,64,sub,1x1,gmpz,2.49 +m4,64,sub,1x1,inplace,3.50 +m4,64,sub,1x1,kernel,1.05 +m4,64,sub,2x2,auto,14.64 +m4,64,sub,2x2,floor,9.20 +m4,64,sub,2x2,gmp,1.78 +m4,64,sub,2x2,gmpz,2.85 +m4,64,sub,2x2,inplace,3.49 +m4,64,sub,2x2,kernel,1.42 +m4,64,sub,4x4,auto,14.38 +m4,64,sub,4x4,floor,10.07 +m4,64,sub,4x4,gmp,1.74 +m4,64,sub,4x4,gmpz,2.73 +m4,64,sub,4x4,inplace,3.42 +m4,64,sub,4x4,kernel,1.94 +m4,64,sub,8x8,auto,15.61 +m4,64,sub,8x8,floor,11.72 +m4,64,sub,8x8,gmp,2.24 +m4,64,sub,8x8,gmpz,3.23 +m4,64,sub,8x8,inplace,4.09 +m4,64,sub,8x8,kernel,3.50 +m4,64,sub,16x16,auto,18.28 +m4,64,sub,16x16,floor,15.30 +m4,64,sub,16x16,gmp,3.10 +m4,64,sub,16x16,gmpz,4.39 +m4,64,sub,16x16,inplace,5.18 +m4,64,sub,16x16,kernel,7.71 +m4,64,vecsort,100000x1,auto,87.87 +m4,64,vecsort,100000x1,builtin,13.21 +m4,64,vecsort,100000x1,copy,2.70 +m4,128,add,1x1,auto,8.26 +m4,128,add,1x1,floor,1.83 +m4,128,add,1x1,gmp,1.49 +m4,128,add,1x1,gmpz,2.99 +m4,128,add,1x1,inplace,3.48 +m4,128,add,1x1,kernel,1.14 +m4,128,add,2x2,auto,17.54 +m4,128,add,2x2,floor,9.55 +m4,128,add,2x2,gmp,1.98 +m4,128,add,2x2,gmpz,4.64 +m4,128,add,2x2,inplace,3.87 +m4,128,add,2x2,kernel,1.39 +m4,128,add,4x4,auto,14.19 +m4,128,add,4x4,floor,10.45 +m4,128,add,4x4,gmp,1.75 +m4,128,add,4x4,gmpz,4.21 +m4,128,add,4x4,inplace,3.54 +m4,128,add,4x4,kernel,1.92 +m4,128,add,8x8,auto,15.21 +m4,128,add,8x8,floor,12.03 +m4,128,add,8x8,gmp,2.24 +m4,128,add,8x8,gmpz,4.23 +m4,128,add,8x8,inplace,4.03 +m4,128,add,8x8,kernel,3.54 +m4,128,add,16x16,auto,16.31 +m4,128,add,16x16,floor,13.77 +m4,128,add,16x16,gmp,3.17 +m4,128,add,16x16,gmpz,4.49 +m4,128,add,16x16,inplace,5.59 +m4,128,add,16x16,kernel,7.54 +m4,128,divrem,2x1,auto,23.18 +m4,128,divrem,2x1,floor,6.87 +m4,128,divrem,2x1,gmp,5.05 +m4,128,divrem,2x1,gmpz,8.68 +m4,128,divrem,2x1,kernel,5.95 +m4,128,divrem,8x4,auto,75.19 +m4,128,divrem,8x4,floor,60.67 +m4,128,divrem,8x4,gmp,40.67 +m4,128,divrem,8x4,gmpz,43.69 +m4,128,divrem,8x4,kernel,46.68 +m4,128,divrem,32x16,auto,430.01 +m4,128,divrem,32x16,floor,417.65 +m4,128,divrem,32x16,gmp,206.94 +m4,128,divrem,32x16,gmpz,208.92 +m4,128,divrem,32x16,kernel,393.73 +m4,128,mul,1x1,auto,9.08 +m4,128,mul,1x1,gmp,3.75 +m4,128,mul,1x1,gmpz,2.74 +m4,128,mul,2x2,auto,18.55 +m4,128,mul,2x2,floor,14.98 +m4,128,mul,2x2,gmp,4.37 +m4,128,mul,2x2,gmpz,6.75 +m4,128,mul,2x2,kernel,6.36 +m4,128,mul,3x3,auto,20.03 +m4,128,mul,3x3,floor,16.88 +m4,128,mul,3x3,gmp,6.22 +m4,128,mul,3x3,gmpz,10.04 +m4,128,mul,3x3,kernel,8.19 +m4,128,mul,4x4,auto,21.55 +m4,128,mul,4x4,floor,16.91 +m4,128,mul,4x4,gmp,9.15 +m4,128,mul,4x4,gmpz,11.06 +m4,128,mul,4x4,kernel,11.22 +m4,128,mul,6x6,auto,27.07 +m4,128,mul,6x6,floor,24.43 +m4,128,mul,6x6,gmp,14.52 +m4,128,mul,6x6,gmpz,17.53 +m4,128,mul,6x6,kernel,19.00 +m4,128,mul,8x8,auto,35.05 +m4,128,mul,8x8,floor,30.59 +m4,128,mul,8x8,gmp,24.11 +m4,128,mul,8x8,gmpz,25.62 +m4,128,mul,8x8,kernel,23.02 +m4,128,mul,12x12,auto,53.88 +m4,128,mul,12x12,floor,49.39 +m4,128,mul,12x12,gmp,44.51 +m4,128,mul,12x12,gmpz,46.99 +m4,128,mul,12x12,kernel,40.98 +m4,128,mul,16x16,auto,82.81 +m4,128,mul,16x16,floor,79.69 +m4,128,mul,16x16,gmp,76.68 +m4,128,mul,16x16,gmpz,78.49 +m4,128,mul,16x16,kernel,70.13 +m4,128,shl,1x13,auto,6.61 +m4,128,shl,1x13,floor,5.14 +m4,128,shl,1x13,gmp,1.24 +m4,128,shl,1x13,gmpz,2.01 +m4,128,shl,1x13,inplace,3.47 +m4,128,shl,1x13,kernel,2.24 +m4,128,shl,1x13,kernelip,1.78 +m4,128,shl,4x13,auto,12.75 +m4,128,shl,4x13,floor,12.57 +m4,128,shl,4x13,gmp,1.98 +m4,128,shl,4x13,gmpz,2.73 +m4,128,shl,4x13,inplace,3.84 +m4,128,shl,4x13,kernel,4.09 +m4,128,shl,4x13,kernelip,3.40 +m4,128,shl,16x13,auto,15.52 +m4,128,shl,16x13,floor,12.15 +m4,128,shl,16x13,gmp,3.49 +m4,128,shl,16x13,gmpz,5.22 +m4,128,shl,16x13,inplace,5.36 +m4,128,shl,16x13,kernel,4.08 +m4,128,shl,16x13,kernelip,6.79 +m4,128,shr,1x13,auto,8.94 +m4,128,shr,1x13,floor,2.50 +m4,128,shr,1x13,gmp,0.99 +m4,128,shr,1x13,gmpz,1.99 +m4,128,shr,1x13,inplace,3.41 +m4,128,shr,1x13,kernel,2.50 +m4,128,shr,4x13,auto,14.00 +m4,128,shr,4x13,floor,12.16 +m4,128,shr,4x13,gmp,1.74 +m4,128,shr,4x13,gmpz,2.73 +m4,128,shr,4x13,inplace,6.88 +m4,128,shr,4x13,kernel,4.47 +m4,128,shr,16x13,auto,17.75 +m4,128,shr,16x13,floor,12.97 +m4,128,shr,16x13,gmp,3.25 +m4,128,shr,16x13,gmpz,4.23 +m4,128,shr,16x13,inplace,5.25 +m4,128,shr,16x13,kernel,4.21 +m4,128,sqr,4x4,auto,23.20 +m4,128,sqr,4x4,floor,16.27 +m4,128,sqr,4x4,gmp,6.96 +m4,128,sqr,4x4,gmpz,9.69 +m4,128,sqr,4x4,kernel,11.00 +m4,128,sqr,16x16,auto,74.21 +m4,128,sqr,16x16,floor,70.51 +m4,128,sqr,16x16,gmp,62.84 +m4,128,sqr,16x16,gmpz,66.25 +m4,128,sqr,16x16,kernel,61.61 +m4,128,sub,1x1,auto,8.36 +m4,128,sub,1x1,floor,1.73 +m4,128,sub,1x1,gmp,1.49 +m4,128,sub,1x1,gmpz,2.49 +m4,128,sub,1x1,inplace,3.61 +m4,128,sub,1x1,kernel,1.03 +m4,128,sub,2x2,auto,8.73 +m4,128,sub,2x2,floor,1.98 +m4,128,sub,2x2,gmp,1.98 +m4,128,sub,2x2,gmpz,2.87 +m4,128,sub,2x2,inplace,3.84 +m4,128,sub,2x2,kernel,1.41 +m4,128,sub,4x4,auto,14.02 +m4,128,sub,4x4,floor,10.47 +m4,128,sub,4x4,gmp,1.74 +m4,128,sub,4x4,gmpz,2.74 +m4,128,sub,4x4,inplace,3.67 +m4,128,sub,4x4,kernel,1.94 +m4,128,sub,8x8,auto,15.53 +m4,128,sub,8x8,floor,11.92 +m4,128,sub,8x8,gmp,2.24 +m4,128,sub,8x8,gmpz,3.23 +m4,128,sub,8x8,inplace,4.32 +m4,128,sub,8x8,kernel,3.50 +m4,128,sub,16x16,auto,17.67 +m4,128,sub,16x16,floor,15.38 +m4,128,sub,16x16,gmp,3.15 +m4,128,sub,16x16,gmpz,4.24 +m4,128,sub,16x16,inplace,5.34 +m4,128,sub,16x16,kernel,7.72 +m4,128,vecsort,100000x1,auto,94.78 +m4,128,vecsort,100000x1,builtin,13.38 +m4,128,vecsort,100000x1,copy,3.90 +m4,256,add,1x1,auto,9.18 +m4,256,add,1x1,floor,1.82 +m4,256,add,1x1,gmp,1.49 +m4,256,add,1x1,gmpz,2.99 +m4,256,add,1x1,inplace,3.35 +m4,256,add,1x1,kernel,1.14 +m4,256,add,2x2,auto,10.64 +m4,256,add,2x2,floor,2.07 +m4,256,add,2x2,gmp,1.98 +m4,256,add,2x2,gmpz,4.02 +m4,256,add,2x2,inplace,4.22 +m4,256,add,2x2,kernel,1.39 +m4,256,add,4x4,auto,9.30 +m4,256,add,4x4,floor,10.21 +m4,256,add,4x4,gmp,1.74 +m4,256,add,4x4,gmpz,4.10 +m4,256,add,4x4,inplace,3.77 +m4,256,add,4x4,kernel,1.98 +m4,256,add,8x8,auto,15.16 +m4,256,add,8x8,floor,12.11 +m4,256,add,8x8,gmp,2.24 +m4,256,add,8x8,gmpz,3.83 +m4,256,add,8x8,inplace,4.23 +m4,256,add,8x8,kernel,3.65 +m4,256,add,16x16,auto,16.59 +m4,256,add,16x16,floor,13.84 +m4,256,add,16x16,gmp,3.23 +m4,256,add,16x16,gmpz,4.37 +m4,256,add,16x16,inplace,5.71 +m4,256,add,16x16,kernel,7.67 +m4,256,divrem,2x1,auto,23.56 +m4,256,divrem,2x1,floor,6.95 +m4,256,divrem,2x1,gmp,5.01 +m4,256,divrem,2x1,gmpz,8.68 +m4,256,divrem,2x1,kernel,5.97 +m4,256,divrem,8x4,auto,74.69 +m4,256,divrem,8x4,floor,54.52 +m4,256,divrem,8x4,gmp,40.87 +m4,256,divrem,8x4,gmpz,44.01 +m4,256,divrem,8x4,kernel,46.39 +m4,256,divrem,32x16,auto,431.31 +m4,256,divrem,32x16,floor,410.74 +m4,256,divrem,32x16,gmp,206.19 +m4,256,divrem,32x16,gmpz,209.26 +m4,256,divrem,32x16,kernel,393.93 +m4,256,mul,1x1,auto,9.32 +m4,256,mul,1x1,gmp,3.74 +m4,256,mul,1x1,gmpz,2.73 +m4,256,mul,2x2,auto,13.60 +m4,256,mul,2x2,floor,8.07 +m4,256,mul,2x2,gmp,4.72 +m4,256,mul,2x2,gmpz,6.72 +m4,256,mul,2x2,kernel,6.49 +m4,256,mul,3x3,auto,19.62 +m4,256,mul,3x3,floor,17.00 +m4,256,mul,3x3,gmp,6.15 +m4,256,mul,3x3,gmpz,9.37 +m4,256,mul,3x3,kernel,8.32 +m4,256,mul,4x4,auto,21.81 +m4,256,mul,4x4,floor,16.75 +m4,256,mul,4x4,gmp,8.83 +m4,256,mul,4x4,gmpz,9.86 +m4,256,mul,4x4,kernel,11.06 +m4,256,mul,6x6,auto,27.48 +m4,256,mul,6x6,floor,23.60 +m4,256,mul,6x6,gmp,14.77 +m4,256,mul,6x6,gmpz,18.89 +m4,256,mul,6x6,kernel,18.91 +m4,256,mul,8x8,auto,34.56 +m4,256,mul,8x8,floor,29.95 +m4,256,mul,8x8,gmp,22.41 +m4,256,mul,8x8,gmpz,24.54 +m4,256,mul,8x8,kernel,22.96 +m4,256,mul,12x12,auto,53.88 +m4,256,mul,12x12,floor,49.50 +m4,256,mul,12x12,gmp,43.91 +m4,256,mul,12x12,gmpz,46.79 +m4,256,mul,12x12,kernel,41.05 +m4,256,mul,16x16,auto,82.50 +m4,256,mul,16x16,floor,78.81 +m4,256,mul,16x16,gmp,76.88 +m4,256,mul,16x16,gmpz,78.64 +m4,256,mul,16x16,kernel,69.49 +m4,256,shl,1x13,auto,7.25 +m4,256,shl,1x13,floor,3.63 +m4,256,shl,1x13,gmp,1.24 +m4,256,shl,1x13,gmpz,2.00 +m4,256,shl,1x13,inplace,3.66 +m4,256,shl,1x13,kernel,2.24 +m4,256,shl,1x13,kernelip,1.46 +m4,256,shl,4x13,auto,13.64 +m4,256,shl,4x13,floor,12.69 +m4,256,shl,4x13,gmp,1.99 +m4,256,shl,4x13,gmpz,2.75 +m4,256,shl,4x13,inplace,4.25 +m4,256,shl,4x13,kernel,3.73 +m4,256,shl,4x13,kernelip,3.40 +m4,256,shl,16x13,auto,16.95 +m4,256,shl,16x13,floor,11.95 +m4,256,shl,16x13,gmp,3.48 +m4,256,shl,16x13,gmpz,5.16 +m4,256,shl,16x13,inplace,5.35 +m4,256,shl,16x13,kernel,4.11 +m4,256,shl,16x13,kernelip,6.79 +m4,256,shr,1x13,auto,10.24 +m4,256,shr,1x13,floor,2.49 +m4,256,shr,1x13,gmp,0.99 +m4,256,shr,1x13,gmpz,1.99 +m4,256,shr,1x13,inplace,3.37 +m4,256,shr,1x13,kernel,2.25 +m4,256,shr,4x13,auto,10.22 +m4,256,shr,4x13,floor,4.47 +m4,256,shr,4x13,gmp,1.74 +m4,256,shr,4x13,gmpz,2.74 +m4,256,shr,4x13,inplace,3.77 +m4,256,shr,4x13,kernel,3.74 +m4,256,shr,16x13,auto,17.77 +m4,256,shr,16x13,floor,12.93 +m4,256,shr,16x13,gmp,3.23 +m4,256,shr,16x13,gmpz,4.37 +m4,256,shr,16x13,inplace,5.29 +m4,256,shr,16x13,kernel,4.14 +m4,256,sqr,4x4,auto,21.16 +m4,256,sqr,4x4,floor,16.57 +m4,256,sqr,4x4,gmp,6.98 +m4,256,sqr,4x4,gmpz,9.44 +m4,256,sqr,4x4,kernel,10.91 +m4,256,sqr,16x16,auto,73.87 +m4,256,sqr,16x16,floor,71.56 +m4,256,sqr,16x16,gmp,63.20 +m4,256,sqr,16x16,gmpz,66.18 +m4,256,sqr,16x16,kernel,63.51 +m4,256,sub,1x1,auto,8.57 +m4,256,sub,1x1,floor,1.75 +m4,256,sub,1x1,gmp,1.74 +m4,256,sub,1x1,gmpz,2.50 +m4,256,sub,1x1,inplace,3.72 +m4,256,sub,1x1,kernel,1.06 +m4,256,sub,2x2,auto,8.81 +m4,256,sub,2x2,floor,2.00 +m4,256,sub,2x2,gmp,2.23 +m4,256,sub,2x2,gmpz,2.99 +m4,256,sub,2x2,inplace,3.85 +m4,256,sub,2x2,kernel,1.42 +m4,256,sub,4x4,auto,9.22 +m4,256,sub,4x4,floor,2.49 +m4,256,sub,4x4,gmp,1.99 +m4,256,sub,4x4,gmpz,2.74 +m4,256,sub,4x4,inplace,3.91 +m4,256,sub,4x4,kernel,2.11 +m4,256,sub,8x8,auto,15.72 +m4,256,sub,8x8,floor,11.49 +m4,256,sub,8x8,gmp,2.48 +m4,256,sub,8x8,gmpz,3.23 +m4,256,sub,8x8,inplace,4.01 +m4,256,sub,8x8,kernel,5.79 +m4,256,sub,16x16,auto,18.45 +m4,256,sub,16x16,floor,15.93 +m4,256,sub,16x16,gmp,3.35 +m4,256,sub,16x16,gmpz,4.38 +m4,256,sub,16x16,inplace,5.35 +m4,256,sub,16x16,kernel,11.69 +m4,256,vecsort,100000x1,auto,96.50 +m4,256,vecsort,100000x1,builtin,13.18 +m4,256,vecsort,100000x1,copy,3.77 +m4,512,add,1x1,auto,10.12 +m4,512,add,1x1,floor,1.82 +m4,512,add,1x1,gmp,1.49 +m4,512,add,1x1,gmpz,3.00 +m4,512,add,1x1,inplace,3.58 +m4,512,add,1x1,kernel,1.14 +m4,512,add,2x2,auto,10.86 +m4,512,add,2x2,floor,2.07 +m4,512,add,2x2,gmp,1.86 +m4,512,add,2x2,gmpz,4.63 +m4,512,add,2x2,inplace,4.15 +m4,512,add,2x2,kernel,1.39 +m4,512,add,4x4,auto,9.97 +m4,512,add,4x4,floor,2.58 +m4,512,add,4x4,gmp,1.74 +m4,512,add,4x4,gmpz,3.97 +m4,512,add,4x4,inplace,3.93 +m4,512,add,4x4,kernel,1.96 +m4,512,add,8x8,auto,8.98 +m4,512,add,8x8,floor,12.01 +m4,512,add,8x8,gmp,2.25 +m4,512,add,8x8,gmpz,4.44 +m4,512,add,8x8,inplace,4.38 +m4,512,add,8x8,kernel,3.65 +m4,512,add,16x16,auto,16.45 +m4,512,add,16x16,floor,14.17 +m4,512,add,16x16,gmp,3.23 +m4,512,add,16x16,gmpz,4.22 +m4,512,add,16x16,inplace,5.73 +m4,512,add,16x16,kernel,7.72 +m4,512,divrem,2x1,auto,23.79 +m4,512,divrem,2x1,floor,6.88 +m4,512,divrem,2x1,gmp,5.01 +m4,512,divrem,2x1,gmpz,8.71 +m4,512,divrem,2x1,kernel,5.96 +m4,512,divrem,8x4,auto,72.58 +m4,512,divrem,8x4,floor,47.16 +m4,512,divrem,8x4,gmp,40.99 +m4,512,divrem,8x4,gmpz,43.88 +m4,512,divrem,8x4,kernel,46.42 +m4,512,divrem,32x16,auto,432.82 +m4,512,divrem,32x16,floor,408.56 +m4,512,divrem,32x16,gmp,206.16 +m4,512,divrem,32x16,gmpz,209.06 +m4,512,divrem,32x16,kernel,392.85 +m4,512,mul,1x1,auto,9.21 +m4,512,mul,1x1,gmp,3.73 +m4,512,mul,1x1,gmpz,2.74 +m4,512,mul,2x2,auto,13.80 +m4,512,mul,2x2,floor,7.45 +m4,512,mul,2x2,gmp,4.47 +m4,512,mul,2x2,gmpz,6.67 +m4,512,mul,2x2,kernel,6.62 +m4,512,mul,3x3,auto,14.79 +m4,512,mul,3x3,floor,9.51 +m4,512,mul,3x3,gmp,6.18 +m4,512,mul,3x3,gmpz,9.90 +m4,512,mul,3x3,kernel,8.10 +m4,512,mul,4x4,auto,15.03 +m4,512,mul,4x4,floor,11.20 +m4,512,mul,4x4,gmp,9.29 +m4,512,mul,4x4,gmpz,10.99 +m4,512,mul,4x4,kernel,11.07 +m4,512,mul,6x6,auto,27.98 +m4,512,mul,6x6,floor,23.95 +m4,512,mul,6x6,gmp,14.79 +m4,512,mul,6x6,gmpz,18.19 +m4,512,mul,6x6,kernel,19.27 +m4,512,mul,8x8,auto,34.51 +m4,512,mul,8x8,floor,31.50 +m4,512,mul,8x8,gmp,23.22 +m4,512,mul,8x8,gmpz,24.51 +m4,512,mul,8x8,kernel,23.15 +m4,512,mul,12x12,auto,55.04 +m4,512,mul,12x12,floor,49.42 +m4,512,mul,12x12,gmp,44.07 +m4,512,mul,12x12,gmpz,46.64 +m4,512,mul,12x12,kernel,41.27 +m4,512,mul,16x16,auto,82.77 +m4,512,mul,16x16,floor,78.74 +m4,512,mul,16x16,gmp,77.26 +m4,512,mul,16x16,gmpz,79.14 +m4,512,mul,16x16,kernel,69.46 +m4,512,shl,1x13,auto,6.95 +m4,512,shl,1x13,floor,4.78 +m4,512,shl,1x13,gmp,1.24 +m4,512,shl,1x13,gmpz,1.99 +m4,512,shl,1x13,inplace,3.95 +m4,512,shl,1x13,kernel,2.24 +m4,512,shl,1x13,kernelip,1.75 +m4,512,shl,4x13,auto,8.42 +m4,512,shl,4x13,floor,5.16 +m4,512,shl,4x13,gmp,1.99 +m4,512,shl,4x13,gmpz,2.73 +m4,512,shl,4x13,inplace,4.49 +m4,512,shl,4x13,kernel,3.75 +m4,512,shl,4x13,kernelip,3.38 +m4,512,shl,16x13,auto,20.20 +m4,512,shl,16x13,floor,11.94 +m4,512,shl,16x13,gmp,3.48 +m4,512,shl,16x13,gmpz,4.88 +m4,512,shl,16x13,inplace,5.34 +m4,512,shl,16x13,kernel,4.15 +m4,512,shl,16x13,kernelip,7.22 +m4,512,shr,1x13,auto,9.45 +m4,512,shr,1x13,floor,2.50 +m4,512,shr,1x13,gmp,0.99 +m4,512,shr,1x13,gmpz,1.98 +m4,512,shr,1x13,inplace,3.34 +m4,512,shr,1x13,kernel,2.23 +m4,512,shr,4x13,auto,9.79 +m4,512,shr,4x13,floor,4.47 +m4,512,shr,4x13,gmp,1.74 +m4,512,shr,4x13,gmpz,2.73 +m4,512,shr,4x13,inplace,3.77 +m4,512,shr,4x13,kernel,3.73 +m4,512,shr,16x13,auto,18.40 +m4,512,shr,16x13,floor,13.15 +m4,512,shr,16x13,gmp,3.24 +m4,512,shr,16x13,gmpz,4.24 +m4,512,shr,16x13,inplace,5.30 +m4,512,shr,16x13,kernel,4.16 +m4,512,sqr,4x4,auto,15.40 +m4,512,sqr,4x4,floor,11.30 +m4,512,sqr,4x4,gmp,6.96 +m4,512,sqr,4x4,gmpz,9.38 +m4,512,sqr,4x4,kernel,11.31 +m4,512,sqr,16x16,auto,74.45 +m4,512,sqr,16x16,floor,69.83 +m4,512,sqr,16x16,gmp,62.63 +m4,512,sqr,16x16,gmpz,66.31 +m4,512,sqr,16x16,kernel,61.06 +m4,512,sub,1x1,auto,8.84 +m4,512,sub,1x1,floor,1.74 +m4,512,sub,1x1,gmp,1.49 +m4,512,sub,1x1,gmpz,2.48 +m4,512,sub,1x1,inplace,3.68 +m4,512,sub,1x1,kernel,1.08 +m4,512,sub,2x2,auto,9.18 +m4,512,sub,2x2,floor,2.00 +m4,512,sub,2x2,gmp,1.98 +m4,512,sub,2x2,gmpz,2.85 +m4,512,sub,2x2,inplace,3.85 +m4,512,sub,2x2,kernel,1.42 +m4,512,sub,4x4,auto,9.77 +m4,512,sub,4x4,floor,2.49 +m4,512,sub,4x4,gmp,1.74 +m4,512,sub,4x4,gmpz,2.73 +m4,512,sub,4x4,inplace,3.79 +m4,512,sub,4x4,kernel,1.99 +m4,512,sub,8x8,auto,8.95 +m4,512,sub,8x8,floor,4.15 +m4,512,sub,8x8,gmp,2.24 +m4,512,sub,8x8,gmpz,3.24 +m4,512,sub,8x8,inplace,4.34 +m4,512,sub,8x8,kernel,3.60 +m4,512,sub,16x16,auto,17.88 +m4,512,sub,16x16,floor,15.00 +m4,512,sub,16x16,gmp,3.13 +m4,512,sub,16x16,gmpz,4.29 +m4,512,sub,16x16,inplace,5.29 +m4,512,sub,16x16,kernel,7.81 +m4,512,vecsort,100000x1,auto,98.30 +m4,512,vecsort,100000x1,builtin,13.27 +m4,512,vecsort,100000x1,copy,4.40 diff --git a/tests/beman/big_int/perf/item1_frontend_results.md b/tests/beman/big_int/perf/item1_frontend_results.md new file mode 100644 index 00000000..acab01fb --- /dev/null +++ b/tests/beman/big_int/perf/item1_frontend_results.md @@ -0,0 +1,779 @@ + + +# Item 1 results: cutting the fixed per-call front-end cost + +Item 1 of [`gmp_gap_analysis.md`](gmp_gap_analysis.md) section 6 targeted the 13-18 ns that `+ - << >>` add on top of the +span kernel at <= 16 limbs, the ~21 ns for small `*`, and the 37-66 ns for small divrem. This document records what changed, +how it was measured, and where the targets were and were not met. Numbers are nanoseconds per operation unless noted; +"before" is the library at cf1cf54 and "after" is `opt_1` at b823c5f, both measured with the same harness (`shape_sweep.cpp`, +with the `floor` row added). The inline-capacity decision is left to the reader: the study in section 7 reports costs and +benefits without recommending a default. + +## 1. Summary of the changes + +- **Assignment.** `operator=(&&)` (`assign_value`) steals an rvalue's heap buffer whenever the destination's allocator can + free it, whatever capacity the destination already holds, instead of copying the limbs and freeing the source. A + propagating allocator that compares unequal no longer leaves the destination holding a buffer it cannot free. So + `c = a op b` costs one allocation (the result) plus the kernel, with no copy. +- **No-zero-fill storage.** New `storage_for_overwrite(n)` ensures capacity and discards the contents: no copy, no + value-initialisation at run time (a debug build poison-fills new blocks, a constant evaluation value-initialises). The + multiply and divide dispatchers no longer require a pre-zeroed result; the zero-fill moved into the tiers that need it, so + the basecase no longer pays two fills. `clear_inline_tail` keeps the "inline limbs above the count are zero" invariant. +- **add and sub.** `add_in_place` and `add_into` use the tail-aware `add_n_tail`/`sub_n_tail` (a plain loop under GCC, 4-way unrolled under clang and MSVC; both stop once the + carry or borrow dies), grow only for the carry limb, set sign and size once, and write the result with direct stores into + `storage_for_overwrite` storage: no zero re-scan, no one-step trim. Single-limb operands take a fast path. +- **Shifts.** `c = a << s` and `c = a >> s` build the result from the source in one pass (`lshift_copy`/`rshift_copy`) into an + exactly sized buffer; `<<=` and `>>=` shift in one in-place pass without reserving a spare limb the value does not need + (so `1 << 127` stays inline in a 128-bit `basic_big_int`). A later change that routed in-place pure bit shifts to the older + `shift_left_n`/`shift_right_n` was reverted after it slowed M4 (sections 3.6 and 3.8); under GCC the shift primitives use + register-carried loops. Negative right shifts keep floor rounding by detecting + discarded bits during the pass. +- **Multiply.** `multiply_into` writes into `storage_for_overwrite` storage and drops the fill and the zero re-scan. `*=` + keeps the allocator and capacity (one-limb rhs in place, small products through a stack buffer, larger ones into a fresh + buffer so `x *= x` works). A header-side basecase shortcut (`multiply_basecase_runtime`, `mul_header_basecase_enabled`) + skips the scratch hooks and the tier ladder when `min(la, lb) < 32` (section 6). +- **Divide.** `divmod_into` uses `storage_for_overwrite` and a 128-limb stack arena for small-shape scratch, builds the + remainder from its trimmed span instead of over-reserving, and a 2x1 quotient that fits inline no longer allocates. `/=` + and `%=` keep the object's allocator and capacity (`const basic_big_int temp(std::move(*this), m_alloc); set_zero();`). +- **Bug fixes (found while making the above safe).** Stale inline-tail limbs when the inline capacity is above one limb + (`set_zero`, `shift_right`, `divmod_in_place_short`, the move-assign steal path) made `inplace_to_bit_uint` and so `==` + against builtins wrong for `basic_big_int<128>`; base 8 and 32 `from_chars` into a reused heap destination OR-ed digits + into stale limbs (this affected `big_int` itself); the move-assign fast path propagated a POCCA/POCMA allocator but kept + the heap buffer it could no longer free. All have regression tests (`inline_tail`, `allocation`, `front_end_*`, + `muldiv_frontend`, `dispatch_contract`). + +## 2. Method + +- **Harness.** `shape_sweep` (see `perf/tools/README.md`). Rows: `auto` is `c = a op b` into a reused destination, `inplace` + is `c += b; c -= b` (shifts: `<<=` then `>>=`) halved, `kernel` is the span kernel, `floor` is the kernel plus one + `std::allocator` allocate/deallocate pair of the result size (two for divrem: quotient and remainder), and `gmpz` is GMP + `mpz` with the destination reused. The floor is the true API floor of `c = a op b`: no pair is charged when the result + fits the inline capacity of the integer type under test (at 64 bits that affects shr 1x13 and the 1-limb remainder of + divrem 2x1). `auto - floor` is what the front end costs on top of the unavoidable allocation. The shift `kernel` row + includes the copy-in of the source, so for shifts `auto - floor` can go negative once `auto` is a single pass. +- **Machines.** x64: i9-11900K, Ubuntu 24.04, g++-14.3 `-O2 -march=native`, SIMD_MUL, IFMA multiply kernels + (`BMI2_ADX=1 AVX512_IFMA=1`), every sweep pinned with `taskset -c 2`, box idle. M4: Apple M4 Max, appleclang 21, + SIMD_MUL, nothing pinned, machine idle and not building during timing. GMP 6.3.0 on both. +- **Runs.** Before and after binaries were interleaved: seeds 1 and 2, two repeats each, so four runs per binary and machine + (`gap_sweep.sh --ops add,sub,shl,shr,mul,sqr,divrem,fromchars --bands small,medium`). Reported values are the median of the + four runs; divrem shapes of up to 32 limbs are the mean over the two seeds (the operands change the Knuth D path, see + caveats). Large-band spot checks are 3-round runs per seed. The per-run CSVs (183 files) were aggregated and removed; the medians + and means behind every table here are in `perf/item1/` (section 10 lists the files): `final_x64.csv`, `final_m4.csv`, + `study.csv`, `shortcut.csv` and `shift_ab.csv`, each with its build configuration (`#const` keys) in the header. +- **Binaries.** Before: harness at cf1cf54 (`base64`). After: `opt_1` (`new64`). The inline-capacity study uses `new64`, + `new128`, `new256`, `new512` (`-DBEMAN_BIG_INT_SWEEP_INLINE_BITS`). The `#const` line of every binary was checked + (`SIMD_MUL=1`, `ndebug=1`, `sweep_inline_bits`, `sweep_int_sizeof`, x64 `BMI2_ADX=1 AVX512_IFMA=1`, `gcc-14.3.0`), and + `selftest.sh` passed on all of them. + +## 3. Before and after + +Cells read `before -> after`. `auto/gmpz` is the user-visible gap to GMP (above 1 means `big_int` is slower). The tables are +the re-measurement on `opt_1` (the add/sub, shift and single-limb follow-ups and the mul-add fix included): all rows come +from the d689f75 runs except the shl/shr rows, which were re-measured at b823c5f, the commit that changes only the shift +code (`final_x64.csv` and `final_m4.csv`: `after_commit` says which; `d689f75_ns` and `first_9145b4e_ns` hold the earlier +measurements); one interleaved check of add/sub/mul/sqr/divrem at b823c5f found +nothing else moved. Rows are the median of four runs per binary. An earlier measurement of 9145b4e is described in +section 3.6. + +### 3.1 x64 (g++-14, -march=native), small band + +| op | shape | auto ns | inplace ns | auto - floor ns | auto/gmpz | auto - kernel ns | +|---|---|---:|---:|---:|---:|---:| +| add | 1x1 | 8.2 -> 5.4 | 4.1 -> 2.6 | -1.6 -> -4.4 | 1.98 -> 1.29 | 6.3 -> 3.4 | +| add | 2x2 | 17.9 -> 15.9 | 5.9 -> 6.2 | 7.6 -> 5.7 | 4.29 -> 3.82 | 15.5 -> 13.5 | +| add | 4x4 | 18.0 -> 15.4 | 5.6 -> 5.7 | 7.1 -> 4.5 | 4.12 -> 3.53 | 15.0 -> 12.4 | +| add | 8x8 | 19.3 -> 16.9 | 7.3 -> 7.3 | 6.8 -> 4.4 | 4.12 -> 3.62 | 14.7 -> 12.4 | +| add | 16x16 | 25.3 -> 21.1 | 11.8 -> 11.2 | 9.8 -> 5.6 | 3.87 -> 3.23 | 17.7 -> 13.5 | +| sub | 1x1 | 9.0 -> 5.5 | 4.0 -> 2.5 | 7.1 -> 3.4 | 2.08 -> 1.26 | 7.5 -> 4.0 | +| sub | 2x2 | 19.1 -> 15.7 | 4.6 -> 4.8 | 9.1 -> 5.8 | 4.39 -> 3.61 | 17.0 -> 13.5 | +| sub | 4x4 | 19.0 -> 16.5 | 5.5 -> 5.6 | 8.3 -> 5.9 | 4.12 -> 3.60 | 16.2 -> 13.7 | +| sub | 8x8 | 20.6 -> 18.1 | 7.2 -> 7.3 | 8.2 -> 5.9 | 4.14 -> 3.64 | 16.2 -> 13.7 | +| sub | 16x16 | 25.7 -> 21.0 | 10.6 -> 10.0 | 10.3 -> 5.7 | 3.78 -> 3.09 | 17.7 -> 13.6 | +| shl | 1x13 | 16.9 -> 12.7 | 4.2 -> 3.5 | 7.0 -> 2.9 | 4.86 -> 3.88 | 14.8 -> 10.6 | +| shl | 4x13 | 18.7 -> 13.4 | 5.3 -> 4.4 | 7.5 -> 2.2 | 4.63 -> 3.32 | 15.2 -> 9.9 | +| shl | 16x13 | 24.6 -> 16.8 | 9.3 -> 8.6 | 9.3 -> 1.5 | 4.22 -> 2.88 | 17.1 -> 9.3 | +| shr | 1x13 | 10.3 -> 8.4 | 3.6 -> 3.1 | 8.0 -> 6.1 | 3.14 -> 2.56 | 8.6 -> 6.7 | +| shr | 4x13 | 21.0 -> 15.9 | 4.8 -> 4.2 | 9.0 -> 4.7 | 5.28 -> 4.01 | 17.5 -> 12.4 | +| shr | 16x13 | 25.9 -> 20.2 | 9.0 -> 8.2 | 8.9 -> 4.7 | 3.96 -> 3.26 | 17.7 -> 11.7 | +| mul | 2x2 | 28.2 -> 18.9 | - | 13.1 -> 3.9 | 4.61 -> 3.08 | 20.7 -> 11.3 | +| mul | 3x3 | 29.7 -> 20.3 | - | 13.1 -> 3.9 | 2.97 -> 2.03 | 20.7 -> 11.4 | +| mul | 4x4 | 32.0 -> 22.4 | - | 13.4 -> 4.2 | 2.63 -> 1.84 | 20.9 -> 11.5 | +| mul | 6x6 | 42.1 -> 28.0 | - | 17.4 -> 3.5 | 2.31 -> 1.54 | 24.6 -> 10.8 | +| mul | 8x8 | 47.5 -> 36.0 | - | 14.9 -> 3.4 | 1.65 -> 1.24 | 22.0 -> 10.5 | +| mul | 12x12 | 70.2 -> 57.4 | - | 15.3 -> 2.5 | 1.38 -> 1.12 | 22.4 -> 9.5 | +| mul | 16x16 | 101.8 -> 88.6 | - | 15.4 -> 1.8 | 1.21 -> 1.05 | 22.5 -> 9.4 | +| sqr | 4x4 | 33.3 -> 22.2 | - | 16.4 -> 5.5 | 3.77 -> 2.51 | 24.2 -> 12.3 | +| sqr | 16x16 | 80.1 -> 67.3 | - | 17.5 -> 4.2 | 1.52 -> 1.29 | 23.9 -> 9.5 | +| divrem | 2x1 | 41.7 -> 36.2 | - | 28.9 -> 23.4 | 3.34 -> 2.92 | 36.2 -> 30.5 | +| divrem | 8x4 | 112.7 -> 94.1 | - | 52.9 -> 26.3 | 2.57 -> 2.13 | 67.0 -> 43.8 | +| divrem | 32x16 | 370.9 -> 342.2 | - | 67.4 -> 36.1 | 1.91 -> 1.77 | 82.4 -> 53.9 | +| fromchars | 4x10 | 194.1 -> 115.6 | - | - | 2.85 -> 1.70 | 155.9 -> 77.6 | +| fromchars | 16x10 | 421.8 -> 425.2 | - | - | 1.63 -> 1.66 | 204.4 -> 225.2 | +| tochars | 4x10 | 134.1 -> 132.0 | - | - | 1.38 -> 1.35 | -46.9 -> -48.9 | +| tochars | 16x10 | 1171.4 -> 1168.1 | - | - | 1.99 -> 1.98 | 132.4 -> 135.3 | + +### 3.2 M4 (appleclang), small band + +| op | shape | auto ns | inplace ns | auto - floor ns | auto/gmpz | auto - kernel ns | +|---|---|---:|---:|---:|---:|---:| +| add | 1x1 | 6.9 -> 5.9 | 3.2 -> 1.9 | -2.5 -> -3.7 | 2.26 -> 1.97 | 5.8 -> 4.7 | +| add | 2x2 | 14.8 -> 15.9 | 3.8 -> 4.1 | 4.9 -> 6.3 | 2.52 -> 4.90 | 13.4 -> 14.5 | +| add | 4x4 | 15.4 -> 14.6 | 3.7 -> 3.8 | 4.5 -> 4.2 | 3.19 -> 3.57 | 13.4 -> 12.6 | +| add | 8x8 | 15.8 -> 15.6 | 5.1 -> 4.3 | 3.9 -> 3.6 | 3.02 -> 3.56 | 12.3 -> 12.1 | +| add | 16x16 | 16.4 -> 17.7 | 9.8 -> 5.7 | 2.2 -> 3.7 | 3.62 -> 3.95 | 8.8 -> 10.1 | +| sub | 1x1 | 7.6 -> 5.7 | 3.4 -> 2.0 | 5.8 -> 4.0 | 2.76 -> 2.30 | 6.5 -> 4.7 | +| sub | 2x2 | 15.0 -> 14.7 | 3.3 -> 3.8 | 5.5 -> 5.4 | 4.64 -> 5.14 | 13.6 -> 13.3 | +| sub | 4x4 | 15.8 -> 15.1 | 3.9 -> 4.0 | 5.8 -> 5.2 | 5.31 -> 5.52 | 13.9 -> 13.2 | +| sub | 8x8 | 16.3 -> 15.7 | 5.0 -> 4.4 | 4.6 -> 4.3 | 4.65 -> 4.86 | 12.7 -> 12.2 | +| sub | 16x16 | 18.4 -> 18.1 | 10.9 -> 5.4 | 3.0 -> 3.0 | 4.22 -> 4.13 | 10.7 -> 10.3 | +| shl | 1x13 | 17.5 -> 12.3 | 2.3 -> 2.5 | 6.5 -> 0.9 | 8.82 -> 6.14 | 15.3 -> 10.0 | +| shl | 4x13 | 19.0 -> 13.1 | 3.6 -> 3.8 | 5.9 -> -0.1 | 6.98 -> 4.73 | 15.3 -> 9.4 | +| shl | 16x13 | 21.5 -> 16.1 | 5.8 -> 5.4 | 9.4 -> 3.9 | 4.26 -> 3.09 | 17.4 -> 12.0 | +| shr | 1x13 | 9.5 -> 6.6 | 2.4 -> 3.3 | 7.0 -> 4.1 | 4.77 -> 3.31 | 7.3 -> 4.3 | +| shr | 4x13 | 17.8 -> 15.2 | 3.3 -> 3.6 | 5.4 -> 2.9 | 6.41 -> 5.56 | 14.0 -> 11.5 | +| shr | 16x13 | 21.6 -> 18.5 | 5.1 -> 5.2 | 8.3 -> 5.2 | 5.07 -> 4.35 | 17.4 -> 14.4 | +| mul | 2x2 | 28.1 -> 18.2 | - | 13.4 -> 3.7 | 4.29 -> 2.81 | 21.5 -> 11.9 | +| mul | 3x3 | 30.1 -> 20.0 | - | 13.2 -> 3.0 | 3.57 -> 1.91 | 21.8 -> 11.9 | +| mul | 4x4 | 29.7 -> 21.6 | - | 12.6 -> 4.8 | 2.87 -> 1.97 | 18.5 -> 10.6 | +| mul | 6x6 | 33.0 -> 27.2 | - | 9.2 -> 2.9 | 1.76 -> 1.46 | 13.9 -> 7.9 | +| mul | 8x8 | 41.9 -> 35.1 | - | 11.0 -> 4.7 | 1.56 -> 1.41 | 18.7 -> 12.1 | +| mul | 12x12 | 61.8 -> 53.8 | - | 12.0 -> 4.3 | 1.33 -> 1.15 | 20.3 -> 12.4 | +| mul | 16x16 | 94.8 -> 82.4 | - | 15.7 -> 4.0 | 1.20 -> 1.05 | 24.9 -> 12.8 | +| sqr | 4x4 | 29.6 -> 21.6 | - | 13.2 -> 5.3 | 3.14 -> 2.22 | 18.2 -> 10.3 | +| sqr | 16x16 | 88.9 -> 75.8 | - | 18.3 -> 4.0 | 1.35 -> 1.15 | 26.7 -> 13.9 | +| divrem | 2x1 | 31.1 -> 25.2 | - | 17.3 -> 11.5 | 3.53 -> 2.89 | 25.1 -> 19.2 | +| divrem | 8x4 | 96.0 -> 75.4 | - | 34.8 -> 14.6 | 2.20 -> 1.72 | 49.8 -> 28.8 | +| divrem | 32x16 | 481.3 -> 432.0 | - | 63.6 -> 21.3 | 2.28 -> 2.05 | 79.1 -> 39.6 | +| fromchars | 4x10 | 91.2 -> 91.9 | - | - | 1.19 -> 1.20 | 58.6 -> 58.9 | +| fromchars | 16x10 | 399.5 -> 394.3 | - | - | 1.26 -> 1.25 | 206.1 -> 201.8 | +| tochars | 4x10 | 84.0 -> 84.4 | - | - | 0.86 -> 0.88 | -19.0 -> -17.8 | +| tochars | 16x10 | 828.8 -> 841.7 | - | - | 1.37 -> 1.40 | 88.0 -> 103.1 | + +### 3.3 x64, medium band + +| op | shape | auto ns | inplace ns | auto - floor ns | auto/gmpz | auto - kernel ns | +|---|---|---:|---:|---:|---:|---:| +| add | 64x64 | 63.7 -> 46.2 | 38.9 -> 35.8 | 22.4 -> 5.1 | 3.86 -> 2.80 | 30.4 -> 13.4 | +| add | 256x256 | 223.9 -> 162.9 | 145.8 -> 144.7 | 65.1 -> 4.4 | 2.82 -> 2.05 | 81.3 -> 20.6 | +| add | 1024x1024 | 841.9 -> 599.1 | 583.0 -> 581.0 | 247.6 -> 5.1 | 2.15 -> 1.53 | 263.4 -> 21.3 | +| add | 2000x2000 | 1721.6 -> 1153.3 | 1134.8 -> 1132.4 | 571.1 -> 3.9 | 3.56 -> 2.40 | 588.6 -> 21.7 | +| sub | 64x64 | 65.9 -> 45.3 | 37.7 -> 34.6 | 23.8 -> 3.1 | 3.94 -> 2.70 | 33.0 -> 12.6 | +| sub | 1024x1024 | 842.3 -> 598.3 | 581.5 -> 579.9 | 247.9 -> 4.1 | 2.15 -> 1.53 | 264.0 -> 20.5 | +| shl | 256x13 | 136.9 -> 121.8 | 93.1 -> 98.7 | 19.7 -> 4.0 | 2.69 -> 2.43 | 41.7 -> 26.6 | +| shl | 2000x13 | 916.7 -> 773.4 | 657.7 -> 712.5 | 177.8 -> 34.0 | 2.69 -> 2.28 | 197.2 -> 54.0 | +| shl | 2000x77 | 1066.1 -> 774.4 | 764.8 -> 712.6 | 194.2 -> -98.4 | 3.09 -> 2.25 | 216.0 -> -75.5 | +| shr | 256x13 | 139.7 -> 133.7 | 93.7 -> 98.3 | 7.5 -> 2.9 | 2.71 -> 2.61 | 28.1 -> 23.1 | +| shr | 2000x13 | 950.7 -> 787.9 | 657.2 -> 711.5 | 118.1 -> -44.1 | 2.78 -> 2.31 | 138.9 -> -24.1 | +| shr | 2000x77 | 1084.3 -> 789.8 | 765.0 -> 712.1 | 233.6 -> -60.9 | 3.16 -> 2.31 | 255.9 -> -39.5 | +| mul | 32x32 | 212.3 -> 205.8 | - | 12.1 -> 9.3 | 0.78 -> 0.76 | 24.7 -> 18.9 | +| mul | 64x64 | 524.5 -> 496.2 | - | 31.0 -> 3.7 | 0.60 -> 0.58 | 41.5 -> 9.5 | +| mul | 256x256 | 6231.5 -> 6176.4 | - | 28.3 -> 2.4 | 0.80 -> 0.79 | 62.6 -> -2.4 | +| mul | 1000x1000 | 59904.3 -> 59861.8 | - | 343.6 -> 312.4 | 1.12 -> 1.12 | 389.1 -> 230.9 | +| mul | 2000x2000 | 177072.6 -> 176923.7 | - | 722.9 -> 199.2 | 1.23 -> 1.23 | 692.2 -> -137.4 | +| mul | 2000x100 | 17852.9 -> 17643.6 | - | 154.6 -> -43.6 | 0.50 -> 0.49 | 193.6 -> -52.9 | +| sqr | 64x64 | 301.5 -> 284.4 | - | 20.2 -> 3.8 | 0.54 -> 0.51 | 30.8 -> 9.1 | +| sqr | 512x512 | 10627.6 -> 10594.1 | - | 56.8 -> -15.5 | 0.70 -> 0.70 | 64.8 -> 27.0 | +| sqr | 2000x2000 | 99937.6 -> 99323.2 | - | 651.3 -> 1067.7 | 0.99 -> 0.99 | 646.3 -> 794.2 | +| divrem | 128x64 | 3342.1 -> 3446.3 | - | 67.0 -> 82.1 | 1.94 -> 2.00 | 141.1 -> 103.2 | +| divrem | 512x256 | 29558.6 -> 29817.4 | - | 292.6 -> 178.0 | 1.81 -> 1.83 | 164.7 -> 226.5 | +| divrem | 2000x1000 | 185318.8 -> 186774.1 | - | 1166.1 -> 819.0 | 1.34 -> 1.35 | 1065.1 -> 1041.0 | +| divrem | 4000x2000 | 478481.8 -> 477627.1 | - | 3873.1 -> -100.7 | 1.43 -> 1.43 | 3326.4 -> -564.6 | +| divrem | 2000x100 | 144654.0 -> 144971.4 | - | 1928.4 -> 1154.1 | 2.00 -> 2.00 | 812.6 -> 886.6 | +| divrem | 65536x1 | 193721.3 -> 168835.7 | - | 18973.6 -> 1400.2 | 1.31 -> 1.14 | 26598.0 -> 1889.2 | +| fromchars | 64x10 | 2790.5 -> 2852.3 | - | - | 2.04 -> 2.08 | 667.6 -> 1045.0 | +| fromchars | 256x10 | 28563.9 -> 27920.0 | - | - | 2.87 -> 2.80 | 2874.6 -> 3489.4 | +| fromchars | 2000x10 | 429183.1 -> 385669.1 | - | - | 2.17 -> 1.95 | 20511.4 -> 23105.6 | +| tochars | 64x10 | 5622.5 -> 5695.4 | - | - | 1.90 -> 1.93 | 377.4 -> 362.4 | +| tochars | 256x10 | 35873.4 -> 36990.2 | - | - | 1.82 -> 1.88 | 1283.7 -> 1461.4 | +| tochars | 2000x10 | 608668.2 -> 620642.7 | - | - | 1.49 -> 1.52 | 24355.7 -> 23098.2 | + +### 3.4 M4, medium band + +| op | shape | auto ns | inplace ns | auto - floor ns | auto/gmpz | auto - kernel ns | +|---|---|---:|---:|---:|---:|---:| +| add | 64x64 | 54.7 -> 37.9 | 51.6 -> 20.0 | -6.4 -> -23.0 | 4.21 -> 2.84 | 5.8 -> -11.2 | +| add | 256x256 | 148.8 -> 109.8 | 240.0 -> 101.2 | -99.1 -> -136.9 | 2.57 -> 1.89 | -85.2 -> -126.7 | +| add | 1024x1024 | 564.9 -> 450.3 | 989.7 -> 437.1 | -423.8 -> -537.8 | 2.23 -> 1.78 | -418.7 -> -527.5 | +| add | 2000x2000 | 1132.3 -> 877.0 | 1948.4 -> 858.2 | -812.1 -> -1054.3 | 2.27 -> 1.76 | -797.7 -> -1050.8 | +| sub | 64x64 | 41.2 -> 36.1 | 51.2 -> 19.8 | -19.0 -> -25.1 | 2.97 -> 2.68 | -8.6 -> -17.3 | +| sub | 1024x1024 | 562.3 -> 454.6 | 988.4 -> 435.0 | -424.5 -> -532.1 | 2.21 -> 1.80 | -416.2 -> -524.8 | +| shl | 256x13 | 87.5 -> 41.5 | 36.5 -> 29.8 | 29.7 -> -16.6 | 2.30 -> 1.08 | 38.0 -> -8.2 | +| shl | 2000x13 | 653.6 -> 260.4 | 283.7 -> 195.9 | 252.2 -> -141.8 | 0.30 -> 0.15 | 267.7 -> -124.8 | +| shl | 2000x77 | 796.3 -> 452.3 | 425.8 -> 228.4 | 253.6 -> -92.1 | 2.74 -> 1.55 | 268.9 -> -76.9 | +| shr | 256x13 | 87.6 -> 42.9 | 35.8 -> 30.4 | 12.0 -> -33.0 | 2.27 -> 1.13 | 21.2 -> -23.7 | +| shr | 2000x13 | 692.2 -> 273.8 | 282.5 -> 196.0 | 164.0 -> -252.4 | 2.32 -> 0.89 | 179.5 -> -239.9 | +| shr | 2000x77 | 1064.4 -> 276.0 | 420.0 -> 196.1 | 535.7 -> -254.6 | 3.67 -> 0.96 | 546.1 -> -245.5 | +| mul | 32x32 | 298.7 -> 281.4 | - | 17.5 -> 2.3 | 1.12 -> 1.07 | 39.3 -> 23.1 | +| mul | 64x64 | 1039.3 -> 1007.8 | - | 23.9 -> -2.6 | 1.34 -> 1.30 | 53.3 -> 22.9 | +| mul | 256x256 | 11896.3 -> 12025.0 | - | 102.9 -> 238.8 | 1.51 -> 1.53 | 138.2 -> 229.3 | +| mul | 1000x1000 | 107250.2 -> 106887.8 | - | 178.5 -> -283.6 | 1.94 -> 1.94 | 80.4 -> 117.0 | +| mul | 2000x2000 | 302932.5 -> 301689.0 | - | -292.8 -> -722.3 | 2.03 -> 2.03 | 668.5 -> -700.3 | +| mul | 2000x100 | 47136.3 -> 46406.5 | - | 294.5 -> -291.4 | 1.35 -> 1.34 | 280.3 -> -261.8 | +| sqr | 64x64 | 626.4 -> 600.5 | - | 24.4 -> 0.7 | 0.93 -> 0.89 | 41.3 -> 18.3 | +| sqr | 512x512 | 24695.3 -> 24317.0 | - | 276.0 -> -72.0 | 1.25 -> 1.18 | 327.2 -> -27.6 | +| sqr | 2000x2000 | 224588.0 -> 223255.6 | - | 967.2 -> -545.5 | 1.85 -> 1.84 | 895.4 -> 31.3 | +| divrem | 128x64 | 3762.2 -> 3733.0 | - | 93.6 -> 75.2 | 2.12 -> 2.10 | 141.9 -> 143.5 | +| divrem | 512x256 | 28815.2 -> 28675.6 | - | 417.8 -> 72.8 | 1.75 -> 1.74 | 466.0 -> 215.7 | +| divrem | 2000x1000 | 234126.7 -> 232426.2 | - | 1975.2 -> 102.0 | 1.73 -> 1.73 | 1606.5 -> 322.1 | +| divrem | 4000x2000 | 695982.0 -> 691382.2 | - | 3426.0 -> 4024.5 | 2.00 -> 2.01 | 3967.5 -> 4357.2 | +| divrem | 2000x100 | 95716.3 -> 94547.3 | - | 1235.2 -> 259.8 | 1.38 -> 1.36 | 1567.7 -> 5.9 | +| divrem | 65536x1 | 246390.5 -> 231899.7 | - | 17772.4 -> -655.0 | 1.44 -> 1.35 | 16128.0 -> 1720.2 | +| fromchars | 64x10 | 2198.3 -> 2187.2 | - | - | 1.37 -> 1.36 | 338.6 -> 330.9 | +| fromchars | 256x10 | 13836.4 -> 13804.5 | - | - | 1.28 -> 1.29 | 2272.2 -> 2257.5 | +| fromchars | 2000x10 | 300171.0 -> 300852.0 | - | - | 1.47 -> 1.48 | 17382.0 -> 17573.8 | +| tochars | 64x10 | 4746.0 -> 4744.5 | - | - | 1.50 -> 1.50 | 250.2 -> 220.3 | +| tochars | 256x10 | 33794.6 -> 33818.9 | - | - | 1.66 -> 1.66 | 858.7 -> 1075.2 | +| tochars | 2000x10 | 679612.5 -> 676793.1 | - | - | 1.63 -> 1.63 | 11123.6 -> 7941.4 | + +### 3.5 Large-band spot checks + +x64: + +| op | shape | auto ns | kernel ns | gmpz ns | auto/gmpz | change in auto | +|---|---|---:|---:|---:|---:|---:| +| add | 16384x16384 | 14810 -> 9330 | 9304 -> 9294 | 6898 | 2.15 -> 1.35 | -37.0% | +| divrem | 4096x2048 | 499034 -> 498477 | 497946 -> 495569 | 368707 | 1.35 -> 1.35 | -0.1% | +| fromchars | 10000x10 | 3670159 -> 3475893 | 3588344 -> 3362265 | 2005997 | 1.83 -> 1.73 | -5.3% | +| mul | 2000x2000 | 177314 -> 177858 | 176337 -> 176741 | 144537 | 1.23 -> 1.23 | +0.3% | +| mul | 16384x16384 | 3778231 -> 3779466 | 3778094 -> 3788186 | 1789186 | 2.11 -> 2.11 | +0.0% | +| tochars | 10000x10 | 5864746 -> 5960832 | 5844018 -> 5920813 | 4223943 | 1.39 -> 1.41 | +1.6% | + +M4: + +| op | shape | auto ns | kernel ns | gmpz ns | auto/gmpz | change in auto | +|---|---|---:|---:|---:|---:|---:| +| add | 16384x16384 | 9474 -> 7305 | 16179 -> 16418 | 4161 | 2.28 -> 1.76 | -22.9% | +| divrem | 4096x2048 | 706592 -> 707065 | 704150 -> 706127 | 358386 | 1.97 -> 1.97 | +0.1% | +| fromchars | 10000x10 | 4198921 -> 4189262 | 4137033 -> 4117279 | 1872966 | 2.24 -> 2.24 | -0.2% | +| mul | 2000x2000 | 303919 -> 311426 | 304464 -> 305902 | 150382 | 2.02 -> 2.07 | +2.5% | +| mul | 16384x16384 | 5443656 -> 5522911 | 5447302 -> 5449136 | 1599088 | 3.40 -> 3.45 | +1.5% | +| tochars | 10000x10 | 8278333 -> 8342500 | 8238222 -> 8247500 | 4195346 | 1.97 -> 1.99 | +0.8% | + +### 3.6 Reading the tables, regressions, and the history of the two measurements + +- Small `auto` on x64 improved by 2-4.5 ns for add/sub at 2-16 limbs (1x1: 8.2 -> 5.4 add, 9.0 -> 5.5 sub with the 1x1 fast + path), 2-7 ns for shifts, 9-14 ns for mul/sqr and 5-35 ns for divrem. On M4, add/sub at 2-16 limbs are within about + +-1.3 ns of the baseline (add 2x2 14.8 -> 15.9, 16x16 16.4 -> 17.7; its allocator pair is cheap, a floor only 3-6 ns above + the kernel, so there was little to remove; add/sub 1x1 improved 6.9 -> 5.9 / 7.6 -> 5.7); M4 shifts improved by 3-6 ns, + mul/sqr by 6-15 ns and divrem by 7-47 ns. +- Medium and large `auto`: x64 add/sub went from -5% at 64 limbs to -29% at 1024-2000 (add 1024x1024 842 -> 599 ns, 2000x2000 + 1722 -> 1153), with `auto - floor` down to 4-5 ns; add 16384 is -37% (x64) and -23% (M4). Shifts at 2000 limbs -16..-31% + (x64) and -42..-74% (M4); mul and divrem above ~256 limbs within +-1.5% except M4 mul 2000x2000 +2.5% and divrem 65536x1 + -10% (x64). +- **x64 `+=`/`-=` regression from the first measurement is fixed.** At 9145b4e add `inplace` 1024x1024 was 583 -> 790 ns + (+35%) because `add_n_tail` did not reach the speed of `add_unsigned_spans` on x64; the follow-up loop shape (a plain loop + under GCC, the 4-way one under clang) brings it to 581 ns, the baseline. +- **Decimal `fromchars` regression is fixed** (it was +8-11% at 64-256 limbs-worth of digits at 9145b4e); see section 3.7 + for the cause. Now x64 `auto` 64x10 2790 -> 2852 ns (+2%), 256x10 28564 -> 27920 (-2%), 2000x10 429k -> 386k (-10%), + 16x10 +0.8%; 10000x10 is -5.3%. M4 is unchanged (<= +-1%). +- **In-place shifts (`c <<= s; c >>= s`, per op) after the shift fix (b823c5f).** The history is: the first build + (9145b4e, single in-place `lshift_copy`/`rshift_copy` pass) was faster than the baseline on M4 and slower on x64; + d689f75 routed whole-limb-free in-place shifts to `shift_left_n`/`shift_right_n`, which made M4 much slower; b823c5f + reverts that and uses register-carried loops under GCC only (section 3.8): + + | `inplace` ns | x64 base | x64 9145b4e | x64 d689f75 | x64 b823c5f | M4 base | M4 9145b4e | M4 d689f75 | M4 b823c5f | + |---|---:|---:|---:|---:|---:|---:|---:|---:| + | shl 4x13 | 5.2 | 4.6 | 5.3 | 4.4 | 3.6 | 3.8 | 5.5 | 3.8 | + | shl 16x13 | 9.2 | 9.1 | 9.3 | 8.6 | 5.8 | 5.3 | 6.9 | 5.4 | + | shl 256x13 | 93.1 | 102.2 | 99.3 | 98.7 | 36.5 | 29.9 | 45.1 | 29.8 | + | shl 2000x13 | 657.7 | 740.9 | 716.9 | 712.5 | 283.7 | 196.0 | 347.8 | 195.9 | + | shr 2000x13 | 657.2 | 738.7 | 716.5 | 711.5 | 282.5 | 196.8 | 347.5 | 196.0 | + + Now M4 in-place shifts are 6-53% faster than the baseline at >= 16 limbs (shl/shr 2000x13 -31%, 256x13 -15..-18%, + 131072x13 -6%) and four small-shape rows are slower by 0.2-1.0 ns (shr 1x13 2.4 -> 3.3 ns +41%, shr 4x13 3.3 -> 3.6 +10%, + shl 1x13 2.3 -> 2.5 +9%, shl 4x13 3.6 -> 3.8 +6%). On x64 the 1-16-limb rows are 8-17% faster than the baseline but + >= 256 limbs stay +5-8% slower (256x13 +5-6%, 2000x13 +8%, 131072x13 +7%); the `auto` rows keep their gains on both + machines (x64 -4..-49%, M4 -14..-74%), with x64 shr 256x13 `auto` down to -4% from -9%. +- **Rows more than 3% slower than the baseline** (`item1_regress.py` on the d689f75 runs with the shift rows replaced by the + b823c5f runs; `auto` and `inplace`, small and medium bands, four-run medians): + - x64: 9 of 107 rows. The in-place shift rows shl/shr 256x13 (+5-6%) and 2000x13 (+8%); add 2x2 `inplace` +0.4 ns + (+5.9%), sub 2x2/4x4 `inplace` +0.2 ns (+4.4%, +3.4%); tochars 256x10 `auto` +3.1% and divrem 128x64 `auto` +3.1% + (both borderline). Kernel rows (library code, not the front end): divrem 8x4 45.7 -> 50.3 ns (+10%), divrem 128x64 + +4.4%, sqr 4x4 9.1 -> 9.9 ns (+9%), within the 1-2 ns kernel noise at the smallest sizes. The 131072x13 in-place shift + spot check is +7% (+3 us). + - M4: 9 of 107 rows. The four small in-place shift rows above (+0.2-1.0 ns); add 2x2 `auto` +1.1 ns (+7%), add 16x16 + `auto` +1.3 ns (+7.7%), add/sub 2x2 `inplace` +0.3-0.6 ns, sub 4x4 `inplace` +0.1 ns. + - No medium or large `auto` row on either machine is more than 3% slower, and no large spot check other than the x64 + 131072x13 in-place shift (+7%) moves by more than +2.5% (M4 mul 2000x2000). + - A single interleaved check at b823c5f of add/sub/mul/sqr/divrem in the small band (one run per binary) found nothing + else moved against the d689f75 numbers beyond noise (x64 add 2x2 `inplace` 6.5 vs 6.2 ns; the M4 mul 6x6 and add 16x16 + `auto` rows flip between their usual noise modes). + - These are the open items for item 2: the x64 GCC in-place shift gap at >= 256 limbs (+5-8%; the baseline in-place path + is faster than every variant tried), the sub-nanosecond small add/sub in-place rows on both machines, and the M4 small + add `auto` rows (+1.1-1.3 ns). +- **History.** The first measurement (opt_1 at 9145b4e, the `first_9145b4e_ns` column) showed x64 add/sub `inplace` +23..+35% at + >= 64 limbs, decimal `fromchars` +8-11%, and in-place shifts +10-13% on x64 but -30..-35% on M4. The follow-ups fixed the + first two and the add/sub `auto` rows improved a further 20-26%; the in-place shift change in d689f75 traded the x64 + result for an M4 loss, and b823c5f (revert plus per-compiler loops) restored the M4 result and kept most of the x64 one. + +### 3.7 Cause of the x64 `fromchars` regression (diagnosis and fix) + +The regression showed with the header basecase shortcut compiled in and disappeared with it off, but the shortcut was only +the trigger. Facts: + +- Instructions per call are equal (+-2%) in the baseline, shortcut-on and shortcut-off builds, so it is not extra work; + IPC is lower with the shortcut (3.93 against 4.17-4.20 in `perf stat`, with 92% of uops from the decoded-uop cache + against 99%). It is a stable +8-10%, independent of ASLR, stack offset and heap layout (checked with `setarch -R`, + environment padding, `GLIBC_TUNABLES`). +- `digits_to_limbs` is byte-identical between the two builds apart from nop padding, with the hot loops at the same + alignment, and 99.96% of the samples are in the out-of-line `digits_to_limbs> [clone .isra.0]`. + Called standalone (inlined into a micro driver), the same function takes 20.9 us at 256 limbs in both builds, against + 25.4 us for the baseline in the harness: the harness copy is slower even without the shortcut. +- The cause is the hot loop of `mul_add_single_limb_in_place`: `widening_mul` returns the product through + `wide::from_int`, a `std::bit_cast` of the 128-bit value, and GCC 14 in that out-of-line copy stores the product to the + stack and reloads both halves every iteration (`mulx; mov %r8,(%rsp); mov %r9,0x8(%rsp); mov (%rsp),%r8; mov 0x8(%rsp),%r9; + add ...`). That store-forward round trip is the baseline's ~20% penalty; the shortcut changes inlining and register + allocation in the harness translation unit enough to make the spilled loop a further ~8-10% slower. +- Fix (`d689f75`): under `BEMAN_BIG_INT_HAS_INT128_FUNDAMENTAL` compute `wide(s[i]) * mul + carry` and split it directly, + keeping the `widening_mul` loop in the `#else`. The loop becomes `mulx; add; adc; mov; inc; cmp; jb`. Measured in a + scratch copy (decimal `fromchars` kernel row, base / shortcut-on / shortcut-off / fixed): 64x10 2116 / 2400 / 2110-2160 / + 1680-1800 ns, 256x10 25.7k / 29.8k / 25.4k / 21.4-22.0k, 2000x10 401k / 398k / 401k / 361k. Changing `from_int` or + `widening_mul` globally instead helps `fromchars` but regresses divrem 512x256 by +17% and tochars by +10%, so the fix + has to stay local to this loop. + +### 3.8 In-place shift A/B (variants of the whole-limb-free path) + +Three variants built from scratch copies of d689f75 (outside the repository) and measured against `new64` (d689f75) and +`base64`, three interleaved runs per binary on both machines (rows `auto`, `inplace`, `kernel`; the unchanged `kernel` +rows are omitted; x64 pinned): + +- V1: revert 2f9150b, so `<<=`/`>>=` go back to `lshift_copy`/`rshift_copy` (the earlier A2 behaviour); +- V2: V1 plus register-carried loops in `lshift_copy`/`rshift_copy` (one load per limb); +- V3: V2 with the combine written as `funnel_shl`/`funnel_shr` (identical to V2 under GCC, which has no builtin). + +x64 (g++-14): + +| op | shape | row | base64 | new64 | v1 | v2 | v3 | +|---|---|---|---:|---:|---:|---:|---:| +| shl | 1x13 | auto | 16.7 | 12.5 (-25%) | 12.6 (-25%) | 12.6 (-25%) | 12.6 (-25%) | +| shl | 1x13 | inplace | 4.1 | 4.2 (+3%) | 3.6 (-11%) | 3.4 (-15%) | 3.4 (-15%) | +| shl | 4x13 | auto | 18.6 | 13.4 (-28%) | 13.4 (-28%) | 13.4 (-28%) | 13.4 (-28%) | +| shl | 4x13 | inplace | 5.2 | 5.2 (+2%) | 4.5 (-12%) | 4.3 (-16%) | 4.3 (-16%) | +| shl | 16x13 | auto | 24.5 | 17.1 (-30%) | 17.0 (-31%) | 16.7 (-32%) | 16.8 (-32%) | +| shl | 16x13 | inplace | 9.2 | 9.2 (+1%) | 9.1 (-1%) | 8.4 (-8%) | 8.4 (-8%) | +| shl | 256x13 | auto | 133.9 | 122.0 (-9%) | 122.2 (-9%) | 122.0 (-9%) | 121.9 (-9%) | +| shl | 256x13 | inplace | 93.0 | 99.3 (+7%) | 102.2 (+10%) | 98.7 (+6%) | 98.7 (+6%) | +| shl | 2000x13 | auto | 917.5 | 772.8 (-16%) | 773.4 (-16%) | 773.6 (-16%) | 773.5 (-16%) | +| shl | 2000x77 | auto | 1063.4 | 774.6 (-27%) | 775.0 (-27%) | 774.5 (-27%) | 774.5 (-27%) | +| shl | 2000x13 | inplace | 656.7 | 716.9 (+9%) | 739.0 (+13%) | 712.5 (+9%) | 712.6 (+9%) | +| shl | 2000x77 | inplace | 765.7 | 741.7 (-3%) | 739.0 (-3%) | 712.5 (-7%) | 712.5 (-7%) | +| shl | 131072x13 | auto | 97977.3 | 49725.2 (-49%) | 49751.9 (-49%) | 49864.2 (-49%) | 49877.8 (-49%) | +| shl | 131072x13 | inplace | 43553.8 | 46689.0 (+7%) | 48260.2 (+11%) | 46565.6 (+7%) | 46554.9 (+7%) | +| shr | 1x13 | auto | 10.1 | 8.2 (-19%) | 8.2 (-19%) | 8.3 (-17%) | 8.3 (-17%) | +| shr | 1x13 | inplace | 3.6 | 3.9 (+7%) | 3.3 (-10%) | 3.0 (-17%) | 3.0 (-17%) | +| shr | 4x13 | auto | 20.9 | 16.1 (-23%) | 16.0 (-23%) | 15.8 (-24%) | 15.8 (-24%) | +| shr | 4x13 | inplace | 4.7 | 4.9 (+5%) | 4.3 (-8%) | 4.1 (-13%) | 4.1 (-13%) | +| shr | 16x13 | auto | 25.8 | 20.0 (-22%) | 20.0 (-22%) | 20.2 (-21%) | 20.2 (-21%) | +| shr | 16x13 | inplace | 8.7 | 8.9 (+2%) | 8.7 (-1%) | 8.0 (-8%) | 8.0 (-8%) | +| shr | 256x13 | auto | 136.6 | 125.0 (-9%) | 125.2 (-8%) | 133.6 (-2%) | 133.7 (-2%) | +| shr | 256x13 | inplace | 93.7 | 99.1 (+6%) | 101.7 (+8%) | 98.3 (+5%) | 98.3 (+5%) | +| shr | 2000x13 | auto | 949.9 | 749.0 (-21%) | 749.6 (-21%) | 788.0 (-17%) | 788.0 (-17%) | +| shr | 2000x77 | auto | 1086.8 | 749.5 (-31%) | 750.0 (-31%) | 790.0 (-27%) | 790.0 (-27%) | +| shr | 2000x13 | inplace | 657.1 | 716.5 (+9%) | 737.4 (+12%) | 711.5 (+8%) | 711.7 (+8%) | +| shr | 2000x77 | inplace | 765.5 | 740.8 (-3%) | 739.3 (-3%) | 712.1 (-7%) | 712.1 (-7%) | +| shr | 131072x13 | auto | 98933.2 | 48304.4 (-51%) | 48328.0 (-51%) | 50322.4 (-49%) | 50313.5 (-49%) | +| shr | 131072x13 | inplace | 43515.1 | 46701.2 (+7%) | 48481.1 (+11%) | 46567.5 (+7%) | 46580.6 (+7%) | + +- new64: 8 of 28 auto/inplace rows more than 3% slower than base64: shl 2000x13 inplace +9%, shr 2000x13 inplace +9%, shr 131072x13 inplace +7%, shl 131072x13 inplace +7%, shl 256x13 inplace +7%, shr 1x13 inplace +7%, shr 256x13 inplace +6%, shr 4x13 inplace +5% +- v1: 6 of 28 auto/inplace rows more than 3% slower than base64: shl 2000x13 inplace +13%, shr 2000x13 inplace +12%, shr 131072x13 inplace +11%, shl 131072x13 inplace +11%, shl 256x13 inplace +10%, shr 256x13 inplace +8% +- v2: 6 of 28 auto/inplace rows more than 3% slower than base64: shl 2000x13 inplace +9%, shr 2000x13 inplace +8%, shr 131072x13 inplace +7%, shl 131072x13 inplace +7%, shl 256x13 inplace +6%, shr 256x13 inplace +5% +- v3: 6 of 28 auto/inplace rows more than 3% slower than base64: shl 2000x13 inplace +9%, shr 2000x13 inplace +8%, shr 131072x13 inplace +7%, shl 131072x13 inplace +7%, shl 256x13 inplace +6%, shr 256x13 inplace +5% + +M4 (appleclang): + +| op | shape | row | base64 | new64 | v1 | v2 | v3 | +|---|---|---|---:|---:|---:|---:|---:| +| shl | 1x13 | auto | 18.5 | 12.2 (-34%) | 12.5 (-32%) | 12.3 (-34%) | 12.1 (-35%) | +| shl | 1x13 | inplace | 2.3 | 3.4 (+48%) | 2.5 (+8%) | 2.5 (+8%) | 2.5 (+8%) | +| shl | 4x13 | auto | 19.9 | 12.7 (-36%) | 13.0 (-35%) | 13.2 (-33%) | 13.1 (-34%) | +| shl | 4x13 | inplace | 3.6 | 5.2 (+46%) | 3.8 (+5%) | 3.7 (+4%) | 3.8 (+7%) | +| shl | 16x13 | auto | 21.4 | 15.7 (-27%) | 15.5 (-28%) | 15.5 (-28%) | 16.3 (-24%) | +| shl | 16x13 | inplace | 5.8 | 6.8 (+18%) | 5.3 (-9%) | 6.3 (+9%) | 6.5 (+12%) | +| shl | 256x13 | auto | 85.6 | 41.3 (-52%) | 41.9 (-51%) | 56.3 (-34%) | 63.0 (-26%) | +| shl | 256x13 | inplace | 36.2 | 44.9 (+24%) | 29.7 (-18%) | 40.1 (+11%) | 48.0 (+32%) | +| shl | 2000x13 | auto | 639.0 | 264.0 (-59%) | 267.0 (-58%) | 395.3 (-38%) | 453.0 (-29%) | +| shl | 2000x77 | auto | 806.2 | 342.5 (-58%) | 478.0 (-41%) | 394.8 (-51%) | 452.9 (-44%) | +| shl | 2000x13 | inplace | 282.3 | 345.7 (+22%) | 196.1 (-31%) | 315.7 (+12%) | 375.2 (+33%) | +| shl | 2000x77 | inplace | 424.7 | 228.6 (-46%) | 230.9 (-46%) | 315.6 (-26%) | 377.1 (-11%) | +| shl | 131072x13 | auto | 48761.7 | 18032.3 (-63%) | 18150.9 (-63%) | 25035.9 (-49%) | 28931.4 (-41%) | +| shl | 131072x13 | inplace | 19766.8 | 22825.0 (+15%) | 18524.2 (-6%) | 20843.9 (+5%) | 24609.1 (+24%) | +| shr | 1x13 | auto | 9.2 | 6.7 (-27%) | 6.9 (-25%) | 6.8 (-26%) | 6.7 (-27%) | +| shr | 1x13 | inplace | 2.4 | 3.1 (+30%) | 3.3 (+40%) | 2.8 (+17%) | 2.8 (+18%) | +| shr | 4x13 | auto | 18.4 | 14.3 (-22%) | 15.1 (-18%) | 14.0 (-24%) | 14.3 (-22%) | +| shr | 4x13 | inplace | 3.3 | 4.5 (+37%) | 3.7 (+13%) | 3.3 (+1%) | 3.5 (+8%) | +| shr | 16x13 | auto | 21.5 | 18.1 (-16%) | 18.7 (-13%) | 18.1 (-16%) | 18.2 (-15%) | +| shr | 16x13 | inplace | 5.1 | 6.1 (+19%) | 5.2 (+1%) | 5.5 (+8%) | 5.7 (+11%) | +| shr | 256x13 | auto | 87.1 | 42.5 (-51%) | 43.1 (-50%) | 43.6 (-50%) | 48.3 (-45%) | +| shr | 256x13 | inplace | 35.8 | 44.6 (+25%) | 30.4 (-15%) | 39.7 (+11%) | 47.3 (+32%) | +| shr | 2000x13 | auto | 657.7 | 262.0 (-60%) | 258.7 (-61%) | 289.1 (-56%) | 340.8 (-48%) | +| shr | 2000x77 | auto | 1015.8 | 251.8 (-75%) | 266.6 (-74%) | 299.9 (-70%) | 339.2 (-67%) | +| shr | 2000x13 | inplace | 282.0 | 345.0 (+22%) | 197.8 (-30%) | 313.6 (+11%) | 374.1 (+33%) | +| shr | 2000x77 | inplace | 420.1 | 195.8 (-53%) | 195.7 (-53%) | 314.8 (-25%) | 377.9 (-10%) | +| shr | 131072x13 | auto | 50334.1 | 18098.3 (-64%) | 18178.1 (-64%) | 18015.5 (-64%) | 20633.5 (-59%) | +| shr | 131072x13 | inplace | 19886.4 | 22564.1 (+13%) | 18551.6 (-7%) | 20782.3 (+5%) | 24608.1 (+24%) | + +- new64: 12 of 28 auto/inplace rows more than 3% slower than base64: shl 1x13 inplace +48%, shl 4x13 inplace +46%, shr 4x13 inplace +37%, shr 1x13 inplace +30%, shr 256x13 inplace +25%, shl 256x13 inplace +24%, shl 2000x13 inplace +22%, shr 2000x13 inplace +22%, shr 16x13 inplace +19%, shl 16x13 inplace +18%, shl 131072x13 inplace +15%, shr 131072x13 inplace +13% +- v1: 4 of 28 auto/inplace rows more than 3% slower than base64: shr 1x13 inplace +40%, shr 4x13 inplace +13%, shl 1x13 inplace +8%, shl 4x13 inplace +5% +- v2: 11 of 28 auto/inplace rows more than 3% slower than base64: shr 1x13 inplace +17%, shl 2000x13 inplace +12%, shr 2000x13 inplace +11%, shr 256x13 inplace +11%, shl 256x13 inplace +11%, shl 16x13 inplace +9%, shl 1x13 inplace +8%, shr 16x13 inplace +8%, shl 131072x13 inplace +5%, shr 131072x13 inplace +5%, shl 4x13 inplace +4% +- v3: 12 of 28 auto/inplace rows more than 3% slower than base64: shl 2000x13 inplace +33%, shr 2000x13 inplace +33%, shl 256x13 inplace +32%, shr 256x13 inplace +32%, shl 131072x13 inplace +24%, shr 131072x13 inplace +24%, shr 1x13 inplace +18%, shl 16x13 inplace +12%, shr 16x13 inplace +11%, shl 1x13 inplace +8%, shr 4x13 inplace +8%, shl 4x13 inplace +7% + +Reading: no variant keeps every `auto`/`inplace` row within 3% of the baseline on both machines. On M4 (clang) V1 is the +best: it keeps the `auto` gains (-51..-64% at >= 256 limbs), makes every medium and large in-place row faster than the +baseline (-6..-31%), and leaves four small-shape in-place rows more than 3% slower, all by under 1 ns; the register-carried +loops (V2, V3) are slower than the original two-load form under clang. On x64 (GCC) V2/V3 improve the small in-place rows +by 8-17% against the baseline and equal `new64` at >= 256 limbs (+6-9%), at the price of 4-7 points on shr `auto`; the +baseline in-place path is still 6-9% faster there than any variant. + +Outcome: b823c5f reverts 2f9150b and uses the register-carried loop bodies under GCC only (the original two-load loops +under clang and MSVC), the per-compiler split these numbers suggested. Re-measured at b823c5f (four interleaved runs per +binary on each machine, plus a 131072x13 spot) the result matches V1 on M4 and V2 on x64 (section 3.6). +The pairings measured are GCC 14 on x64 and appleclang on AArch64; clang on x64 and GCC on AArch64 were not. + +## 4. Plan targets + +Targets from the plan, with the achieved values (median of four runs; b823c5f for shifts, d689f75 for the rest). Kernel-row noise at <= 16 limbs is 1-2 ns +(section 8), so a miss of less than ~1.5 ns on an `inplace - kernel` target is within noise. + +| target | x64 | M4 | +|---|---|---| +| `inplace - kernel` <= 3 ns for add/sub/shl/shr at 2-16 limbs | **mostly met**: add 3.9/2.7/2.8/3.6 (2/4/8/16 limbs, before 3.5/2.5/2.7/4.3), sub 2.7/2.8/3.0/2.6, shl 0.9/1.1, shr 0.7/-0.3 (add 2x2 and 16x16 are within kernel noise of the line) | **met**: add 2.7/1.9/0.8/-1.8, sub 2.4/2.1/0.9/-2.3, shl 0.0/1.3, shr -0.1/1.0 | +| `auto - floor` <= 3 ns at <= 16 limbs, add/sub/shl/shr | **not met except shifts**: add 5.7/4.5/4.4/5.6 (before 7.6/7.1/6.8/9.8; 1x1 -4.4), sub 5.8/5.9/5.9/5.7, shl 2.2/1.5 (met), shr 4.7/4.7 | **mostly not met**: add 6.3/4.2/3.6/3.7, sub 5.4/5.2/4.3/3.0, shl -0.1/3.9, shr 2.9/5.2 | +| `auto - floor` <= 5 ns at <= 16 limbs, mul/sqr | **met**: mul 1.8-4.2 (before 13.1-17.4); sqr 5.5 (4x4, marginal) and 4.2 | **met**: mul 2.9-4.8 (before 9.2-15.7); sqr 5.3 (4x4, marginal) and 4.0 | +| x64 add 1000 `auto`: 840 -> <= 650 ns | **met**: add 1024x1024 842 -> 599 ns (2000x2000 1722 -> 1153; `auto - floor` 4-5 ns at 64-2000 limbs) | n/a (M4: 565 -> 450 ns) | +| divrem 8x4 `auto - kernel`: 66 -> <= 2 floors + 5 ns (i.e. `auto - floor` <= 5) | **not met**: `auto - kernel` 67.0 -> 43.8, `auto - floor` 52.9 -> 26.3 | **not met**: `auto - kernel` 49.8 -> 28.8, `auto - floor` 34.8 -> 14.6 | +| medium and large rows: no regression > 3% | **not met only for in-place shifts at >= 256 limbs**: shl/shr 256x13 +5-6%, 2000x13 +8%, 131072x13 +7%; every other medium and large `auto` and `inplace` row and every other large spot check is within 3% | **met for medium and large rows**; only small rows exceed 3% (four in-place shift rows by 0.2-1.0 ns, add 2x2/16x16 `auto` +1.1/+1.3 ns, sub/add 2x2 `inplace` +0.3-0.6 ns); largest large spot check M4 mul 2000x2000 +2.5% | + +The allocation targets in `alloc_count.test.cpp` all hold (section 5). + +## 5. Allocation counts + +Steady-state allocations per call (one warm-up call, then 64 calls, `alloc_count.test.cpp`), before -> after where a baseline +exists. Environments: `big_int` with a counting allocator, `pmr::big_int` with a counting default memory resource, and +`basic_big_int`. All cases are enforced (`item1_pending = false`); the 128 and 512 columns were not +measured at cf1cf54. `c = a / b (2x1)` needs the heap for the 2-limb quotient at 64 bits (1 allocation); it is 0 from 128 +bits up. + +| case | `big_int` (counting allocator) | `pmr::big_int` | inline 128 | inline 256 | inline 512 | +|---|---|---|---|---|---| +| `c = a + b (16x16)` | 1 -> 1 | 1 -> 1 | 1 | 1 -> 1 | 1 | +| `c = a - b (16x16)` | 1 -> 1 | 1 -> 1 | 1 | 1 -> 1 | 1 | +| `c += b; c -= b (16x16)` | 0 -> 0 | 0 -> 0 | 0 | 0 -> 0 | 0 | +| `c = a << 13 (16 limbs)` | 1 -> 1 | 1 -> 1 | 1 | 1 -> 1 | 1 | +| `c = a >> 13 (16 limbs)` | 1 -> 1 | 1 -> 1 | 1 | 1 -> 1 | 1 | +| `c <<= 13; c >>= 13 (16 limbs)` | 0 -> 0 | 0 -> 0 | 0 | 0 -> 0 | 0 | +| `c = a * b (8x8)` | 1 -> 1 | 1 -> 1 | 1 | 1 -> 1 | 1 | +| `c = a; c *= b (8x8 -> 16)` | 1 -> 0 | 1 -> 0 | 0 | 1 -> 0 | 0 | +| `c = a; c *= 7 (8 limbs)` | 1 -> 0 | 1 -> 0 | 0 | 1 -> 0 | 0 | +| `c = a / b (8x4)` | 2 -> 1 | 2 -> 1 | 1 | 2 -> 1 | 0 | +| `c = a % b (8x4)` | 2 -> 1 | 2 -> 1 | 1 | 2 -> 0 | 0 | +| `div_rem_to_zero (8x4)` | 4 -> 2 | 4 -> 2 | 2 | 3 -> 1 | 0 | +| `c = a / b (2x1)` | 1 | 1 | - | - | - | +| `c = a; c /= b (8x4) [stretch]` | 3 -> 0 | 3 -> 0 | 0 | 2 -> 0 | 0 | +| `c = a; c %= b (8x4) [stretch]` | 2 -> 0 | 2 -> 0 | 0 | 2 -> 0 | 0 | +| `from_chars 2000 digits (reserved)` | 2 -> 2 | 2 -> 2 | 2 | 2 -> 2 | 2 | +| `c = a / b (2x1, inline q)` | - | - | 0 | 0 -> 0 | 0 | + +The baseline `div_rem_to_zero` counts are the quotient (5 limbs), a 22-limb scratch and a remainder over-reserved to 9 limbs, +plus one more where the operands need normalising (an earlier version of the test also counted its own check's temporaries). +For the inline-capacity study, allocations per call at 1, 2, 4 and 8 limbs (operands with the top bit set, so sums, shifts +and products have the full limb count; `a / d` is n limbs over n/2): + +| op | limbs | 64 | 128 | 256 | 512 | +|---|---:|---:|---:|---:|---:| +| `c = a + b` | 1 | 1 | 0 | 0 | 0 | +| `c = a - b` | 1 | 0 | 0 | 0 | 0 | +| `c = a << 13` | 1 | 1 | 0 | 0 | 0 | +| `c = a >> 13` | 1 | 0 | 0 | 0 | 0 | +| `c = a * b` | 1 | 1 | 0 | 0 | 0 | +| `c = a * a` | 1 | 1 | 0 | 0 | 0 | +| `c = a / d (n x n/2)` | 1 | 0 | 0 | 0 | 0 | +| `div_rem_to_zero (n x n/2)` | 1 | 0 | 0 | 0 | 0 | +| `c = a + b` | 2 | 1 | 1 | 0 | 0 | +| `c = a - b` | 2 | 1 | 0 | 0 | 0 | +| `c = a << 13` | 2 | 1 | 1 | 0 | 0 | +| `c = a >> 13` | 2 | 1 | 0 | 0 | 0 | +| `c = a * b` | 2 | 1 | 1 | 0 | 0 | +| `c = a * a` | 2 | 1 | 1 | 0 | 0 | +| `c = a / d (n x n/2)` | 2 | 1 | 0 | 0 | 0 | +| `div_rem_to_zero (n x n/2)` | 2 | 1 | 0 | 0 | 0 | +| `c = a + b` | 4 | 1 | 1 | 1 | 0 | +| `c = a - b` | 4 | 1 | 1 | 0 | 0 | +| `c = a << 13` | 4 | 1 | 1 | 1 | 0 | +| `c = a >> 13` | 4 | 1 | 1 | 0 | 0 | +| `c = a * b` | 4 | 1 | 1 | 1 | 0 | +| `c = a * a` | 4 | 1 | 1 | 1 | 0 | +| `c = a / d (n x n/2)` | 4 | 1 | 1 | 0 | 0 | +| `div_rem_to_zero (n x n/2)` | 4 | 2 | 1 | 0 | 0 | +| `c = a + b` | 8 | 1 | 1 | 1 | 1 | +| `c = a - b` | 8 | 1 | 1 | 1 | 0 | +| `c = a << 13` | 8 | 1 | 1 | 1 | 1 | +| `c = a >> 13` | 8 | 1 | 1 | 1 | 0 | +| `c = a * b` | 8 | 1 | 1 | 1 | 1 | +| `c = a * a` | 8 | 1 | 1 | 1 | 1 | +| `c = a / d (n x n/2)` | 8 | 1 | 1 | 1 | 0 | +| `div_rem_to_zero (n x n/2)` | 8 | 2 | 2 | 1 | 0 | + +## 6. Header basecase shortcut (`mul_header_basecase_enabled`) + +Measured on the 9145b4e tree, before the follow-ups: `new64` (shortcut on) against a build of the same tree with +`mul_header_basecase_enabled = false` (edited in a scratch copy, not committed), interleaved, three runs each, `auto` row: + +x64: + +| op | shape | shortcut on (new64) ns | off ns | on - off ns | runs on / off | +|---|---|---:|---:|---:|---| +| mul | 2x2 | 18.9 | 22.7 | -3.8 | 18.9/18.9/18.9 vs 22.7/22.7/22.7 | +| mul | 3x3 | 20.3 | 24.6 | -4.3 | 20.3/20.4/20.3 vs 24.6/24.4/24.7 | +| mul | 4x4 | 22.3 | 26.3 | -4.0 | 22.3/22.4/22.3 vs 26.3/26.3/26.4 | +| mul | 6x6 | 28.0 | 32.8 | -4.8 | 27.9/28.0/28.0 vs 33.4/32.7/32.8 | +| mul | 8x8 | 35.7 | 41.1 | -5.5 | 35.7/35.7/35.7 vs 41.3/41.1/41.0 | +| mul | 12x12 | 57.7 | 62.6 | -4.9 | 57.7/57.6/57.8 vs 62.6/62.6/62.5 | +| mul | 16x16 | 95.0 | 115.5 | -20.5 | 96.1/95.0/95.0 vs 99.8/115.5/116.0 | +| sqr | 4x4 | 22.0 | 27.3 | -5.3 | 22.2/22.0/21.9 vs 27.3/27.4/27.3 | +| sqr | 16x16 | 68.8 | 72.8 | -4.0 | 68.8/66.3/77.2 vs 71.2/73.3/72.8 | + +M4: + +| op | shape | shortcut on (new64) ns | off ns | on - off ns | runs on / off | +|---|---|---:|---:|---:|---| +| mul | 2x2 | 19.9 | 19.4 | +0.5 | 18.6/19.9/20.2 vs 19.4/19.3/19.9 | +| mul | 3x3 | 20.3 | 21.2 | -1.0 | 20.1/20.4/20.3 vs 23.8/21.2/21.1 | +| mul | 4x4 | 21.5 | 22.3 | -0.8 | 21.6/21.5/21.2 vs 22.2/22.3/22.4 | +| mul | 6x6 | 27.3 | 28.1 | -0.7 | 27.3/27.1/27.4 vs 28.1/28.3/27.6 | +| mul | 8x8 | 34.8 | 35.8 | -1.1 | 34.8/35.1/34.8 vs 35.9/35.7/35.8 | +| mul | 12x12 | 54.3 | 54.5 | -0.1 | 53.9/54.3/54.5 vs 54.2/54.6/54.5 | +| mul | 16x16 | 82.8 | 83.4 | -0.7 | 82.2/83.5/82.8 vs 82.9/83.4/83.9 | +| sqr | 4x4 | 21.5 | 23.2 | -1.7 | 21.4/21.5/21.6 vs 23.2/23.1/23.3 | +| sqr | 16x16 | 76.1 | 78.0 | -1.9 | 76.1/74.9/76.1 vs 78.3/78.0/77.3 | + +Verdict: on x64 the shortcut is worth 3.8-5.5 ns on mul 2x2..12x12 and 4-5 ns on sqr 4 and 16 (the 16x16 difference of 20 ns +is noisy: one of the three shortcut-off runs was 99.8 ns and two were 115 ns, the rest of the data show ~4 ns), clearly above +the 1 ns threshold. On M4 it is worth -1.1 to +0.5 ns on mul 2..16 (in the noise of the runs; 2x2 is 0.5 ns slower with it) and 1.7-1.9 ns on sqr 4 and 16, so +it is at best marginal there. The x64 `fromchars` regression seen at that commit appeared only with the shortcut compiled in, but its cause was a spilled +product in `mul_add_single_limb_in_place` (section 3.7), which is fixed; with the fix the shortcut has no known cost, so +the verdict is: keep it on x64 (-4..5 ns, 13-20% of the small mul/sqr rows) and it is harmless and marginally useful on M4. +The A/B was not repeated on d689f75. + +## 7. Inline-capacity study + +**Decision (2026-10-06): the default inline capacity stays at 64 bits (`big_int`, one limb).** The data below (`study.csv`) +is kept for reference. + +Measured on the 9145b4e tree (before the add/sub, shift and single-limb follow-ups and the mul-add fix); the study was not +repeated on d689f75, so small add/sub/shift rows are a few ns better there at every N. The same front-end code with the +inline capacity of the integer type set to N = 64 (`big_int`), 128, 256 and 512 bits. +`sizeof`: 16, 24, 40 and 72 bytes. `auto` and `inplace` are medians of two runs (seeds 1 and 2, so divrem is the mean of +seeds); the vecsort row sorts and sums a `std::vector` of 100000 one-limb values of mixed sign (copy-assign + `std::sort` + +sum, ns per element; the builtin row is `std::vector`; the copy row is the copy-assign alone). + +### 7.1 x64 + +**auto ns** + +| op | shape | N=64 | N=128 | N=256 | N=512 | +|---|---|---:|---:|---:|---:| +| add | 1x1 | 6.7 | 8.3 | 8.6 | 8.7 | +| add | 4x4 | 15.4 | 17.6 | 10.6 | 10.6 | +| add | 16x16 | 21.0 | 24.2 | 23.5 | 23.5 | +| sub | 1x1 | 7.8 | 9.6 | 10.0 | 10.2 | +| sub | 4x4 | 16.6 | 19.5 | 12.1 | 12.1 | +| sub | 16x16 | 21.1 | 24.5 | 23.5 | 25.8 | +| shl | 1x13 | 12.6 | 7.2 | 8.2 | 8.3 | +| shl | 4x13 | 13.4 | 14.9 | 16.3 | 9.1 | +| shl | 16x13 | 17.1 | 18.2 | 20.5 | 19.5 | +| shr | 1x13 | 8.2 | 8.9 | 9.1 | 9.3 | +| shr | 4x13 | 16.0 | 16.8 | 11.9 | 11.9 | +| shr | 16x13 | 19.9 | 20.9 | 21.7 | 22.0 | +| mul | 1x1 | 16.5 | 10.5 | 10.8 | 10.8 | +| mul | 2x2 | 18.9 | 20.3 | 14.3 | 14.3 | +| mul | 4x4 | 22.3 | 24.0 | 24.3 | 17.8 | +| mul | 8x8 | 35.6 | 37.4 | 38.0 | 37.9 | +| mul | 16x16 | 88.5 | 90.4 | 91.9 | 91.3 | +| sqr | 4x4 | 22.1 | 23.8 | 24.4 | 17.4 | +| sqr | 16x16 | 67.2 | 68.0 | 69.3 | 70.2 | +| divrem | 2x1 | 35.4 | 34.5 | 36.7 | 36.6 | +| divrem | 8x4 | 91.9 | 94.9 | 89.1 | 80.2 | +| divrem | 32x16 | 339.6 | 344.6 | 343.7 | 342.3 | + +**inplace ns** + +| op | shape | N=64 | N=128 | N=256 | N=512 | +|---|---|---:|---:|---:|---:| +| add | 1x1 | 4.9 | 4.9 | 4.9 | 4.9 | +| add | 4x4 | 5.7 | 6.1 | 5.7 | 5.7 | +| add | 16x16 | 11.7 | 13.1 | 12.8 | 11.9 | +| sub | 1x1 | 4.7 | 4.7 | 4.6 | 4.6 | +| sub | 4x4 | 5.6 | 6.0 | 5.6 | 5.6 | +| sub | 16x16 | 11.1 | 11.1 | 11.2 | 11.0 | +| shl | 1x13 | 3.7 | 4.3 | 4.3 | 4.3 | +| shl | 4x13 | 4.6 | 4.8 | 4.7 | 5.2 | +| shl | 16x13 | 9.1 | 8.9 | 9.6 | 8.9 | +| shr | 1x13 | 3.3 | 3.4 | 3.5 | 3.4 | +| shr | 4x13 | 4.3 | 4.4 | 4.5 | 4.5 | +| shr | 16x13 | 8.7 | 8.5 | 8.5 | 9.0 | + +**vecsort, ns per element (100000 one-limb values: copy + sort + sum)** + +| row | N=64 | N=128 | N=256 | N=512 | +|---|---:|---:|---:|---:| +| auto | 86.13 | 92.31 | 98.59 | 103.44 | +| builtin | 39.44 | 39.39 | 39.41 | 39.33 | +| copy | 1.13 | 2.50 | 2.62 | 3.14 | + +### 7.2 M4 + +**auto ns** + +| op | shape | N=64 | N=128 | N=256 | N=512 | +|---|---|---:|---:|---:|---:| +| add | 1x1 | 7.5 | 8.3 | 9.2 | 10.1 | +| add | 4x4 | 14.1 | 14.2 | 9.3 | 10.0 | +| add | 16x16 | 17.0 | 16.3 | 16.6 | 16.4 | +| sub | 1x1 | 7.9 | 8.4 | 8.6 | 8.8 | +| sub | 4x4 | 14.4 | 14.0 | 9.2 | 9.8 | +| sub | 16x16 | 18.3 | 17.7 | 18.5 | 17.9 | +| shl | 1x13 | 11.9 | 6.6 | 7.3 | 6.9 | +| shl | 4x13 | 12.7 | 12.7 | 13.6 | 8.4 | +| shl | 16x13 | 15.5 | 15.5 | 17.0 | 20.2 | +| shr | 1x13 | 7.0 | 8.9 | 10.2 | 9.4 | +| shr | 4x13 | 13.9 | 14.0 | 10.2 | 9.8 | +| shr | 16x13 | 18.2 | 17.7 | 17.8 | 18.4 | +| mul | 1x1 | 14.5 | 9.1 | 9.3 | 9.2 | +| mul | 2x2 | 18.6 | 18.5 | 13.6 | 13.8 | +| mul | 4x4 | 21.6 | 21.6 | 21.8 | 15.0 | +| mul | 8x8 | 35.3 | 35.0 | 34.6 | 34.5 | +| mul | 16x16 | 82.8 | 82.8 | 82.5 | 82.8 | +| sqr | 4x4 | 22.8 | 23.2 | 21.2 | 15.4 | +| sqr | 16x16 | 75.6 | 74.2 | 73.9 | 74.5 | +| divrem | 2x1 | 24.6 | 23.2 | 23.6 | 23.8 | +| divrem | 8x4 | 75.0 | 75.2 | 74.7 | 72.6 | +| divrem | 32x16 | 429.7 | 430.0 | 431.3 | 432.8 | + +**inplace ns** + +| op | shape | N=64 | N=128 | N=256 | N=512 | +|---|---|---:|---:|---:|---:| +| add | 1x1 | 3.5 | 3.5 | 3.4 | 3.6 | +| add | 4x4 | 3.5 | 3.5 | 3.8 | 3.9 | +| add | 16x16 | 5.6 | 5.6 | 5.7 | 5.7 | +| sub | 1x1 | 3.5 | 3.6 | 3.7 | 3.7 | +| sub | 4x4 | 3.4 | 3.7 | 3.9 | 3.8 | +| sub | 16x16 | 5.2 | 5.3 | 5.3 | 5.3 | +| shl | 1x13 | 2.5 | 3.5 | 3.7 | 4.0 | +| shl | 4x13 | 3.8 | 3.8 | 4.2 | 4.5 | +| shl | 16x13 | 5.3 | 5.4 | 5.3 | 5.3 | +| shr | 1x13 | 3.3 | 3.4 | 3.4 | 3.3 | +| shr | 4x13 | 3.6 | 6.9 | 3.8 | 3.8 | +| shr | 16x13 | 5.2 | 5.2 | 5.3 | 5.3 | + +**vecsort, ns per element (100000 one-limb values: copy + sort + sum)** + +| row | N=64 | N=128 | N=256 | N=512 | +|---|---:|---:|---:|---:| +| auto | 87.87 | 94.78 | 96.50 | 98.30 | +| builtin | 13.21 | 13.38 | 13.18 | 13.27 | +| copy | 2.70 | 3.90 | 3.77 | 4.40 | + +Neutral summary of the trade-off (no default is recommended here): + +- **What a larger inline capacity removes.** Results that fit inline skip the allocator entirely: at 128 bits a 2-limb + product, a 1-limb sum or a 2x1 quotient no longer allocates, at 256 bits operands up to 3-4 limbs, at 512 bits up to ~8. + That is 4-11 ns per operation in the rows where the result fits (x64: add 4x4 15.4 ns at N=64 against 10.6 at N=256, + mul 2x2 18.9 against 14.3 at N=256, mul 1x1 16.5 against 10.5 at N=128, divrem 8x4 91.9 against 80.2 at N=512; M4: add + 4x4 14.1 against 9.3, mul 2x2 18.6 against 13.6, mul 1x1 14.5 against 9.1). +- **What it costs.** `sizeof` grows from 16 to 24, 40 and 72 bytes. Rows whose result does not fit inline are slightly + slower or unchanged: on x64 add 16x16 is 21.0 ns at N=64 and 23.5-24.2 at N=128-512, shl 16x13 17.1 against 18.2-20.5, mul + 16x16 88.5 against 90.4-91.9; on M4 the 16-limb rows are within 1 ns except shl 16x13 (15.5 at N=64, 20.2 at N=512). Some + single-limb rows are slower at larger N (x64 add 1x1 6.7 ns at N=64, 8.3-8.7 at N=128-512; M4 7.5 against 8.3-10.1). The + container workload (vecsort) pays for the bigger element: x64 + 86.1 -> 92.3 -> 98.6 -> 103.4 ns per element (+7%, +14%, +20% against N=64), M4 87.9 -> 94.8 -> 96.5 -> 98.3 ns (+8%, + +10%, +12%); the copy-assign alone goes 1.1 -> 2.5 -> 2.6 -> 3.1 ns on x64 and 2.7 -> 3.9 -> 3.8 -> 4.4 ns on M4. A + `std::int64_t` vector costs 39.4 ns (x64) and 13.2 ns (M4) per element for reference. +- **In-place operations** and divrem at 32x16 do not depend on N within ~1.5 ns. + +## 8. Caveats + +- **Noise modes at <= 16 limbs.** Kernel rows at tiny sizes flip between two stable modes from process to process with the + same data and binary (x64 baseline: `add kernel 16x16` 7.56 / 9.55 / 7.56 ns, `shl kernel 4x13` 3.38 / 4.65 / 3.45, + `shr kernel 16x13` 9.39 / 8.04 / 8.03, `sqr kernel 16x16` 56.2 / 61.2 / 58.5), probably buffer or code alignment. Differences + of 1-2 ns on kernel rows, and therefore on `auto - kernel` and `inplace - kernel` at <= 16 limbs, are not significant; + the `floor` and `inplace` rows are the steadier references. The M4 baseline mul 2x2 `auto` varied 26.6-36.3 ns across its + four runs (the new binary 18.2-19.1). +- **divrem seed dependence.** Different random operands change the division path: divrem 8x4 differs 11-26% between seeds + (kernel 39.7 vs 50.0 ns in the baseline). Small divrem rows are means of seeds 1 and 2; compare before and after only on + the same seeds. +- The shift `kernel` rows include the copy-in of the source, so `auto - kernel` and `auto - floor` for shifts at + >= 256 limbs are negative once `auto` is a single pass. +- The x64 numbers are GCC 14 with `-march=native`; the decimal `fromchars` regression (section 3.7) was seen only there. + The loop-shape choices for add/sub and the shift variants were measured with GCC 14 on x64 and appleclang on M4 only; + clang on x64 and GCC on AArch64 were not measured. +- The 2-run study and the 3-run shortcut comparison are small samples: treat differences below ~1 ns as noise. + +## 9. Correctness matrix + +At d689f75 (ctest counts include skipped benches); at b823c5f, which changes only the shift code, the GCC-only loop path was +re-run: x64 `gcc-release` and `gcc-debug` (g++-14) 1363/1363 each, Docker linux/arm64 `gcc-release` and `gcc-debug` +1292/1292 each, M4 `appleclang-release` 1304/1304: + +- M4 (appleclang): `appleclang-debug` workflow (MaxSan) 1304/1304, `appleclang-release` 1304/1304, + `appleclang-release-namespace` 1290/1290. +- Docker linux/arm64 (GCC 16, `ctest -E float_construction`): `gcc-release` and `gcc-debug` 1292/1292 each. +- x64 box: `llvm-debug` (MaxSan) and `llvm-release` (clang-23, `-march=native`) 1381/1381 each; `gcc-release` with g++-14 + and with g++-13 (`-march=native`), the IFMA=OFF g++-14 build and `gcc-debug` (g++-14, no NDEBUG, so the debug poison fill + and the debug assertions of the new primitives are compiled) 1363/1363 each, with no build failures and no + `-Wno-error` overrides. +- 32-bit limbs (`llvm-release`, `-DBEMAN_BIG_INT_FORCED_LIMB_WIDTH=32`, x64): 36 of 1088 tests fail, all of them failures + that the cf1cf54 tree has as well (bitwise, increment_decrement, karatsuba, pmr and wide_ops do not compile; some + Addition, Subtraction, CompoundSubtraction, BitShift and IntegerAssignment cases assume 64-bit limbs; the baseline has 40 + failures). `inline_tail`, `span_primitives`, `front_end_add_sub`, `front_end_shift`, `muldiv_frontend`, + `dispatch_contract`, `alloc_count` (85 tests) and `allocation` (70 tests) all pass at 32-bit limbs. +- At 9145b4e (the first measurement) the GCC builds failed on `front_end_reference.hpp:44` (`-Werror=padded`) and the + new `front_end_add_sub` and `muldiv_frontend` tests failed at 32-bit limbs through 64-bit assumptions; both were fixed + in the follow-ups. + + +## 10. Reproducing + +``` +# harness binaries (M4 shown; x64: PATH=:$PATH, preset gcc-release, -DCMAKE_CXX_FLAGS=-march=native) +C="-DBEMAN_BIG_INT_BUILD_BENCHMARKS=ON -DBEMAN_BIG_INT_SHAPE_SWEEP_ONLY=ON -DBEMAN_BIG_INT_SWEEP_GMP=ON -DBEMAN_BIG_INT_SIMD_MUL=ON" +cmake --preset appleclang-release -B build/item1/mac-new64 $C -DBEMAN_BIG_INT_SWEEP_INLINE_BITS=64 # also 128, 256, 512 +cmake --build build/item1/mac-new64 --target beman.big_int.benchmarks.shape_sweep +# before binary: the same command on a checkout of cf1cf54 with this harness (floor/vecsort rows, inline-bits option) + +# before/after, interleaved (seeds 1 and 2, two repeats), x64 pinned with --pin "taskset -c 2" +tools/gap_sweep.sh out.csv --seed S --ops add,sub,shl,shr,mul,sqr,divrem,fromchars --bands small,medium +# large spot checks (3 rounds): shape_sweep add --rows auto,inplace,kernel,floor,gmp,gmpz --rounds 3 16384x16384; +# mul 2000x2000 and 16384x16384; divrem 4096x2048; tochars/fromchars 10000x10 +# study: tools/gap_sweep.sh out.csv --seed S --ops add,sub,shl,shr,mul,sqr,divrem --bands small +# tools/gap_sweep.sh out.csv --ops vecsort --bands medium +# raw per-run CSVs (named __seed.csv etc.) are written by tools/item1_bench.sh and by the +# re-measurement and A/B variants of it into any directory; they are not kept in the repository. Tables, the regression +# check and the summaries are produced from such a directory: +# tables: python3 -I tools/item1_tables.py x64r3 (x64, mac, macr3 for the other sets) [small medium spots shortcut study] +# regression check (rows more than 3% slower than base64): python3 -I tools/item1_regress.py x64r3 3 [rows] +# in-place shift A/B tables: python3 -I tools/item1_vshift.py x64 (or: mac) +# the summary CSVs kept in perf/item1: python3 -I tools/item1_summary.py perf/item1 +# re-measurement driver (base64 vs new64 vs previous build): the same loop as tools/item1_bench.sh over +# --ops add,sub,shl,shr,mul,sqr,divrem,fromchars,tochars --bands small,medium +# allocation counts: cmake --build --target beman.big_int.tests.alloc_count; run the binary and read the +# "[alloc_count]" lines +``` + +Kept data files in `perf/item1/` (medians as used by the tables above; ns per operation; the header comments hold the +build configuration): + +| file | content | +|---|---| +| `final_x64.csv`, `final_m4.csv` | op, shape, row, `base_ns` (cf1cf54), `first_9145b4e_ns`, `d689f75_ns`, `after_ns`, `after_commit`; small and medium bands and the large spot checks (sections 3-4) | +| `study.csv` | machine, inline_bits, op, shape, row, ns, including the vecsort rows (section 7) | +| `shortcut.csv` | machine, op, shape, row, `on_ns`, `off_ns` (section 6) | +| `shift_ab.csv` | machine, variant, op, shape, row, ns (section 3.8) | + +The raw-run figures quoted in section 8 (individual-run noise values) come from per-run CSVs that are not kept. + +The shortcut comparison builds a copy of the tree with `mul_header_basecase_enabled = false` in +`include/beman/big_int/detail/mul_impl.hpp` and otherwise identical flags. diff --git a/tests/beman/big_int/perf/shape_sweep.cpp b/tests/beman/big_int/perf/shape_sweep.cpp index 7e830e9f..5703a470 100644 --- a/tests/beman/big_int/perf/shape_sweep.cpp +++ b/tests/beman/big_int/perf/shape_sweep.cpp @@ -14,6 +14,11 @@ // shl shr NxS: N limbs shifted by S bits // mul AxB sqr N or NxN div rem divrem dividend x divisor // tochars NxBASE fromchars NxBASE (the decimal-or-other string of a random N-limb value) +// vecsort NxL N one-limb values (L = 1) in a vector: copy + sort + sum, ns per element (see +// run_vecsort) +// +// Every row that uses the library integer type uses sweep_int = basic_big_int +// (default 64, that is big_int); the #const line prints sweep_inline_bits and sweep_int_sizeof. // // Rows (CSV column 'path'; default 'auto'; rows of one shape run back to back): // auto the public big_int API into a pre-existing destination (c = a + b, ...) @@ -22,6 +27,10 @@ // ladder // (unsliced) and shares its timings with --rows unsliced. Kernels that work in place (shifts, gcd, tochars // and gmp's get_str/gcd/set_str inputs) include the copy-in of their operands in the timed loop. +// floor add sub shl shr mul sqr divrem: the kernel row plus one std::allocator allocate/deallocate pair of the +// result +// size (quotient and remainder for divrem), the API floor of `c = a op b`. No pair is charged for a result +// that fits the inline capacity of the integer type under test. // kernelip shl/shr only: in-place pair shl then shr on one buffer, halved, no copy-in // gmp GMP mpn on preallocated buffers (needs -DBEMAN_BIG_INT_SWEEP_GMP, 64-bit limbs, -lgmp) // gmpz GMP mpz with the destination reused, operands loaded with mpz_import @@ -129,6 +138,13 @@ namespace dt = ::BEMAN_BIG_INT_NAMESPACE::detail; using limb_t = bb::uint_multiprecision_t; using u64 = std::uint64_t; +// The big integer type behind every timed row (auto, inplace, floor, vecsort, ...). The default 64 inline bits is +// bb::big_int; -DBEMAN_BIG_INT_SWEEP_INLINE_BITS=128/256/512 is the inline-capacity study. +#if !defined(BEMAN_BIG_INT_SWEEP_INLINE_BITS) + #define BEMAN_BIG_INT_SWEEP_INLINE_BITS 64 +#endif +using sweep_int = bb::basic_big_int; + constexpr unsigned limb_bits = std::numeric_limits::digits; // Residue primes below 2^31, so every product fits a u64 on any host. @@ -285,6 +301,8 @@ std::string const_line() { kv(s, "SIMD_MUL", 0); #endif kv(s, "limb_bits", limb_bits); + kv(s, "sweep_inline_bits", BEMAN_BIG_INT_SWEEP_INLINE_BITS); + kv(s, "sweep_int_sizeof", sizeof(sweep_int)); #if defined(NDEBUG) kv(s, "ndebug", 1); #else @@ -368,7 +386,7 @@ u64 residue(const std::span x, const u64 p) { return r; } -bb::big_int make_big(const std::vector& v) { return bb::big_int(v.begin(), v.end()); } +sweep_int make_big(const std::vector& v) { return sweep_int(v.begin(), v.end()); } [[noreturn]] void fail(const std::string& msg) { std::fprintf(stderr, "shape_sweep: FAIL: %s\n", msg.c_str()); @@ -506,9 +524,9 @@ void check_division(const char* what, const std::size_t lb, const std::span a, const std::span b, - const bb::big_int& q, - const bb::big_int& r, - const bb::big_int& divisor) { + const sweep_int& q, + const sweep_int& r, + const sweep_int& divisor) { for (const u64 p : {prime_a, prime_b}) { const u64 lhs = (mulmod(residue(q.representation(), p), residue(b, p), p) + residue(r.representation(), p)) % p; @@ -613,6 +631,51 @@ void time_row(const options& opt, print_row(opt, name, la, lb, t); } +// The API floor of `c = a op b`: the kernel plus one std::allocator allocate/deallocate pair of the result size (two +// pairs, held together, for divrem: quotient and remainder). It is what `c = a op b` costs once the front end is free. +// A result that fits the inline capacity of sweep_int needs no heap block, so no pair is charged for it. On = false +// compiles to nothing, so the kernel rows are unchanged. +template +class api_floor { + public: + explicit api_floor([[maybe_unused]] const std::size_t n1, [[maybe_unused]] const std::size_t n2 = 0) { + if constexpr (On) { + std::allocator alloc; + m_n1 = n1 > sweep_int::inplace_capacity ? n1 : 0; + m_n2 = n2 > sweep_int::inplace_capacity ? n2 : 0; + if (m_n1 != 0) { + m_p1 = alloc.allocate(m_n1); + escape(m_p1); + } + if (m_n2 != 0) { + m_p2 = alloc.allocate(m_n2); + escape(m_p2); + } + } + } + ~api_floor() { + if constexpr (On) { + std::allocator alloc; + if (m_n2 != 0) { + escape(m_p2); + alloc.deallocate(m_p2, m_n2); + } + if (m_n1 != 0) { + escape(m_p1); + alloc.deallocate(m_p1, m_n1); + } + } + } + api_floor(const api_floor&) = delete; + api_floor& operator=(const api_floor&) = delete; + + private: + limb_t* m_p1{nullptr}; + limb_t* m_p2{nullptr}; + std::size_t m_n1{0}; + std::size_t m_n2{0}; +}; + #if defined(BEMAN_BIG_INT_SWEEP_GMP) // GMP mpz operand/destination with the limbs readable through the hand-declared struct. struct zint { @@ -650,14 +713,14 @@ void run_mul(const options& opt, const std::size_t la, const std::size_t lb) { for (const auto& row : opt.rows) { if (row == "auto") { - const bb::big_int a = make_big(va); - const bb::big_int b = make_big(vb); + const sweep_int a = make_big(va); + const sweep_int b = make_big(vb); { - const bb::big_int c = a * b; + const sweep_int c = a * b; check_product("mul", la, lb, va, vb, c.representation()); } - bb::big_int c; - auto work = [&] { + sweep_int c; + auto work = [&] { c = a * b; escape(&c); escape(c.representation().data()); @@ -665,17 +728,18 @@ void run_mul(const options& opt, const std::size_t la, const std::size_t lb) { const timing t = measure(work, opt); check_product("mul (last timed product)", la, lb, va, vb, c.representation()); print_row(opt, "auto", la, lb, t); - } else if (row == "sliced" || row == "unsliced" || row == "kernel") { - const std::string path = row == "kernel" ? "unsliced" : row; + } else if (row == "sliced" || row == "unsliced" || row == "kernel" || row == "floor") { + const bool is_kernel = row == "kernel" || row == "floor"; + const std::string path = is_kernel ? "unsliced" : row; if (la < 2 || lb < 2) { - if (row == "kernel") { + if (is_kernel) { skip_row(opt, row, "the tier ladder needs both operands of at least 2 limbs"); continue; } usage_error("--path " + row + " needs both operands of at least 2 limbs"); } #if !defined(BEMAN_BIG_INT_HAS_SHAPE_DISPATCH) - if (row == "kernel") { + if (is_kernel) { skip_row(opt, row, "needs BEMAN_BIG_INT_HAS_SHAPE_DISPATCH"); continue; } @@ -684,25 +748,34 @@ void run_mul(const options& opt, const std::size_t la, const std::size_t lb) { std::allocator alloc; const dt::scratch_allocator> hooks(alloc); std::vector result(la + lb); - std::size_t n = 0; - auto work = [&] { - std::ranges::fill(result, limb_t{0}); - n = fn(result, va, vb, hooks.heap()); - escape(&n); - escape(result.data()); - }; - const auto verify = [&](const char* what) { - check_product(what, la, lb, va, vb, result); - if (n != trimmed_size(result)) { - fail(path + " returned size " + std::to_string(n) + " but the trimmed size is " + - std::to_string(trimmed_size(result)) + " at " + sh); - } + std::size_t n = 0; + const auto go = [&](const auto floor_tag) { + constexpr bool floor_on = decltype(floor_tag)::value; + auto work = [&] { + const api_floor pair(la + lb); + std::ranges::fill(result, limb_t{0}); + n = fn(result, va, vb, hooks.heap()); + escape(&n); + escape(result.data()); + }; + const auto verify = [&](const char* what) { + check_product(what, la, lb, va, vb, result); + if (n != trimmed_size(result)) { + fail(path + " returned size " + std::to_string(n) + " but the trimmed size is " + + std::to_string(trimmed_size(result)) + " at " + sh); + } + }; + work(); + verify(path.c_str()); + const timing t = measure(work, opt); + verify((path + " (last timed product)").c_str()); + print_row(opt, row.c_str(), la, lb, t); }; - work(); - verify(path.c_str()); - const timing t = measure(work, opt); - verify((path + " (last timed product)").c_str()); - print_row(opt, row.c_str(), la, lb, t); + if (row == "floor") { + go(std::true_type{}); + } else { + go(std::false_type{}); + } } else if (row == "fft") { if constexpr (limb_bits != 64) { usage_error("--path fft needs 64-bit limbs"); @@ -775,13 +848,13 @@ void run_sqr(const options& opt, const std::size_t n) { for (const auto& row : opt.rows) { if (row == "auto") { - const bb::big_int a = make_big(va); + const sweep_int a = make_big(va); { - const bb::big_int c = a * a; // same object on both sides selects the squaring path + const sweep_int c = a * a; // same object on both sides selects the squaring path check_product("sqr", n, n, va, va, c.representation()); } - bb::big_int c; - auto work = [&] { + sweep_int c; + auto work = [&] { c = a * a; escape(&c); escape(c.representation().data()); @@ -789,7 +862,7 @@ void run_sqr(const options& opt, const std::size_t n) { const timing t = measure(work, opt); check_product("sqr (last timed product)", n, n, va, va, c.representation()); print_row(opt, "auto", n, n, t); - } else if (row == "kernel") { + } else if (row == "kernel" || row == "floor") { if (n < 2) { skip_row(opt, row, "the squaring ladder needs at least 2 limbs"); continue; @@ -799,23 +872,32 @@ void run_sqr(const options& opt, const std::size_t n) { const dt::scratch_allocator> hooks(alloc); std::vector result(2 * n); std::size_t size = 0; - time_row( - opt, - "kernel", - n, - n, - [&] { - std::ranges::fill(result, limb_t{0}); - size = dt::square_runtime(result, va, hooks.heap()); - escape(&size); - escape(result.data()); - }, - [&] { - check_product("kernel sqr", n, n, va, va, result); - if (size != trimmed_size(result)) { - fail("kernel sqr returned a wrong size at " + shape_str(n, n)); - } - }); + const auto go = [&](const auto floor_tag) { + constexpr bool floor_on = decltype(floor_tag)::value; + time_row( + opt, + floor_on ? "floor" : "kernel", + n, + n, + [&] { + const api_floor pair(2 * n); + std::ranges::fill(result, limb_t{0}); + size = dt::square_runtime(result, va, hooks.heap()); + escape(&size); + escape(result.data()); + }, + [&] { + check_product("kernel sqr", n, n, va, va, result); + if (size != trimmed_size(result)) { + fail("kernel sqr returned a wrong size at " + shape_str(n, n)); + } + }); + }; + if (row == "floor") { + go(std::true_type{}); + } else { + go(std::false_type{}); + } #if defined(BEMAN_BIG_INT_SWEEP_GMP) } else if (row == "gmp") { const auto ga = to_gmp(va); @@ -864,13 +946,13 @@ void run_div(const options& opt, const std::size_t la, const std::size_t lb) { const auto vb = random_limbs(rng, lb); const auto sh = shape_str(la, lb); - const bb::big_int a = make_big(va); - const bb::big_int b = make_big(vb); - limbs ref_q; - limbs ref_r; + const sweep_int a = make_big(va); + const sweep_int b = make_big(vb); + limbs ref_q; + limbs ref_r; { - const bb::big_int q = a / b; - const bb::big_int r = a % b; + const sweep_int q = a / b; + const sweep_int r = a % b; check_division(opt.op.c_str(), la, lb, va, vb, q, r, b); ref_q = trimmed(q.representation()); ref_r = trimmed(r.representation()); @@ -879,9 +961,9 @@ void run_div(const options& opt, const std::size_t la, const std::size_t lb) { for (const auto& row : opt.rows) { if (row == "auto") { - bb::big_int q; - bb::big_int r; - auto work = [&] { + sweep_int q; + sweep_int r; + auto work = [&] { if (opt.op == "div") { q = a / b; escape(&q); @@ -907,49 +989,62 @@ void run_div(const options& opt, const std::size_t la, const std::size_t lb) { } check_division((opt.op + " (last timed result)").c_str(), la, lb, va, vb, q, r, b); print_row(opt, "auto", la, lb, t); - } else if (row == "kernel") { + } else if (row == "kernel" || row == "floor") { if (!is_divrem) { skip_row(opt, row, "kernel rows time quotient and remainder together, use divrem"); - } else if (lb == 1) { - limbs q(la); - limb_t rem = 0; - time_row( - opt, - "kernel", - la, - lb, - [&] { - rem = dt::divide_unsigned_short(q, va, vb[0]); - escape(&rem); - escape(q.data()); - }, - [&] { - expect_same("kernel divrem quotient", sh, q, ref_q); - expect_same("kernel divrem remainder", sh, std::span{&rem, 1}, ref_r); - }); + continue; + } + const auto go = [&](const auto floor_tag) { + constexpr bool floor_on = decltype(floor_tag)::value; + const char* name = floor_on ? "floor" : "kernel"; + if (lb == 1) { + limbs q(la); + limb_t rem = 0; + time_row( + opt, + name, + la, + lb, + [&] { + const api_floor pair(la, lb); + rem = dt::divide_unsigned_short(q, va, vb[0]); + escape(&rem); + escape(q.data()); + }, + [&] { + expect_same("kernel divrem quotient", sh, q, ref_q); + expect_same("kernel divrem remainder", sh, std::span{&rem, 1}, ref_r); + }); + } else { + const std::size_t q_len = la - lb + 1; + const std::size_t r_cap = la + 1; + std::allocator alloc; + dt::scratch_allocator> scratch( + r_cap + dt::divide_unsigned_storage_size(la, lb), alloc); + const std::span rem = scratch.allocate(r_cap); + limbs q(q_len); + time_row( + opt, + name, + la, + lb, + [&] { + const api_floor pair(q_len, lb); + std::ranges::fill(q, limb_t{0}); + dt::divide_dispatch(q, rem, va, vb, scratch, alloc); + escape(q.data()); + escape(rem.data()); + }, + [&] { + expect_same("kernel divrem quotient", sh, q, ref_q); + expect_same("kernel divrem remainder", sh, rem, ref_r); + }); + } + }; + if (row == "floor") { + go(std::true_type{}); } else { - const std::size_t q_len = la - lb + 1; - const std::size_t r_cap = la + 1; - std::allocator alloc; - dt::scratch_allocator> scratch(r_cap + dt::divide_unsigned_storage_size(la, lb), - alloc); - const std::span rem = scratch.allocate(r_cap); - limbs q(q_len); - time_row( - opt, - "kernel", - la, - lb, - [&] { - std::ranges::fill(q, limb_t{0}); - dt::divide_dispatch(q, rem, va, vb, scratch, alloc); - escape(q.data()); - escape(rem.data()); - }, - [&] { - expect_same("kernel divrem quotient", sh, q, ref_q); - expect_same("kernel divrem remainder", sh, rem, ref_r); - }); + go(std::false_type{}); } #if defined(BEMAN_BIG_INT_SWEEP_GMP) } else if (row == "gmp") { @@ -996,8 +1091,8 @@ void run_div(const options& opt, const std::size_t la, const std::size_t lb) { escape(gr.data()); }, [&] { - const bb::big_int gq_big(gq.begin(), gq.end()); - const bb::big_int gr_big(gr.begin(), gr.end()); + const sweep_int gq_big(gq.begin(), gq.end()); + const sweep_int gr_big(gr.begin(), gr.end()); check_division("gmp divrem", la, lb, va, vb, gq_big, gr_big, b); expect_same("gmp divrem quotient", sh, from_gmp(gq, gq.size()), ref_q); expect_same("gmp divrem remainder", sh, from_gmp(gr, gr.size()), ref_r); @@ -1067,11 +1162,11 @@ void run_addsub(const options& opt, const std::size_t la, const std::size_t lb) } const auto sh = shape_str(la, lb); - const bb::big_int a = make_big(va); - const bb::big_int b = make_big(vb); - limbs ref; + const sweep_int a = make_big(va); + const sweep_int b = make_big(vb); + limbs ref; { - const bb::big_int c = Add ? a + b : a - b; + const sweep_int c = Add ? a + b : a - b; check_addsub("api", sh, va, vb, trimmed(c.representation())); ref = trimmed(c.representation()); } @@ -1079,7 +1174,7 @@ void run_addsub(const options& opt, const std::size_t la, const std::size_t lb) for (const auto& row : opt.rows) { if (row == "auto") { - bb::big_int c; + sweep_int c; time_row( opt, "auto", @@ -1098,7 +1193,7 @@ void run_addsub(const options& opt, const std::size_t la, const std::size_t lb) } else if (row == "inplace") { // One timed unit is two operations (c += b; c -= b, or the reverse for sub); per-op time is half. { - bb::big_int t = a; + sweep_int t = a; if constexpr (Add) { t += b; } else { @@ -1106,7 +1201,7 @@ void run_addsub(const options& opt, const std::size_t la, const std::size_t lb) } expect_same("inplace first op", sh, t.representation(), ref); } - bb::big_int c = a; + sweep_int c = a; time_row( opt, "inplace", @@ -1125,35 +1220,44 @@ void run_addsub(const options& opt, const std::size_t la, const std::size_t lb) }, [&] { expect_same("inplace pair", sh, c.representation(), va_ref); }, 0.5); - } else if (row == "kernel") { + } else if (row == "kernel" || row == "floor") { limbs res(la); bool carry = false; std::size_t size = 0; - time_row( - opt, - "kernel", - la, - lb, - [&] { - if constexpr (Add) { - carry = dt::add_unsigned_spans(res, va, vb); - escape(&carry); - } else { - size = dt::subtract_unsigned_spans(res, va, vb); - escape(&size); - } - escape(res.data()); - }, - [&] { - limbs full = res; - if (Add && carry) { - full.push_back(1); - } - expect_same("kernel", sh, full, ref); - if (!Add && size != trimmed_size(res)) { - fail("kernel sub returned a wrong size at " + sh); - } - }); + const auto go = [&](const auto floor_tag) { + constexpr bool floor_on = decltype(floor_tag)::value; + time_row( + opt, + floor_on ? "floor" : "kernel", + la, + lb, + [&] { + const api_floor pair(Add ? la + 1 : la); + if constexpr (Add) { + carry = dt::add_unsigned_spans(res, va, vb); + escape(&carry); + } else { + size = dt::subtract_unsigned_spans(res, va, vb); + escape(&size); + } + escape(res.data()); + }, + [&] { + limbs full = res; + if (Add && carry) { + full.push_back(1); + } + expect_same("kernel", sh, full, ref); + if (!Add && size != trimmed_size(res)) { + fail("kernel sub returned a wrong size at " + sh); + } + }); + }; + if (row == "floor") { + go(std::true_type{}); + } else { + go(std::false_type{}); + } #if defined(BEMAN_BIG_INT_SWEEP_GMP) } else if (row == "gmp") { const auto ga = to_gmp(va); @@ -1228,10 +1332,10 @@ void run_cmp(const options& opt, const std::size_t la, const std::size_t lb) { const int want = va[0] < vb[0] ? -1 : 1; const auto sh = shape_str(la, lb); - const bb::big_int a = make_big(va); - const bb::big_int b = make_big(vb); - int s = 0; - const auto check = [&](const char* what) { + const sweep_int a = make_big(va); + const sweep_int b = make_big(vb); + int s = 0; + const auto check = [&](const char* what) { if (s != want) { fail(std::string(what) + " cmp sign mismatch at " + sh); } @@ -1346,10 +1450,10 @@ void run_shift(const options& opt, const std::size_t n, const std::size_t bits) const unsigned partial = static_cast(bits % limb_bits); const int s = static_cast(bits); - const bb::big_int a = make_big(va); - limbs ref; + const sweep_int a = make_big(va); + limbs ref; { - const bb::big_int c = Left ? a << s : a >> s; + const sweep_int c = Left ? a << s : a >> s; check_shift("api", sh, va, bits, trimmed(c.representation())); ref = trimmed(c.representation()); } @@ -1357,7 +1461,7 @@ void run_shift(const options& opt, const std::size_t n, const std::size_t bits) for (const auto& row : opt.rows) { if (row == "auto") { - bb::big_int c; + sweep_int c; time_row( opt, "auto", @@ -1377,7 +1481,7 @@ void run_shift(const options& opt, const std::size_t n, const std::size_t bits) // One timed unit is two operations, c <<= s; c >>= s for shl and c >>= s; c <<= s for shr (the latter // settles on a with its low S bits cleared); per-op time is half. { - bb::big_int t = a; + sweep_int t = a; if constexpr (Left) { t <<= s; } else { @@ -1385,7 +1489,7 @@ void run_shift(const options& opt, const std::size_t n, const std::size_t bits) } expect_same("inplace first op", sh, t.representation(), ref); } - bb::big_int c = a; + sweep_int c = a; time_row( opt, "inplace", @@ -1406,49 +1510,60 @@ void run_shift(const options& opt, const std::size_t n, const std::size_t bits) if constexpr (Left) { expect_same("inplace pair", sh, c.representation(), va_ref); } else { - bb::big_int t = c; + sweep_int t = c; t >>= s; expect_same("inplace pair", sh, t.representation(), ref); } }, 0.5); - } else if (row == "kernel") { + } else if (row == "kernel" || row == "floor") { // The span kernels work in place, so the copy-in of the source is part of the timed loop. - if constexpr (Left) { - limbs buf(n + whole + 1); - std::size_t size = 0; - time_row( - opt, - "kernel", - n, - bits, - [&] { - std::ranges::copy(va, buf.begin()); - size = dt::shift_left_bits(buf, n, bits); - escape(&size); - escape(buf.data()); - }, - [&] { expect_same("kernel", sh, std::span{buf.data(), size}, ref); }); + const auto go = [&](const auto floor_tag) { + constexpr bool floor_on = decltype(floor_tag)::value; + const char* name = floor_on ? "floor" : "kernel"; + if constexpr (Left) { + limbs buf(n + whole + 1); + std::size_t size = 0; + time_row( + opt, + name, + n, + bits, + [&] { + const api_floor pair(n + whole + (partial != 0 ? 1 : 0)); + std::ranges::copy(va, buf.begin()); + size = dt::shift_left_bits(buf, n, bits); + escape(&size); + escape(buf.data()); + }, + [&] { expect_same("kernel", sh, std::span{buf.data(), size}, ref); }); + } else { + // The same two steps as shift_right_bits (move down whole limbs, shift_right_n the rest), without + // its debug assertion that every dropped bit is zero. + const std::size_t m = n - whole; + limbs buf(m); + limb_t dropped = 0; + time_row( + opt, + name, + n, + bits, + [&] { + const api_floor pair(m); + std::copy(va.begin() + static_cast(whole), va.end(), buf.begin()); + if (partial != 0) { + dropped = dt::shift_right_n(buf, partial); + escape(&dropped); + } + escape(buf.data()); + }, + [&] { expect_same("kernel", sh, buf, ref); }); + } + }; + if (row == "floor") { + go(std::true_type{}); } else { - // The same two steps as shift_right_bits (move down whole limbs, shift_right_n the rest), without its - // debug assertion that every dropped bit is zero. - const std::size_t m = n - whole; - limbs buf(m); - limb_t dropped = 0; - time_row( - opt, - "kernel", - n, - bits, - [&] { - std::copy(va.begin() + static_cast(whole), va.end(), buf.begin()); - if (partial != 0) { - dropped = dt::shift_right_n(buf, partial); - escape(&dropped); - } - escape(buf.data()); - }, - [&] { expect_same("kernel", sh, buf, ref); }); + go(std::false_type{}); } } else if (row == "kernelip") { if constexpr (!Left) { @@ -1563,16 +1678,16 @@ void run_tochars(const options& opt, const std::size_t n, const std::size_t base const auto sh = shape_str(n, base_arg); const auto cap = digit_capacity(n, base); - const bb::big_int a = make_big(va); - std::string ref(cap, '\0'); + const sweep_int a = make_big(va); + std::string ref(cap, '\0'); { const auto r = bb::to_chars(ref.data(), ref.data() + ref.size(), a, base); if (r.ec != std::errc{}) { fail("to_chars failed at " + sh); } ref.resize(static_cast(r.ptr - ref.data())); - bb::big_int back; - const auto fr = bb::from_chars(ref.data(), ref.data() + ref.size(), back, base); + sweep_int back; + const auto fr = bb::from_chars(ref.data(), ref.data() + ref.size(), back, base); if (fr.ec != std::errc{}) { fail("from_chars failed on the to_chars output at " + sh); } @@ -1670,8 +1785,8 @@ void run_fromchars(const options& opt, const std::size_t n, const std::size_t ba const auto sh = shape_str(n, base_arg); const limbs ref = trimmed(va); - const bb::big_int a = make_big(va); - std::string str(digit_capacity(n, base), '\0'); + const sweep_int a = make_big(va); + std::string str(digit_capacity(n, base), '\0'); { const auto r = bb::to_chars(str.data(), str.data() + str.size(), a, base); if (r.ec != std::errc{}) { @@ -1686,7 +1801,7 @@ void run_fromchars(const options& opt, const std::size_t n, const std::size_t ba for (const auto& row : opt.rows) { if (row == "auto") { - bb::big_int c; + sweep_int c; std::from_chars_result r{}; time_row( opt, @@ -1789,11 +1904,11 @@ void run_gcd(const options& opt, const std::size_t la, const std::size_t lb) { vb[0] |= 1; const auto sh = shape_str(la, lb); - const bb::big_int a = make_big(va); - const bb::big_int b = make_big(vb); - limbs ref; + const sweep_int a = make_big(va); + const sweep_int b = make_big(vb); + limbs ref; { - const bb::big_int g = bb::gcd(a, b); + const sweep_int g = bb::gcd(a, b); if (g <= 0 || !((a % g) == 0) || !((b % g) == 0)) { fail("gcd result does not divide both operands at " + sh); } @@ -1802,7 +1917,7 @@ void run_gcd(const options& opt, const std::size_t la, const std::size_t lb) { for (const auto& row : opt.rows) { if (row == "auto") { - bb::big_int c; + sweep_int c; time_row( opt, "auto", @@ -1875,6 +1990,127 @@ void run_gcd(const options& opt, const std::size_t la, const std::size_t lb) { } } +// --------------------------------------------------------------------------- +// vecsort: the container cost of the inline capacity. Shape NxL: a vector of N random one-limb values (L must be 1) of +// mixed sign and magnitude below 2^63. One timed unit is copy-assigning the unsorted source into the work vector, +// std::sort, and summing; the reported time is per element (ns), so the rows are comparable across inline sizes. +// auto std::vector copy + sort + sum (the sum is a sweep_int, += per element) +// builtin std::vector the same with a wrapping uint64 sum, the reference +// copy std::vector the copy-assign alone, to subtract from auto +// --------------------------------------------------------------------------- + +void run_vecsort(const options& opt, const std::size_t n, const std::size_t value_limbs) { + if (value_limbs != 1) { + usage_error("vecsort needs NxL with L = 1 (one-limb values), got " + shape_str(n, value_limbs)); + } + std::mt19937_64 rng(splitmix(opt.seed ^ splitmix(n * 1000003 + 1))); + std::vector raw(n); + for (auto& x : raw) { + const auto mag = static_cast(rng() >> 1); + x = (rng() & 1) != 0 ? -mag : mag; + } + std::vector sorted = raw; + std::ranges::sort(sorted); + std::vector src_big(raw.begin(), raw.end()); + const auto sh = shape_str(n, 1); + const double scale = 1.0 / static_cast(n); + +#ifdef BEMAN_BIG_INT_HAS_INT128_EXTENSION + __int128 ref_sum = 0; + for (const auto x : raw) { + ref_sum += x; + } + const sweep_int want_total = (sweep_int(static_cast(ref_sum >> 64)) << 64) + + sweep_int(static_cast(static_cast(ref_sum))); + u64 want_wrap = static_cast(static_cast(ref_sum)); +#else + u64 want_wrap = 0; + for (const auto x : raw) { + want_wrap += static_cast(x); + } +#endif + + for (const auto& row : opt.rows) { + if (row == "auto") { + std::vector work = src_big; + sweep_int total; + time_row( + opt, + "auto", + n, + 1, + [&] { + work = src_big; + std::sort(work.begin(), work.end()); + total = 0; + for (const auto& v : work) { + total += v; + } + escape(&total); + escape(work.data()); + }, + [&] { + for (std::size_t i = 0; i < n; ++i) { + if (!(work[i] == sorted[i])) { + fail("vecsort auto: element " + std::to_string(i) + " out of order at " + sh); + } + } +#ifdef BEMAN_BIG_INT_HAS_INT128_EXTENSION + if (!(total == want_total)) { + fail("vecsort auto: wrong sum at " + sh); + } +#endif + }, + scale); + } else if (row == "builtin") { + std::vector work = raw; + u64 total = 0; + time_row( + opt, + "builtin", + n, + 1, + [&] { + work = raw; + std::sort(work.begin(), work.end()); + total = 0; + for (const auto v : work) { + total += static_cast(v); + } + escape(&total); + escape(work.data()); + }, + [&] { + if (work != sorted || total != want_wrap) { + fail("vecsort builtin: wrong result at " + sh); + } + }, + scale); + } else if (row == "copy") { + std::vector work = src_big; + time_row( + opt, + "copy", + n, + 1, + [&] { + work = src_big; + escape(work.data()); + }, + [&] { + for (std::size_t i = 0; i < n; ++i) { + if (!(work[i] == raw[i])) { + fail("vecsort copy: element " + std::to_string(i) + " differs at " + sh); + } + } + }, + scale); + } else { + skip_row(opt, row, "not applicable"); + } + } +} + // --------------------------------------------------------------------------- // Command line. // --------------------------------------------------------------------------- @@ -1948,8 +2184,9 @@ std::vector split_commas(const std::string& s) { } const char* const all_ops[] = { - "mul", "sqr", "div", "rem", "divrem", "add", "sub", "shl", "shr", "cmp", "tochars", "fromchars", "gcd"}; -const char* const all_rows[] = {"auto", "inplace", "kernel", "kernelip", "gmp", "gmpz", "sliced", "unsliced", "fft"}; + "mul", "sqr", "div", "rem", "divrem", "add", "sub", "shl", "shr", "cmp", "tochars", "fromchars", "gcd", "vecsort"}; +const char* const all_rows[] = { + "auto", "inplace", "kernel", "kernelip", "floor", "gmp", "gmpz", "sliced", "unsliced", "fft", "builtin", "copy"}; bool one_of(const std::string& s, const auto& list) { return std::ranges::find(list, s) != std::end(list); } @@ -1991,8 +2228,9 @@ options parse_args(const int argc, char** argv) { } } if (!one_of(opt.op, all_ops)) { - usage_error("usage: shape_sweep " - "[--rows auto,inplace,kernel,kernelip,gmp,gmpz,sliced,unsliced,fft] [--path ROW] [--rounds N] " + usage_error("usage: shape_sweep " + "[--rows auto,inplace,kernel,kernelip,floor,gmp,gmpz,sliced,unsliced,fft,builtin,copy] " + "[--path ROW] [--rounds N] " "[--round-ms X] [--seed S] [--gmp] [--grid FILE] [SHAPE ...]"); } if (opt.rounds == 0 || opt.round_ms <= 0.0) { @@ -2080,6 +2318,8 @@ int main(int argc, char** argv) { run_fromchars(opt, la, lb); } else if (op == "gcd") { run_gcd(opt, la, lb); + } else if (op == "vecsort") { + run_vecsort(opt, la, lb); } else { run_div(opt, la, lb); } diff --git a/tests/beman/big_int/perf/tools/README.md b/tests/beman/big_int/perf/tools/README.md index ceab5ce8..f64cc768 100644 --- a/tests/beman/big_int/perf/tools/README.md +++ b/tests/beman/big_int/perf/tools/README.md @@ -7,16 +7,18 @@ SPDX-License-Identifier: BSL-1.0 Scripts that produced the data in [`../gmp_gap_analysis.md`](../gmp_gap_analysis.md). The harness itself is `../shape_sweep.cpp` (CMake target `beman.big_int.benchmarks.shape_sweep`, built with -`-DBEMAN_BIG_INT_BUILD_BENCHMARKS=ON -DBEMAN_BIG_INT_SHAPE_SWEEP_ONLY=ON -DBEMAN_BIG_INT_SWEEP_GMP=ON`). Nothing here is part +`-DBEMAN_BIG_INT_BUILD_BENCHMARKS=ON -DBEMAN_BIG_INT_SHAPE_SWEEP_ONLY=ON -DBEMAN_BIG_INT_SWEEP_GMP=ON`; `-DBEMAN_BIG_INT_SWEEP_INLINE_BITS=64|128|256|512` sets the inline capacity of the integer type behind every timed row, default 64 = `big_int`). Nothing here is part of the CMake build or the test suite. | Tool | What it does and how it is invoked | |---|---| -| `gap_sweep.sh` | `gap_sweep.sh [--seed S] [--ops a,b] [--bands small,medium,large] [--pin "taskset -c 2"]`: runs the report's size grid with every applicable row; writes one CSV (`#const` line once, then `op,path,la,lb,reps,rounds,median_ns,min_ns,round_ms_actual`). About 5 minutes per full run. | +| `gap_sweep.sh` | `gap_sweep.sh [--seed S] [--ops a,b] [--bands small,medium,large] [--pin "taskset -c 2"]`: runs the report's size grid with every applicable row (add, sub, shl, shr, mul, sqr and divrem also run the `floor` row: the kernel plus one allocate/deallocate pair of the result size, the API floor of `c = a op b`; `--ops vecsort` adds the inline-capacity container row, default off); writes one CSV (`#const` line once, then `op,path,la,lb,reps,rounds,median_ns,min_ns,round_ms_actual`). About 5 minutes per full run. | | `selftest.sh` | `selftest.sh `: every op and row at small shapes; exit status 0 means all correctness checks passed (timings meaningless). | -| `summarize.py` | `summarize.py a.csv [b.csv] [probe.csv] [--ghz G] > summary.md`: per op and band ratio tables (auto/gmpz, kernel/gmp, inplace/gmpz, auto-kernel ns), top gaps, probe table; with two sweep CSVs also the seed spread. Run with `python3 -I`. | +| `summarize.py` | `summarize.py a.csv [b.csv] [probe.csv] [--ghz G] > summary.md`: per op and band ratio tables (auto/gmpz, kernel/gmp, inplace/gmpz, auto-kernel ns), top gaps, probe table; with two sweep CSVs also the seed spread. Tables carry the floor, `auto - floor` and `inplace - kernel` ns columns when those rows exist. Run with `python3 -I`. | | `compare.py` | `compare.py [heat\|tables]`: the report's cross-configuration tables (comma-separated CSV lists average seeds). | | `probe_tables.py` | `probe_tables.py [n] [opt]`: the report's probe table (cycles/limb per compiler, M4 ns/limb x 4.47). | +| `item1_bench.sh` / `item1_tables.py` | The item 1 before/after driver (`item1_bench.sh [pin]`, needs binaries `shape_sweep.{base64,new64,new128,new256,new512,nosc64}`) and the tables for [`../item1_frontend_results.md`](../item1_frontend_results.md) (`python3 -I item1_tables.py [small medium spots shortcut study]`). | +| `item1_regress.py` / `item1_vshift.py` / `item1_summary.py` | Regression check (rows more than 3% slower than `base64`), the in-place shift A/B tables, and the aggregation of the raw per-run CSVs into the summary CSVs kept in `../item1/` (`final_x64.csv`, `final_m4.csv`, `study.csv`, `shortcut.csv`, `shift_ab.csv`). The raw per-run CSVs are not kept in the repository. | | `bucket_tables.py` | `bucket_tables.py `: compact profile table plus the "time saved if linear helpers ran at GMP speed" estimate. | | `prof.sh` | `prof.sh [filter]` (Linux x86-64, `perf`, `kernel.perf_event_paranoid<=1`): LBR profiles and flame graphs for the report's 12 shapes. Env: `FLAMEGRAPH_DIR` (default `$HOME/FlameGraph`), `PIN`, `MINW`, `LABEL`, `SEED`. | | `classify.py` | `classify.py > buckets.md`: buckets the folded stacks from `prof.sh` by the frame nearest the leaf and scales by the median. | diff --git a/tests/beman/big_int/perf/tools/__pycache__/item1_tables.cpython-314.pyc b/tests/beman/big_int/perf/tools/__pycache__/item1_tables.cpython-314.pyc new file mode 100644 index 00000000..10e921b0 Binary files /dev/null and b/tests/beman/big_int/perf/tools/__pycache__/item1_tables.cpython-314.pyc differ diff --git a/tests/beman/big_int/perf/tools/gap_sweep.sh b/tests/beman/big_int/perf/tools/gap_sweep.sh index 6a18ccff..9c93fc7e 100755 --- a/tests/beman/big_int/perf/tools/gap_sweep.sh +++ b/tests/beman/big_int/perf/tools/gap_sweep.sh @@ -9,7 +9,9 @@ # uses --rounds 3, gcd 65536 uses --rounds 1. Output: one CSV (op,path,la,lb,reps,rounds,median_ns,min_ns, # round_ms_actual) with the '#const' line kept once at the top, after '# gap_sweep' comment lines. # Works on macOS bash 3.2 and Ubuntu bash. --bands is for partial runs (default all three). -# Ops: add sub shl shr cmp mul sqr divrem tochars fromchars gcd (default all). +# Ops: add sub shl shr cmp mul sqr divrem tochars fromchars gcd (default all), and vecsort (container cost of the inline +# capacity, only when named in --ops). The floor row (kernel plus one allocate/deallocate pair of the result size) is +# run for add sub shl shr mul sqr divrem. set -eu if [ "$#" -lt 2 ]; then @@ -80,32 +82,32 @@ sweep() { } if want add; then - sweep add auto,inplace,kernel,gmp,gmpz "1x1 2x2 4x4 8x8 16x16" "64x64 256x256 1024x1024 2000x2000 1024x1" \ + sweep add auto,inplace,kernel,floor,gmp,gmpz "1x1 2x2 4x4 8x8 16x16" "64x64 256x256 1024x1024 2000x2000 1024x1" \ "16384x16384 131072x131072 1048576x1048576" fi if want sub; then - sweep sub auto,inplace,kernel,gmp,gmpz "1x1 2x2 4x4 8x8 16x16" "64x64 256x256 1024x1024 2000x2000 1024x1" \ + sweep sub auto,inplace,kernel,floor,gmp,gmpz "1x1 2x2 4x4 8x8 16x16" "64x64 256x256 1024x1024 2000x2000 1024x1" \ "16384x16384 131072x131072 1048576x1048576" fi if want shl; then - sweep shl auto,inplace,kernel,kernelip,gmp,gmpz "1x13 4x13 16x13" "256x13 2000x13 2000x77" "131072x13 1048576x13" + sweep shl auto,inplace,kernel,kernelip,floor,gmp,gmpz "1x13 4x13 16x13" "256x13 2000x13 2000x77" "131072x13 1048576x13" fi if want shr; then - sweep shr auto,inplace,kernel,gmp,gmpz "1x13 4x13 16x13" "256x13 2000x13 2000x77" "131072x13 1048576x13" + sweep shr auto,inplace,kernel,floor,gmp,gmpz "1x13 4x13 16x13" "256x13 2000x13 2000x77" "131072x13 1048576x13" fi if want cmp; then sweep cmp auto,kernel,gmp,gmpz "1 4 16" "256 2000" "131072" fi if want mul; then - sweep mul auto,kernel,gmp,gmpz "1x1 2x2 3x3 4x4 6x6 8x8 12x12 16x16" \ + sweep mul auto,kernel,floor,gmp,gmpz "1x1 2x2 3x3 4x4 6x6 8x8 12x12 16x16" \ "32x32 64x64 128x128 256x256 512x512 1000x1000 2000x2000 2000x100" \ "4096x4096 16384x16384 65536x65536 262144x262144 1048576x1048576 262144x8192" fi if want sqr; then - sweep sqr auto,kernel,gmp,gmpz "4 16" "64 512 2000" "16384 262144 1048576" + sweep sqr auto,kernel,floor,gmp,gmpz "4 16" "64 512 2000" "16384 262144 1048576" fi if want divrem; then - sweep divrem auto,kernel,gmp,gmpz "2x1 8x4 32x16" "128x64 512x256 2000x1000 4000x2000 2000x100 65536x1" \ + sweep divrem auto,kernel,floor,gmp,gmpz "2x1 8x4 32x16" "128x64 512x256 2000x1000 4000x2000 2000x100 65536x1" \ "16384x8192 131072x65536 1048576x524288" fi if want tochars; then @@ -120,4 +122,8 @@ if want gcd; then sweep gcd auto,kernel,gmp,gmpz "2x2 4x4 16x16" "64x64 256x256 1024x1024 2000x2000" "16384x16384" if band large; then run gcd auto,kernel,gmp,gmpz 1 65536x65536; fi fi +if want vecsort; then + # Not in the default op list: the shape is N values of one limb, medium band; rows are per element. + if band medium; then run vecsort auto,builtin,copy 9 100000x1; fi +fi echo "[gap_sweep] $(date +%H:%M:%S) done: $OUT" >&2 diff --git a/tests/beman/big_int/perf/tools/item1_bench.sh b/tests/beman/big_int/perf/tools/item1_bench.sh new file mode 100755 index 00000000..fb6ecd13 --- /dev/null +++ b/tests/beman/big_int/perf/tools/item1_bench.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# SPDX-License-Identifier: BSL-1.0 +# bench_all.sh [pin command] +# Runs the item 1 before/after sweeps, spot checks, shortcut A/B and inline-capacity study. Never run while building. +BIN=$1; OUT=$2; TOOLS=$3; P=$4; PIN=${5:-} +mkdir -p $OUT +GS=$TOOLS/gap_sweep.sh +OPS=add,sub,shl,shr,mul,sqr,divrem,fromchars +log() { echo "[bench] $(date +%H:%M:%S) $*" >&2; } + +log selftest +for b in new64 new128 new256 new512 nosc64; do + if [ -n "$PIN" ]; then :; fi + $TOOLS/selftest.sh $BIN/shape_sweep.$b 2>&1 | grep -vc "^selftest: ok" | sed "s/^/selftest-failures $b: /" +done + +# B. before/after, interleaved +for seed in 1 2; do for rep in a b; do + for v in base64 new64; do + log "sweep $v seed$seed$rep" + $GS $BIN/shape_sweep.$v $OUT/${P}_${v}_seed${seed}${rep}.csv --seed $seed --ops $OPS --bands small,medium ${PIN:+--pin "$PIN"} 2>/dev/null + done +done; done + +# C. spot checks +spot() { # ; appends + bin=$1; csv=$2; seed=$3 + rm -f $csv; first=1 + one() { op=$1; rows=$2; shape=$3 + $PIN $bin $op --rows $rows --rounds 3 --seed $seed $shape > $OUT/.spot.tmp || echo "spot failed $op $shape" >&2 + if [ $first = 1 ]; then grep -m1 '^#const' $OUT/.spot.tmp >> $csv; grep -m1 '^op,' $OUT/.spot.tmp >> $csv; first=0; fi + grep -v '^#' $OUT/.spot.tmp | grep -v '^op,' >> $csv + } + one add auto,inplace,kernel,floor,gmp,gmpz 16384x16384 + one mul auto,kernel,floor,gmp,gmpz 2000x2000 + one mul auto,kernel,floor,gmp,gmpz 16384x16384 + one divrem auto,kernel,floor,gmp,gmpz 4096x2048 + one tochars auto,kernel,gmp,gmpz 10000x10 + one fromchars auto,kernel,gmp,gmpz 10000x10 +} +for seed in 1 2; do + for v in base64 new64; do log "spot $v seed$seed"; spot $BIN/shape_sweep.$v $OUT/${P}_${v}_spot_seed${seed}.csv $seed; done +done + +# D. basecase shortcut A/B: new64 (shortcut on) vs nosc64 (off), interleaved +for rep in 1 2 3; do + for v in new64 nosc64; do + log "shortcut $v run$rep" + f=$OUT/${P}_${v}_shortcut_run${rep}.csv; rm -f $f + { $PIN $BIN/shape_sweep.$v mul --rows auto,kernel --rounds 9 --seed 1 2x2 3x3 4x4 6x6 8x8 12x12 16x16 + $PIN $BIN/shape_sweep.$v sqr --rows auto,kernel --rounds 9 --seed 1 4 16; } | grep -v '^# \|^op,' > $f.tmp + { echo "# shortcut A/B $v run$rep"; echo "op,path,la,lb,reps,rounds,median_ns,min_ns,round_ms_actual"; grep -v '^#const' $f.tmp; } > $f; rm -f $f.tmp + done +done + +# E. inline-capacity study (small band on the new code) and vecsort, interleaved over N +for rep in 1 2; do + for b in 64 128 256 512; do + log "study N=$b run$rep" + $GS $BIN/shape_sweep.new$b $OUT/${P}_study${b}_run${rep}.csv --seed $rep --ops add,sub,shl,shr,mul,sqr,divrem --bands small ${PIN:+--pin "$PIN"} 2>/dev/null + $GS $BIN/shape_sweep.new$b $OUT/${P}_study${b}_vecsort_run${rep}.csv --seed $rep --ops vecsort --bands medium ${PIN:+--pin "$PIN"} 2>/dev/null + done +done +rm -f $OUT/.spot.tmp +echo BENCH-ALLDONE >&2 diff --git a/tests/beman/big_int/perf/tools/item1_regress.py b/tests/beman/big_int/perf/tools/item1_regress.py new file mode 100644 index 00000000..4771fb0f --- /dev/null +++ b/tests/beman/big_int/perf/tools/item1_regress.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# SPDX-License-Identifier: BSL-1.0 +# item1_regress.py [threshold_percent] [rows] +# Lists every (op, shape, row) where the new64 binary is more than the threshold (default 3%) slower than base64, +# using the same aggregation as item1_tables.py (median of runs; divrem up to 32 limbs: mean of seeds). Rows default to +# auto,inplace. Run with python3 -I. +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import item1_tables as t # noqa: E402 + + +def main(): + d, prefix = sys.argv[1], sys.argv[2] + thr = float(sys.argv[3]) if len(sys.argv) > 3 else 3.0 + rows = sys.argv[4].split(",") if len(sys.argv) > 4 else ["auto", "inplace"] + base = t.sweep_runs(d, prefix, "base64") + new = t.sweep_runs(d, prefix, "new64") + worse, total = [], 0 + for key in sorted(base): + if key[3] not in rows: + continue + b, n = t.agg(base, key), t.agg(new, key) + if b is None or n is None: + continue + total += 1 + pct = (n - b) / b * 100 + if pct > thr: + worse.append((pct, key, b, n)) + print(f"{len(worse)} of {total} rows ({','.join(rows)}) are more than {thr}% slower than base64") + for pct, key, b, n in sorted(worse, reverse=True): + print(f" {key[0]} {key[1]}x{key[2]} {key[3]}: {b:.1f} -> {n:.1f} ns ({pct:+.1f}%, {n - b:+.1f} ns)") + + +if __name__ == "__main__": + main() diff --git a/tests/beman/big_int/perf/tools/item1_summary.py b/tests/beman/big_int/perf/tools/item1_summary.py new file mode 100644 index 00000000..9d082dbf --- /dev/null +++ b/tests/beman/big_int/perf/tools/item1_summary.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# SPDX-License-Identifier: BSL-1.0 +# item1_summary.py +# Aggregates the raw per-run CSVs written by item1_bench.sh (and the re-measurement and A/B variants of it) into the +# summary CSVs kept under perf/item1: final_x64.csv, final_m4.csv, study.csv, shortcut.csv, shift_ab.csv. Aggregation is +# the one item1_tables.py uses for the tables of item1_frontend_results.md (median of runs; divrem up to 32 limbs: mean +# of seeds). The raw inputs are named __seed.csv (sweeps), __spot_seed.csv, +# __big.csv (131072x13 shift spot), __shortcut_run.csv, +# _study_run.csv, _study_vecsort_run.csv and _vshift__run.csv. +# Run with python3 -I. +import glob +import os +import statistics +import sys +from collections import defaultdict + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import item1_tables as t # noqa: E402 + +KEYS = ["sweep_inline_bits", "sweep_int_sizeof", "ARCH", "BMI2_ADX", "AVX512_IFMA", "SIMD_MUL", "limb_bits", "ndebug", + "compiler", "build_type", "cxx_flags"] # fmt: skip + + +def const_keys(path): + """The provenance keys of the first #const line of a raw CSV, as one 'key=value ...' string.""" + with open(path) as f: + for line in f: + if line.startswith("#const"): + kv = dict(tok.split("=", 1) for tok in line.split()[1:] if "=" in tok) + return " ".join(f"{k}={kv[k]}" for k in KEYS if k in kv) + return "" + + +def fmt(v): + return "" if v is None else f"{v:.2f}" + + +def big_runs(d, prefix, variant): + out = defaultdict(list) + for p in sorted(glob.glob(os.path.join(d, f"{prefix}_{variant}_big*.csv"))): + for k, v in t.load(p).items(): + out[k].append(v) + return {k: statistics.median(v) for k, v in out.items()} + + +def final(d, out, machine, pfirst, pr3, pr4): + """pfirst: 9145b4e measurement; pr3: d689f75 re-measurement; pr4: b823c5f shift re-measurement.""" + base3, new3 = t.sweep_runs(d, pr3, "base64"), t.sweep_runs(d, pr3, "new64") + base4, new4 = t.sweep_runs(d, pr4, "base64"), t.sweep_runs(d, pr4, "new64") + first = t.sweep_runs(d, pfirst, "new64") + sbase3, snew3 = t.spot_runs(d, pr3, "base64"), t.spot_runs(d, pr3, "new64") + sfirst = t.spot_runs(d, pfirst, "new64") + bbase4, bnew4 = big_runs(d, pr4, "base64"), big_runs(d, pr4, "new64") + rows = [] + keys = set(base3) | set(base4) | set(sbase3) | set(bbase4) + for key in sorted(keys, key=lambda k: (k[0], k[1], k[2], k[3])): + op, la, lb, row = key + shift = op in ("shl", "shr") + if key in bbase4: + base, after, d689, commit = bbase4[key], bnew4.get(key), None, "b823c5f" + nine = None + elif shift and key in base4: + base, after, d689, commit = t.agg(base4, key), t.agg(new4, key), t.agg(new3, key), "b823c5f" + nine = t.agg(first, key) + elif key in base3: + base, after, d689, commit = t.agg(base3, key), t.agg(new3, key), t.agg(new3, key), "d689f75" + nine = t.agg(first, key) + else: + base, after, d689, commit = t.agg(sbase3, key), t.agg(snew3, key), t.agg(snew3, key), "d689f75" + nine = t.agg(sfirst, key) + if base is None and after is None: + continue + rows.append((op, f"{la}x{lb}", row, fmt(base), fmt(nine), fmt(d689), fmt(after), commit)) + path = os.path.join(out, f"final_{machine}.csv") + with open(path, "w") as f: + f.write(f"# item 1 before/after medians, {machine}; ns per operation (vecsort and the study are in study.csv)\n") + f.write("# base_ns: cf1cf54 harness (base64); first_9145b4e_ns: first measurement of opt_1; d689f75_ns: re-measurement;\n") + f.write("# after_ns: final value (b823c5f for shl/shr, d689f75 for the other ops); empty cells were not measured\n") + f.write("# aggregation: median of the runs, divrem up to 32 limbs the mean of seeds 1 and 2; shl/shr base_ns from the\n") + f.write("# b823c5f runs; spots (large band, 3-round runs) are medians of two seeds\n") + for lab, pre, var in (("base64", pr3, "base64"), ("after", pr3, "new64"), ("after_shifts", pr4, "new64")): + probe = sorted(glob.glob(os.path.join(d, f"{pre}_{var}_seed*.csv"))) + if probe: + f.write(f"# {lab}: {const_keys(probe[0])}\n") + f.write("op,shape,row,base_ns,first_9145b4e_ns,d689f75_ns,after_ns,after_commit\n") + for r in rows: + f.write(",".join(r) + "\n") + return path + + +def study(d, out): + path = os.path.join(out, "study.csv") + with open(path, "w") as f: + f.write("# inline-capacity study (opt_1 at 9145b4e, before the add/sub/shift/single-limb follow-ups); ns per operation\n") + f.write("# (vecsort: ns per element for 100000 one-limb values, rows auto/builtin/copy); median of two runs (seeds 1 and 2,\n") + f.write("# divrem the mean of seeds); small band, add/sub/shl/shr/mul/sqr/divrem\n") + for machine, pre in (("x64", "x64"), ("m4", "mac")): + for n in (64, 128, 256, 512): + probe = sorted(glob.glob(os.path.join(d, f"{pre}_study{n}_run1.csv"))) + if probe: + f.write(f"# {machine} N={n}: {const_keys(probe[0])}\n") + f.write("machine,inline_bits,op,shape,row,ns\n") + for machine, pre in (("x64", "x64"), ("m4", "mac")): + for n in (64, 128, 256, 512): + files = [(os.path.basename(p).split("_run")[1].split(".")[0], p) + for p in sorted(glob.glob(os.path.join(d, f"{pre}_study{n}_run*.csv"))) if "vecsort" not in p] + runs = t.collect(files) + for key in sorted(runs): + f.write(f"{machine},{n},{key[0]},{key[1]}x{key[2]},{key[3]},{fmt(t.agg(runs, key))}\n") + vfiles = [(os.path.basename(p).split("_run")[1].split(".")[0], p) + for p in sorted(glob.glob(os.path.join(d, f"{pre}_study{n}_vecsort_run*.csv")))] + vruns = t.collect(vfiles) + for key in sorted(vruns): + f.write(f"{machine},{n},{key[0]},{key[1]}x{key[2]},{key[3]},{fmt(t.agg(vruns, key))}\n") + return path + + +def shortcut(d, out): + path = os.path.join(out, "shortcut.csv") + with open(path, "w") as f: + f.write("# mul_header_basecase_enabled A/B (opt_1 at 9145b4e; the 'off' build is a scratch copy with the constant\n") + f.write("# set to false, not committed); ns per operation, median of three interleaved runs\n") + f.write("machine,op,shape,row,on_ns,off_ns\n") + for machine, pre in (("x64", "x64"), ("m4", "mac")): + res = {} + for v in ("new64", "nosc64"): + files = [(os.path.basename(p), p) for p in sorted(glob.glob(os.path.join(d, f"{pre}_{v}_shortcut_run*.csv")))] + res[v] = t.collect(files) + for key in sorted(res["new64"]): + on = [x for _, x in res["new64"][key]] + off = [x for _, x in res["nosc64"].get(key, [])] + if on and off: + f.write(f"{machine},{key[0]},{key[1]}x{key[2]},{key[3]},{statistics.median(on):.2f},{statistics.median(off):.2f}\n") + return path + + +def shift_ab(d, out): + path = os.path.join(out, "shift_ab.csv") + with open(path, "w") as f: + f.write("# in-place shift A/B (variants built from scratch copies of d689f75; new64 = d689f75, v1 = revert of 2f9150b,\n") + f.write("# v2 = v1 + register-carried lshift_copy/rshift_copy loops, v3 = v2 with funnel_shl/funnel_shr); ns per operation,\n") + f.write("# median of three interleaved runs; same compiler and flags as new64 (see final_*.csv)\n") + f.write("machine,variant,op,shape,row,ns\n") + for machine, pre in (("x64", "x64"), ("m4", "mac")): + for v in ("base64", "new64", "v1", "v2", "v3"): + data = defaultdict(list) + for p in sorted(glob.glob(os.path.join(d, f"{pre}_vshift_{v}_run*.csv"))): + for k, val in t.load(p).items(): + data[k].append(val) + for key in sorted(data): + f.write(f"{machine},{v},{key[0]},{key[1]}x{key[2]},{key[3]},{statistics.median(data[key]):.2f}\n") + return path + + +def main(): + d, out = sys.argv[1], sys.argv[2] + os.makedirs(out, exist_ok=True) + for p in (final(d, out, "x64", "x64", "x64r3", "x64r4"), final(d, out, "m4", "mac", "macr3", "macr4"), + study(d, out), shortcut(d, out), shift_ab(d, out)): + print(p, os.path.getsize(p)) + + +if __name__ == "__main__": + main() diff --git a/tests/beman/big_int/perf/tools/item1_tables.py b/tests/beman/big_int/perf/tools/item1_tables.py new file mode 100644 index 00000000..2f6137d0 --- /dev/null +++ b/tests/beman/big_int/perf/tools/item1_tables.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# SPDX-License-Identifier: BSL-1.0 +# item1_tables.py [section ...] > tables.md +# Tables for item1_frontend_results.md from the CSVs written by the item 1 benchmark driver +# (__seed.csv, __spot_seed.csv, __shortcut_run.csv, +# _study_run.csv, _study_vecsort_run.csv). Run with python3 -I. +# Aggregation: median across runs; divrem shapes of up to 32 limbs use the mean over seeds (the operands change the +# division path), each seed first reduced to the median of its repeats. +import csv +import glob +import os +import statistics +import sys +from collections import defaultdict + + +def load(path): + d = {} + with open(path) as f: + for line in f: + if line.startswith("#") or line.startswith("op,") or not line.strip(): + continue + p = line.strip().split(",") + d[(p[0], int(p[2]), int(p[3]), p[1])] = float(p[6]) + return d + + +def collect(files): + """files: list of (seed, path). Returns {key: [(seed, value)]}.""" + out = defaultdict(list) + for seed, path in files: + for k, v in load(path).items(): + out[k].append((seed, v)) + return out + + +def agg(runs, key): + vals = runs.get(key) + if not vals: + return None + op, la = key[0], key[1] + if op == "divrem" and la <= 32: + per_seed = defaultdict(list) + for s, v in vals: + per_seed[s].append(v) + return statistics.mean(statistics.median(v) for v in per_seed.values()) + return statistics.median(v for _, v in vals) + + +def sweep_runs(d, prefix, variant): + files = [] + for p in sorted(glob.glob(os.path.join(d, f"{prefix}_{variant}_seed*.csv"))): + tag = os.path.basename(p).split("_seed")[1].split(".")[0] + files.append((tag[0], p)) + return collect(files) + + +def spot_runs(d, prefix, variant): + files = [] + for p in sorted(glob.glob(os.path.join(d, f"{prefix}_{variant}_spot_seed*.csv"))): + files.append((os.path.basename(p).split("_seed")[1].split(".")[0], p)) + return collect(files) + + +def f1(x, digits=1): + return "-" if x is None else f"{x:.{digits}f}" + + +def arrow(a, b, digits=1): + if a is None and b is None: + return "-" + return f"{f1(a, digits)} -> {f1(b, digits)}" + + +SMALL = { + "add": [(1, 1), (2, 2), (4, 4), (8, 8), (16, 16)], + "sub": [(1, 1), (2, 2), (4, 4), (8, 8), (16, 16)], + "shl": [(1, 13), (4, 13), (16, 13)], + "shr": [(1, 13), (4, 13), (16, 13)], + "mul": [(2, 2), (3, 3), (4, 4), (6, 6), (8, 8), (12, 12), (16, 16)], + "sqr": [(4, 4), (16, 16)], + "divrem": [(2, 1), (8, 4), (32, 16)], + "fromchars": [(4, 10), (16, 10)], + "tochars": [(4, 10), (16, 10)], +} +MEDIUM = { + "add": [(64, 64), (256, 256), (1024, 1024), (2000, 2000)], + "sub": [(64, 64), (1024, 1024)], + "shl": [(256, 13), (2000, 13), (2000, 77)], + "shr": [(256, 13), (2000, 13), (2000, 77)], + "mul": [(32, 32), (64, 64), (256, 256), (1000, 1000), (2000, 2000), (2000, 100)], + "sqr": [(64, 64), (512, 512), (2000, 2000)], + "divrem": [(128, 64), (512, 256), (2000, 1000), (4000, 2000), (2000, 100), (65536, 1)], + "fromchars": [(64, 10), (256, 10), (2000, 10)], + "tochars": [(64, 10), (256, 10), (2000, 10)], +} + + +def before_after(d, prefix, table): + base, new = sweep_runs(d, prefix, "base64"), sweep_runs(d, prefix, "new64") + print("| op | shape | auto ns | inplace ns | auto - floor ns | auto/gmpz | auto - kernel ns |") + print("|---|---|---:|---:|---:|---:|---:|") + for op, shapes in table.items(): + for la, lb in shapes: + def g(runs, row): + return agg(runs, (op, la, lb, row)) + + ab, an = g(base, "auto"), g(new, "auto") + ib, i_n = g(base, "inplace"), g(new, "inplace") + fb, fn = g(base, "floor"), g(new, "floor") + zb, zn = g(base, "gmpz"), g(new, "gmpz") + kb, kn = g(base, "kernel"), g(new, "kernel") + + def sub(a, b): + return None if a is None or b is None else a - b + + def rat(a, b): + return None if a is None or b is None or b == 0 else a / b + + print( + f"| {op} | {la}x{lb} | {arrow(ab, an)} | {arrow(ib, i_n)} | {arrow(sub(ab, fb), sub(an, fn))} | " + f"{arrow(rat(ab, zb), rat(an, zn), 2)} | {arrow(sub(ab, kb), sub(an, kn))} |" + ) + + +def spots(d, prefix): + base, new = spot_runs(d, prefix, "base64"), spot_runs(d, prefix, "new64") + keys = sorted({k for k in list(base) + list(new)}, key=lambda k: (k[0], k[1], k[3])) + shapes = [] + for k in keys: + if (k[0], k[1], k[2]) not in shapes: + shapes.append((k[0], k[1], k[2])) + print("| op | shape | auto ns | kernel ns | gmpz ns | auto/gmpz | change in auto |") + print("|---|---|---:|---:|---:|---:|---:|") + for op, la, lb in shapes: + def g(runs, row): + return agg(runs, (op, la, lb, row)) + + ab, an = g(base, "auto"), g(new, "auto") + z = g(new, "gmpz") + ch = None if not ab or not an else (an - ab) / ab * 100 + print( + f"| {op} | {la}x{lb} | {arrow(ab, an, 0)} | {arrow(g(base, 'kernel'), g(new, 'kernel'), 0)} | " + f"{f1(z, 0)} | {arrow(None if not z else ab / z, None if not z else an / z, 2)} | " + f"{'-' if ch is None else f'{ch:+.1f}%'} |" + ) + + +def shortcut(d, prefix): + rows = {} + for v in ("new64", "nosc64"): + files = [(os.path.basename(p), p) for p in sorted(glob.glob(os.path.join(d, f"{prefix}_{v}_shortcut_run*.csv")))] + rows[v] = collect(files) + keys = sorted({k for k in rows["new64"] if k[3] == "auto"}, key=lambda k: (k[0], k[1])) + print("| op | shape | shortcut on (new64) ns | off ns | on - off ns | runs on / off |") + print("|---|---|---:|---:|---:|---|") + for k in keys: + on = [v for _, v in rows["new64"][k]] + off = [v for _, v in rows["nosc64"].get(k, [])] + if not on or not off: + continue + mo, mf = statistics.median(on), statistics.median(off) + print( + f"| {k[0]} | {k[1]}x{k[2]} | {mo:.1f} | {mf:.1f} | {mo - mf:+.1f} | " + f"{'/'.join(f'{x:.1f}' for x in on)} vs {'/'.join(f'{x:.1f}' for x in off)} |" + ) + + +STUDY_ROWS = [ + ("add", (1, 1)), ("add", (4, 4)), ("add", (16, 16)), + ("sub", (1, 1)), ("sub", (4, 4)), ("sub", (16, 16)), + ("shl", (1, 13)), ("shl", (4, 13)), ("shl", (16, 13)), + ("shr", (1, 13)), ("shr", (4, 13)), ("shr", (16, 13)), + ("mul", (1, 1)), ("mul", (2, 2)), ("mul", (4, 4)), ("mul", (8, 8)), ("mul", (16, 16)), + ("sqr", (4, 4)), ("sqr", (16, 16)), + ("divrem", (2, 1)), ("divrem", (8, 4)), ("divrem", (32, 16)), +] # fmt: skip + + +def study(d, prefix): + ns = (64, 128, 256, 512) + runs = {} + for n in ns: + files = [] + for p in sorted(glob.glob(os.path.join(d, f"{prefix}_study{n}_run*.csv"))): + if "vecsort" in p: + continue + files.append((os.path.basename(p).split("_run")[1].split(".")[0], p)) + runs[n] = collect(files) + for row in ("auto", "inplace"): + print(f"\n**{row} ns**\n") + print("| op | shape | " + " | ".join(f"N={n}" for n in ns) + " |") + print("|---|---|" + "---:|" * len(ns)) + for op, (la, lb) in STUDY_ROWS: + vals = [agg(runs[n], (op, la, lb, row)) for n in ns] + if all(v is None for v in vals): + continue + print(f"| {op} | {la}x{lb} | " + " | ".join(f1(v) for v in vals) + " |") + print("\n**vecsort, ns per element (100000 one-limb values: copy + sort + sum)**\n") + print("| row | " + " | ".join(f"N={n}" for n in ns) + " |") + print("|---|" + "---:|" * len(ns)) + vs = {} + for n in ns: + files = [(os.path.basename(p).split("_run")[1].split(".")[0], p) + for p in sorted(glob.glob(os.path.join(d, f"{prefix}_study{n}_vecsort_run*.csv")))] + vs[n] = collect(files) + for row in ("auto", "builtin", "copy"): + vals = [] + for n in ns: + v = [x for _, x in vs[n].get(("vecsort", 100000, 1, row), [])] + vals.append(None if not v else statistics.median(v)) + print(f"| {row} | " + " | ".join(f1(v, 2) for v in vals) + " |") + + +if __name__ == "__main__": + d, prefix = sys.argv[1], sys.argv[2] + sections = sys.argv[3:] or ["small", "medium", "spots", "shortcut", "study"] + for s in sections: + print(f"\n### {s}\n") + if s == "small": + before_after(d, prefix, SMALL) + elif s == "medium": + before_after(d, prefix, MEDIUM) + elif s == "spots": + spots(d, prefix) + elif s == "shortcut": + shortcut(d, prefix) + elif s == "study": + study(d, prefix) diff --git a/tests/beman/big_int/perf/tools/item1_vshift.py b/tests/beman/big_int/perf/tools/item1_vshift.py new file mode 100644 index 00000000..e7925c58 --- /dev/null +++ b/tests/beman/big_int/perf/tools/item1_vshift.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# SPDX-License-Identifier: BSL-1.0 +# item1_vshift.py > table.md +# Median over runs of the in-place shift A/B (_vshift__run.csv), one row per op/shape/row with the +# percentage against base64. Run with python3 -I. +import glob +import os +import statistics +import sys +from collections import defaultdict + +VARIANTS = ["base64", "new64", "v1", "v2", "v3"] + + +def main(): + d, prefix = sys.argv[1], sys.argv[2] + data = defaultdict(lambda: defaultdict(list)) + for v in VARIANTS: + for p in sorted(glob.glob(os.path.join(d, f"{prefix}_vshift_{v}_run*.csv"))): + for line in open(p): + if line.startswith("op,") or not line.strip(): + continue + f = line.strip().split(",") + data[(f[0], int(f[2]), int(f[3]), f[1])][v].append(float(f[6])) + print("| op | shape | row | " + " | ".join(VARIANTS) + " |") + print("|---|---|---|" + "---:|" * len(VARIANTS)) + worst = {v: [] for v in VARIANTS[1:]} + for key in sorted(data, key=lambda k: (k[0], k[1], k[3] != "auto", k[3] != "inplace", k[3])): + cells = [] + base = statistics.median(data[key]["base64"]) if data[key].get("base64") else None + for v in VARIANTS: + vals = data[key].get(v) + if not vals: + cells.append("-") + continue + m = statistics.median(vals) + if v == "base64" or base is None: + cells.append(f"{m:.1f}") + else: + pct = (m - base) / base * 100 + cells.append(f"{m:.1f} ({pct:+.0f}%)") + if key[3] in ("auto", "inplace"): + worst[v].append((pct, key)) + print(f"| {key[0]} | {key[1]}x{key[2]} | {key[3]} | " + " | ".join(cells) + " |") + print() + for v in VARIANTS[1:]: + over = [(p, k) for p, k in worst[v] if p > 3.0] + print(f"- {v}: {len(over)} of {len(worst[v])} auto/inplace rows more than 3% slower than base64" + + (": " + ", ".join(f"{k[0]} {k[1]}x{k[2]} {k[3]} {p:+.0f}%" for p, k in sorted(over, reverse=True)) if over else "")) + + +if __name__ == "__main__": + main() diff --git a/tests/beman/big_int/perf/tools/selftest.sh b/tests/beman/big_int/perf/tools/selftest.sh index 905b5cd5..de36eeab 100755 --- a/tests/beman/big_int/perf/tools/selftest.sh +++ b/tests/beman/big_int/perf/tools/selftest.sh @@ -11,7 +11,7 @@ while IFS= read -r spec; do op=${spec%% *} shapes=${spec#* } # shellcheck disable=SC2086 - if ! "$B" "$op" --rows auto,inplace,kernel,kernelip,gmp,gmpz --rounds 1 --round-ms 2 $shapes >/dev/null; then + if ! "$B" "$op" --rows auto,inplace,kernel,kernelip,floor,gmp,gmpz,builtin,copy --rounds 1 --round-ms 2 $shapes >/dev/null; then echo "selftest: FAILED $spec" >&2 rc=1 else @@ -30,5 +30,6 @@ div 8x4 tochars 1x10 4x10 16x10 64x10 256x10 16x16 2000x10 fromchars 1x10 4x10 16x10 64x10 256x10 2000x10 gcd 2x2 4x4 16x16 64x64 256x256 100x30 +vecsort 1000x1 SPECS exit $rc diff --git a/tests/beman/big_int/perf/tools/summarize.py b/tests/beman/big_int/perf/tools/summarize.py index fefea022..bc2ee83a 100755 --- a/tests/beman/big_int/perf/tools/summarize.py +++ b/tests/beman/big_int/perf/tools/summarize.py @@ -38,7 +38,7 @@ def spread(key): def shape_spread(op, la, lb): - s = [(spread((op, la, lb, p)), p) for p in ("auto", "inplace", "kernel", "gmp", "gmpz")] + s = [(spread((op, la, lb, p)), p) for p in ("auto", "inplace", "kernel", "floor", "gmp", "gmpz", "builtin", "copy")] s = [x for x in s if x[0] is not None] return max(s) if s else (None, "") @@ -72,7 +72,7 @@ def spr_txt(op, la, lb): if m.startswith("# gap_sweep") or m.startswith("#const"): out.append(" " + m[:200]) out.append("") -out.append("Ratios: >1 means big_int is slower. inplace is per-op (already halved). Gap ns = auto - kernel (front end).\n") +out.append("Ratios: >1 means big_int is slower. inplace is per-op (already halved). Gap ns: auto - kernel is the whole front end, auto - floor is what remains above one allocate/deallocate pair of the result size (API floor), inplace - kernel is the in-place front end.\n") gaps = [] ops = [] allspr = [(spread(k), k) for k in data if spread(k) is not None] @@ -87,18 +87,31 @@ def spr_txt(op, la, lb): ops.append(k[0]) for op in ops: out.append(f"## {op}\n") + if op == "vecsort": + out.append("ns per element for copy-assign + sort + sum of N one-limb values (builtin = std::int64_t).\n") + out.append("| N | auto ns | builtin ns | auto/builtin | copy ns | auto-copy ns |") + out.append("|---:|---:|---:|---:|---:|---:|") + for _, la, lb in [k for k in order if k[0] == op]: + a_, b_, c_ = (med(op, la, lb, x) for x in ("auto", "builtin", "copy")) + d_ = None if a_ is None or c_ is None else a_ - c_ + out.append(f"| {la} | {fmt(a_,2)} | {fmt(b_,2)} | {fmt(ratio(a_, b_))} | {fmt(c_,2)} | {fmt(d_,2)} |") + out.append("") + continue for b in ("small", "medium", "large"): shapes = [k for k in order if k[0] == op and band(op, k[1]) == b] if not shapes: continue out.append(f"**{b}**\n") - out.append("| shape | auto ns | gmpz ns | auto/gmpz | kernel/gmp | inplace/gmpz | auto-kernel ns | spread % (row) |") - out.append("|---|---:|---:|---:|---:|---:|---:|---:|") + out.append("| shape | auto ns | gmpz ns | auto/gmpz | kernel/gmp | inplace/gmpz | auto-kernel ns | floor ns | auto-floor ns | inplace-kernel ns | spread % (row) |") + out.append("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|") for _, la, lb in shapes: a, kn, g, z, ip = (med(op, la, lb, x) for x in ("auto", "kernel", "gmp", "gmpz", "inplace")) r1, r2, r3 = ratio(a, z), ratio(kn, g), ratio(ip, z) + fl = med(op, la, lb, "floor") d = None if a is None or kn is None else a - kn - out.append(f"| {la}x{lb} | {fmt(a,1)} | {fmt(z,1)} | {fmt(r1)} | {fmt(r2)} | {fmt(r3)} | {fmt(d,1)} | {spr_txt(op, la, lb)} |") + d2 = None if a is None or fl is None else a - fl + d3 = None if ip is None or kn is None else ip - kn + out.append(f"| {la}x{lb} | {fmt(a,1)} | {fmt(z,1)} | {fmt(r1)} | {fmt(r2)} | {fmt(r3)} | {fmt(d,1)} | {fmt(fl,1)} | {fmt(d2,1)} | {fmt(d3,1)} | {spr_txt(op, la, lb)} |") if r1: gaps.append((r1, "auto/gmpz", op, la, lb)) if r2: diff --git a/tests/beman/big_int/scratch_heap.test.cpp b/tests/beman/big_int/scratch_heap.test.cpp index 4f6af5ec..d634d44d 100644 --- a/tests/beman/big_int/scratch_heap.test.cpp +++ b/tests/beman/big_int/scratch_heap.test.cpp @@ -8,6 +8,8 @@ #include +#include "util/util_counting_allocator.hpp" + #include #include @@ -22,37 +24,8 @@ namespace { namespace detail = BEMAN_BIG_INT_NAMESPACE::detail; using uint_t = BEMAN_BIG_INT_NAMESPACE::uint_multiprecision_t; -struct counting_state { - std::map allocations; - std::map live_elements; -}; - -template -struct counting_allocator { - using value_type = T; - - counting_state* state = nullptr; - - counting_allocator() = default; - explicit counting_allocator(counting_state* s) : state(s) {} - template - counting_allocator(const counting_allocator& other) : state(other.state) {} - - T* allocate(const std::size_t n) { - ++state->allocations[std::type_index(typeid(T))]; - state->live_elements[std::type_index(typeid(T))] += n; - return std::allocator{}.allocate(n); - } - void deallocate(T* p, const std::size_t n) noexcept { - state->live_elements[std::type_index(typeid(T))] -= n; - std::allocator{}.deallocate(p, n); - } - - template - bool operator==(const counting_allocator& other) const { - return state == other.state; - } -}; +using BEMAN_BIG_INT_NAMESPACE::test_util::counting_allocator; +using BEMAN_BIG_INT_NAMESPACE::test_util::counting_state; TEST(ScratchHeap, HooksRouteThroughTheOwningAllocatorsRebinds) { counting_state state; diff --git a/tests/beman/big_int/span_primitives.test.cpp b/tests/beman/big_int/span_primitives.test.cpp new file mode 100644 index 00000000..7c1ce014 --- /dev/null +++ b/tests/beman/big_int/span_primitives.test.cpp @@ -0,0 +1,267 @@ +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-License-Identifier: BSL-1.0 + +// Tests for the front-end span primitives add_n_tail, sub_n_tail, lshift_copy and rshift_copy. + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace bb = BEMAN_BIG_INT_NAMESPACE; + +namespace { + +using limb = bb::uint_multiprecision_t; +constexpr unsigned limb_width = static_cast(std::numeric_limits::digits); +constexpr limb limb_max = std::numeric_limits::max(); + +// Converts without a cast when the types already agree (GCC's -Wuseless-cast), and with one otherwise. +template +constexpr limb to_limb(const U v) noexcept { + if constexpr (std::is_same_v) { + return v; + } else { + return static_cast(v); + } +} + +using bits_t = std::vector; // little-endian bit string, the reference representation + +bits_t to_bits(const std::vector& v) { + bits_t out(v.size() * limb_width); + for (std::size_t i = 0; i < out.size(); ++i) { + out[i] = ((v[i / limb_width] >> (i % limb_width)) & 1U) != 0; + } + return out; +} + +std::vector from_bits(const bits_t& b) { + std::vector out(b.size() / limb_width, limb{0}); + for (std::size_t i = 0; i < b.size(); ++i) { + if (b[i]) { + out[i / limb_width] |= limb{1} << (i % limb_width); + } + } + return out; +} + +// Reference a +/- b over a.size() limbs, returning the carry/borrow out. +bool ref_addsub(std::vector& out, const std::vector& a, const std::vector& b, const bool sub) { + const bits_t abits = to_bits(a); + bits_t bbits(abits.size(), false); + const bits_t braw = to_bits(b); + for (std::size_t i = 0; i < braw.size(); ++i) { + bbits[i] = braw[i]; + } + bits_t res(abits.size()); + bool c = false; + for (std::size_t i = 0; i < abits.size(); ++i) { + if (!sub) { + res[i] = (abits[i] != bbits[i]) != c; + c = (abits[i] && bbits[i]) || (c && (abits[i] != bbits[i])); + } else { + res[i] = (abits[i] != bbits[i]) != c; + c = (!abits[i] && bbits[i]) || (c && !(abits[i] != bbits[i])); + } + } + out = from_bits(res); + return c; +} + +limb pick(std::mt19937_64& rng) { + switch (rng() % 5U) { + case 0: + return 0; + case 1: + return limb_max; + case 2: + return 1; + default: + return to_limb(rng()); + } +} + +std::vector random_limbs(std::mt19937_64& rng, const std::size_t n) { + std::vector v(n); + for (auto& x : v) { + x = pick(rng); + } + return v; +} + +constexpr limb poison = to_limb(0xA5A5A5A5A5A5A5A5ULL); + +template +void check_addsub(const bool in_place) { + std::mt19937_64 rng{Sub ? 11U : 7U}; + for (std::size_t m = 0; m <= 9; ++m) { + for (std::size_t n = 0; n <= m; ++n) { + for (int trial = 0; trial < 300; ++trial) { + std::vector a = random_limbs(rng, m); + std::vector b = random_limbs(rng, n); + if (trial % 7 == 0) { // long carry / borrow ripples + for (auto& x : a) { + x = Sub ? limb{0} : limb_max; + } + if (n != 0) { + b[0] = 1; + } + } + std::vector expected; + const bool expected_flag = ref_addsub(expected, a, b, Sub); + + std::vector dst_storage = in_place ? a : std::vector(m, poison); + const limb* a_ptr = in_place ? dst_storage.data() : a.data(); + const std::span a_span{a_ptr, m}; + const std::span dst_span{dst_storage.data(), m}; + const bool flag = Sub ? bb::detail::sub_n_tail(dst_span, a_span, std::span{b}) + : bb::detail::add_n_tail(dst_span, a_span, std::span{b}); + ASSERT_EQ(flag, expected_flag) << "m=" << m << " n=" << n; + ASSERT_EQ(dst_storage, expected) << "m=" << m << " n=" << n; + } + } + } +} + +} // namespace + +TEST(SpanPrimitives, AddNTailSeparateDestination) { check_addsub(false); } +TEST(SpanPrimitives, AddNTailInPlace) { check_addsub(true); } +TEST(SpanPrimitives, SubNTailSeparateDestination) { check_addsub(false); } +TEST(SpanPrimitives, SubNTailInPlace) { check_addsub(true); } + +namespace { + +std::vector ref_shift(const std::vector& src, const unsigned bits, const bool left, limb& out_bits) { + const bits_t in = to_bits(src); + bits_t res(in.size(), false); + out_bits = 0; + for (std::size_t i = 0; i < in.size(); ++i) { + if (left) { + if (i + bits < in.size()) { + res[i + bits] = in[i]; + } else if (in[i]) { + out_bits |= limb{1} << (i + bits - in.size()); + } + } else { + if (i >= bits) { + res[i - bits] = in[i]; + } else if (in[i]) { + out_bits |= limb{1} << (limb_width - bits + i); + } + } + } + return from_bits(res); +} + +void check_shift(const bool left, const bool aliased) { + std::mt19937_64 rng{left ? 3U : 5U}; + for (std::size_t n = 0; n <= 9; ++n) { + for (unsigned bits = 1; bits < limb_width; bits += (bits < 5 || bits + 6 > limb_width ? 1 : 7)) { + for (int trial = 0; trial < 20; ++trial) { + const std::vector src = random_limbs(rng, n); + limb expected_out = 0; + const auto expected = ref_shift(src, bits, left, expected_out); + + // Non-aliased: destination below (rshift) or above (lshift) in a bigger buffer. + std::vector buf(n + 3, poison); + limb* dst = nullptr; + const limb* s = nullptr; + std::vector src_buf = src; + if (aliased) { + buf.assign(src.begin(), src.end()); + buf.resize(n + 3, poison); + dst = buf.data(); + s = buf.data(); + } else { + dst = buf.data() + 1; + s = src_buf.data(); + } + const limb out = + left ? bb::detail::lshift_copy(dst, s, n, bits) : bb::detail::rshift_copy(dst, s, n, bits); + ASSERT_EQ(out, expected_out) << "n=" << n << " bits=" << bits; + ASSERT_EQ(std::vector(dst, dst + n), expected) << "n=" << n << " bits=" << bits; + if (!aliased) { + ASSERT_EQ(buf[0], poison); + ASSERT_EQ(buf[n + 1], poison); + ASSERT_EQ(buf[n + 2], poison); + } + } + } + } +} + +// Overlapping but not identical ranges: lshift with dst above src, rshift with dst below src. +void check_shift_overlap(const bool left) { + std::mt19937_64 rng{left ? 13U : 17U}; + for (std::size_t n = 1; n <= 9; ++n) { + for (std::size_t offset = 1; offset <= 3; ++offset) { + const unsigned bits = 1 + static_cast(rng() % (limb_width - 1)); + const std::vector src = random_limbs(rng, n); + limb expected_out = 0; + const auto expected = ref_shift(src, bits, left, expected_out); + + std::vector buf(n + offset, poison); + limb* s = left ? buf.data() : buf.data() + offset; + limb* dst = left ? buf.data() + offset : buf.data(); + std::copy(src.begin(), src.end(), s); + const limb out = + left ? bb::detail::lshift_copy(dst, s, n, bits) : bb::detail::rshift_copy(dst, s, n, bits); + ASSERT_EQ(out, expected_out); + ASSERT_EQ(std::vector(dst, dst + n), expected) << "n=" << n << " offset=" << offset; + } + } +} + +} // namespace + +TEST(SpanPrimitives, LshiftCopySeparate) { check_shift(true, false); } +TEST(SpanPrimitives, LshiftCopyAliased) { check_shift(true, true); } +TEST(SpanPrimitives, LshiftCopyOverlapping) { check_shift_overlap(true); } +TEST(SpanPrimitives, RshiftCopySeparate) { check_shift(false, false); } +TEST(SpanPrimitives, RshiftCopyAliased) { check_shift(false, true); } +TEST(SpanPrimitives, RshiftCopyOverlapping) { check_shift_overlap(false); } + +// ----- constant evaluation ----- + +namespace { + +consteval bool constexpr_primitives() { + std::array a{limb_max, limb_max, limb_max, limb_max, 5}; + std::array b{1, 0}; + std::array dst{}; + // a + b: the carry ripples through four limbs. + if (bb::detail::add_n_tail(dst, a, b) || dst[0] != 0 || dst[1] != 0 || dst[2] != 0 || dst[3] != 0 || dst[4] != 6) { + return false; + } + // In-place subtract undoes it. + const std::array sum = dst; + if (bb::detail::sub_n_tail(dst, sum, b) || dst != a) { + return false; + } + // 0 - 1 borrows out of the top. + std::array z{}; + std::array one{1, 0}; + if (!bb::detail::sub_n_tail(z, std::array{}, one) || z[0] != limb_max || z[1] != limb_max) { + return false; + } + // Shifts round-trip. + std::array v{0x123, 0x456, 0x789}; + std::array w{}; + const limb out = bb::detail::lshift_copy(w.data(), v.data(), 3, 4); + if (out != 0 || w[0] != 0x1230 || w[1] != 0x4560 || w[2] != 0x7890) { + return false; + } + const limb lost = bb::detail::rshift_copy(w.data(), w.data(), 3, 4); + return lost == 0 && w == v; +} +static_assert(constexpr_primitives()); + +} // namespace diff --git a/tests/beman/big_int/util/util_counting_allocator.hpp b/tests/beman/big_int/util/util_counting_allocator.hpp new file mode 100644 index 00000000..3c3bab66 --- /dev/null +++ b/tests/beman/big_int/util/util_counting_allocator.hpp @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-License-Identifier: BSL-1.0 + +// A stateful counting allocator shared by the allocation tests. The counters live in a counting_state owned by the +// test, so every rebound copy of the allocator reports into the same place. Allocators that were default constructed +// (state == nullptr) forward to std::allocator without counting. + +#ifndef BEMAN_BIG_INT_TEST_UTIL_COUNTING_ALLOCATOR_HPP +#define BEMAN_BIG_INT_TEST_UTIL_COUNTING_ALLOCATOR_HPP + +#include +#include +#include +#include +#include + +#include + +BEMAN_BIG_INT_BEGIN_NAMESPACE +namespace test_util { + +struct counting_state { + std::map allocations; + std::map live_elements; + std::size_t total_allocations{0}; + std::size_t total_deallocations{0}; + + void reset_totals() noexcept { + total_allocations = 0; + total_deallocations = 0; + } +}; + +template +struct counting_allocator { + using value_type = T; + + counting_state* state = nullptr; + + counting_allocator() = default; + explicit counting_allocator(counting_state* s) : state(s) {} + template + counting_allocator(const counting_allocator& other) : state(other.state) {} + + T* allocate(const std::size_t n) { + if (state != nullptr) { + ++state->allocations[std::type_index(typeid(T))]; + state->live_elements[std::type_index(typeid(T))] += n; + ++state->total_allocations; + } + return std::allocator{}.allocate(n); + } + void deallocate(T* p, const std::size_t n) noexcept { + if (state != nullptr) { + state->live_elements[std::type_index(typeid(T))] -= n; + ++state->total_deallocations; + } + std::allocator{}.deallocate(p, n); + } + + template + bool operator==(const counting_allocator& other) const { + return state == other.state; + } +}; + +} // namespace test_util +BEMAN_BIG_INT_END_NAMESPACE + +#endif // BEMAN_BIG_INT_TEST_UTIL_COUNTING_ALLOCATOR_HPP