diff --git a/benchmarks/memory/BUILD.bazel b/benchmarks/memory/BUILD.bazel index 8d0c940..8f54468 100644 --- a/benchmarks/memory/BUILD.bazel +++ b/benchmarks/memory/BUILD.bazel @@ -5,6 +5,7 @@ cc_binary( srcs = ["memcpy_benchmark.cpp"], deps = [ "//lib/memory:memcpy", + "//lib/memory/copy:copy", "@google_benchmark//:benchmark_main", ], ) diff --git a/benchmarks/memory/memcpy_benchmark.cpp b/benchmarks/memory/memcpy_benchmark.cpp index 49d976e..7ed6368 100644 --- a/benchmarks/memory/memcpy_benchmark.cpp +++ b/benchmarks/memory/memcpy_benchmark.cpp @@ -1,3 +1,4 @@ +#include "lib/memory/copy/copy.h" #include "lib/memory/memcpy.h" #include @@ -18,14 +19,14 @@ std::vector MakeInput(std::size_t size) { return input; } -void BM_Memcpy(benchmark::State& state) { +template +void BM_Copy(benchmark::State& state) { const auto size = static_cast(state.range(0)); const std::vector source = MakeInput(size); std::vector destination(size); for (auto _ : state) { - memcopy_bench::memory::CopyWithMemcpy( - destination.data(), source.data(), size); + CopyFn(destination.data(), source.data(), size); benchmark::DoNotOptimize(destination.data()); benchmark::ClobberMemory(); } @@ -34,8 +35,42 @@ void BM_Memcpy(benchmark::State& state) { static_cast(size)); } -BENCHMARK(BM_Memcpy)->RangeMultiplier(4)->Range(64, 64 << 20); +inline void* CopyNaiveWrapper(void* d, const void* s, std::size_t n) { + return memory_bench::copy_naive(d, s, n); +} + +} // namespace + +template +void RegisterBMCopy(const char *name) +{ + benchmark::RegisterBenchmark(name, BM_Copy) + ->RangeMultiplier(4) + ->Range(64, 64 << 20); +} + +static const bool Registered = [] +{ + RegisterBMCopy("BM_Memcpy"); + RegisterBMCopy("BM_CopyNaive"); + RegisterBMCopy("BM_CopyDuff"); + RegisterBMCopy("BM_CopyWords64"); + RegisterBMCopy("BM_CopyWords64Unrolled4"); + RegisterBMCopy("BM_CopySmallOverlapping"); + return true; +}(); -} // namespace +#if defined(__x86_64__) || defined(_M_X64) +static const bool x86Registered = [] +{ + RegisterBMCopy("BM_CopyRepMovsb"); + RegisterBMCopy("BM_CopyRepMovsq"); + RegisterBMCopy("BM_CopySSE2"); + RegisterBMCopy("BM_CopyAVX2"); + RegisterBMCopy("BM_CopyAVX2Stream"); + RegisterBMCopy("BM_CopyAVX2Prefetch"); + return true; +}(); +#endif BENCHMARK_MAIN(); diff --git a/justfile b/justfile new file mode 100644 index 0000000..bf5f35f --- /dev/null +++ b/justfile @@ -0,0 +1,39 @@ +# Set default recipe to show available commands +default: + @just --list + +# Build all targets in the workspace +build: + bazel build //... + +# Run all unit tests with error logs on failure +test: + bazel test //... --test_output=errors + +# Run AddressSanitizer and UndefinedBehaviorSanitizer checks +sanitizers: + bazel test --config=asan //... --test_output=errors + bazel test --config=ubsan //... --test_output=errors + +# Run C++ linter (clang-tidy) +lint: + sh tools/run-clang-tidy.sh + +# Run complete local verification (format, lint, tests, and sanitizers) +check: lint test sanitizers + +# Run benchmarks (optional filter argument: `just bench BM_Memcpy`) +bench filter="": + bazel run //benchmarks/memory:memcpy_benchmark -- {{ if filter != "" { "--benchmark_filter=" + filter } else { "" } }} + +# Refresh compile_commands.json for clangd and IDE completion +compile-commands: + bazel run @hedron_compile_commands//:refresh_all + +# Remove local build outputs and symlinks +clean: + bazel clean + +# Wipe the entire local Bazel cache and reset external downloads +clean-all: + bazel clean --expunge \ No newline at end of file diff --git a/lib/memory/copy/BUILD.bazel b/lib/memory/copy/BUILD.bazel new file mode 100644 index 0000000..dd44924 --- /dev/null +++ b/lib/memory/copy/BUILD.bazel @@ -0,0 +1,17 @@ +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_cc//cc:cc_test.bzl", "cc_test") + +cc_library( + name = "copy", + hdrs = ["copy.h"], + visibility = ["//visibility:public"], +) + +cc_test( + name = "copy_test", + srcs = ["copy_test.cpp"], + deps = [ + ":copy", + "@googletest//:gtest_main", + ], +) diff --git a/lib/memory/copy/copy.h b/lib/memory/copy/copy.h new file mode 100644 index 0000000..9df9799 --- /dev/null +++ b/lib/memory/copy/copy.h @@ -0,0 +1,385 @@ +#ifndef MEMCP_BENCH_MEMORY_COPY_COPY_H +#define MEMCP_BENCH_MEMORY_COPY_COPY_H + +#include +#include +#include + +#if defined(__x86_64__) || defined(_M_X64) +#include +#endif + +namespace memory_bench { + +/** + * Standard library memcpy wrapper baseline. + */ +inline void *copy_memcpy(void *dest, const void *src, size_t len) { + return std::memcpy(dest, src, len); +} + +/** +Copy bytes from src to dest in a loop +*/ +inline void *copy_naive(void *dest, const void *src, size_t len) { + auto *dest_b = static_cast(dest); + const auto *src_b = static_cast(src); + while (len--) { + *dest_b++ = *src_b++; + } + return dest; +} + +/** +A naive copy specialized by the . I.e. we know the size of the copy +at compile-time. +*/ +template +inline constexpr void *copy_naive(void *dest, const void *src) { + auto *dest_b = static_cast(dest); + const auto *src_b = static_cast(src); + auto len = Len; + while (len--) { + *dest_b++ = *src_b++; + } + return dest; +} + +/** + * Fixed-size copy delegating to std::memcpy, which compilers inline + * into optimal register-width move instructions for compile-time sizes. + */ +template inline void *copy_fixed(void *dest, const void *src) { + return std::memcpy(dest, src, Len); +} + +/** + * Duff's device byte-level unrolled copy. + */ +inline void *copy_duff(void *dest, const void *src, size_t len) { + if (len == 0) { + return dest; + } + auto *d = static_cast(dest); + const auto *s = static_cast(src); + size_t n = (len + 7) / 8; + switch (len % 8) { + case 0: + do { + *d++ = *s++; + case 7: + *d++ = *s++; + case 6: + *d++ = *s++; + case 5: + *d++ = *s++; + case 4: + *d++ = *s++; + case 3: + *d++ = *s++; + case 2: + *d++ = *s++; + case 1: + *d++ = *s++; + } while (--n > 0); + } + return dest; +} + +/** + * 64-bit scalar word copy with byte-level scalar tail. (copy by 8 bytes is + * possible, then by byte) + */ +inline void *copy_words64(void *dest, const void *src, size_t len) { + auto *d = static_cast(dest); + const auto *s = static_cast(src); + + while (len >= sizeof(uint64_t)) { + uint64_t w; + std::memcpy(&w, s, sizeof(uint64_t)); + std::memcpy(d, &w, sizeof(uint64_t)); + d += sizeof(uint64_t); + s += sizeof(uint64_t); + len -= sizeof(uint64_t); + } + while (len--) { + *d++ = *s++; + } + return dest; +} + +/** + * 64-bit scalar words unrolled 4x (32 bytes per iteration). + */ +inline void *copy_words64_unrolled4(void *dest, const void *src, size_t len) { + auto *d = static_cast(dest); + const auto *s = static_cast(src); + + while (len >= 4 * sizeof(uint64_t)) { + uint64_t w0, w1, w2, w3; + std::memcpy(&w0, s, sizeof(uint64_t)); + std::memcpy(&w1, s + 8, sizeof(uint64_t)); + std::memcpy(&w2, s + 16, sizeof(uint64_t)); + std::memcpy(&w3, s + 24, sizeof(uint64_t)); + std::memcpy(d, &w0, sizeof(uint64_t)); + std::memcpy(d + 8, &w1, sizeof(uint64_t)); + std::memcpy(d + 16, &w2, sizeof(uint64_t)); + std::memcpy(d + 24, &w3, sizeof(uint64_t)); + d += 32; + s += 32; + len -= 32; + } + while (len >= sizeof(uint64_t)) { + uint64_t w; + std::memcpy(&w, s, sizeof(uint64_t)); + std::memcpy(d, &w, sizeof(uint64_t)); + d += sizeof(uint64_t); + s += sizeof(uint64_t); + len -= sizeof(uint64_t); + } + while (len--) { + *d++ = *s++; + } + return dest; +} + +/** + * Head-and-tail overlapping copy for small payloads (<= 16 bytes). + * Employs two overlapping loads/stores to avoid tail branches. + */ +inline void *copy_small_overlapping(void *dest, const void *src, size_t len) { + auto *d = static_cast(dest); + const auto *s = static_cast(src); + if (len == 0) { + return dest; + } + if (len == 1) { + *d = *s; + return dest; + } + if (len <= 3) { + uint16_t head, tail; + std::memcpy(&head, s, 2); + std::memcpy(&tail, s + len - 2, 2); + std::memcpy(d, &head, 2); + std::memcpy(d + len - 2, &tail, 2); + return dest; + } + if (len <= 7) { + uint32_t head, tail; + std::memcpy(&head, s, 4); + std::memcpy(&tail, s + len - 4, 4); + std::memcpy(d, &head, 4); + std::memcpy(d + len - 4, &tail, 4); + return dest; + } + if (len <= 16) { + uint64_t head, tail; + std::memcpy(&head, s, 8); + std::memcpy(&tail, s + len - 8, 8); + std::memcpy(d, &head, 8); + std::memcpy(d + len - 8, &tail, 8); + return dest; + } + return copy_words64(dest, src, len); +} + +#if defined(__x86_64__) || defined(_M_X64) + +/** + * x86 Enhanced REP MOVSB (ERMS / FSRM). + * Handled directly by CPU microcode. (Basically moving the loop logic to + * microcode). + */ +inline void *copy_rep_movsb(void *dest, const void *src, size_t len) { + void *orig_dest = dest; + __asm__ volatile("rep movsb" : "+D"(dest), "+S"(src), "+c"(len) : : "memory"); + return orig_dest; +} + +/** + * x86 REP MOVSQ for 8-byte quadwords with REP MOVSB tail. + */ +inline void *copy_rep_movsq(void *dest, const void *src, size_t len) { + void *orig_dest = dest; + size_t qwords = len / 8; + size_t bytes = len % 8; + __asm__ volatile("rep movsq\n\t" + "mov %[bytes], %%rcx\n\t" + "rep movsb" + : "+D"(dest), "+S"(src), "+c"(qwords) + : [bytes] "r"(bytes) + : "memory"); + return orig_dest; +} + +/** + * SSE2 128-bit unaligned vector copy unrolled 4x (64 bytes/iter). SSE2 exists + * on any x86 (AMD included) + */ +inline void *copy_sse2(void *dest, const void *src, size_t len) { + auto *d = static_cast(dest); + const auto *s = static_cast(src); + + while (len >= 64) { + __m128i v0 = _mm_loadu_si128(reinterpret_cast(s)); + __m128i v1 = _mm_loadu_si128(reinterpret_cast(s + 16)); + __m128i v2 = _mm_loadu_si128(reinterpret_cast(s + 32)); + __m128i v3 = _mm_loadu_si128(reinterpret_cast(s + 48)); + _mm_storeu_si128(reinterpret_cast<__m128i *>(d), v0); + _mm_storeu_si128(reinterpret_cast<__m128i *>(d + 16), v1); + _mm_storeu_si128(reinterpret_cast<__m128i *>(d + 32), v2); + _mm_storeu_si128(reinterpret_cast<__m128i *>(d + 48), v3); + d += 64; + s += 64; + len -= 64; + } + while (len >= 16) { + __m128i v = _mm_loadu_si128(reinterpret_cast(s)); + _mm_storeu_si128(reinterpret_cast<__m128i *>(d), v); + d += 16; + s += 16; + len -= 16; + } + while (len--) { + *d++ = *s++; + } + return dest; +} + +/** + * AVX2 256-bit unaligned vector copy unrolled 4x (128 bytes/iter). + */ +#if defined(__GNUC__) || defined(__clang__) +__attribute__((target("avx2"))) +#endif +inline void *copy_avx2(void *dest, const void *src, size_t len) { + auto *d = static_cast(dest); + const auto *s = static_cast(src); + + while (len >= 128) { + __m256i v0 = _mm256_loadu_si256(reinterpret_cast(s)); + __m256i v1 = _mm256_loadu_si256(reinterpret_cast(s + 32)); + __m256i v2 = _mm256_loadu_si256(reinterpret_cast(s + 64)); + __m256i v3 = _mm256_loadu_si256(reinterpret_cast(s + 96)); + _mm256_storeu_si256(reinterpret_cast<__m256i *>(d), v0); + _mm256_storeu_si256(reinterpret_cast<__m256i *>(d + 32), v1); + _mm256_storeu_si256(reinterpret_cast<__m256i *>(d + 64), v2); + _mm256_storeu_si256(reinterpret_cast<__m256i *>(d + 96), v3); + d += 128; + s += 128; + len -= 128; + } + while (len >= 32) { + __m256i v = _mm256_loadu_si256(reinterpret_cast(s)); + _mm256_storeu_si256(reinterpret_cast<__m256i *>(d), v); + d += 32; + s += 32; + len -= 32; + } + while (len--) { + *d++ = *s++; + } + return dest; +} + +/** + * AVX2 Non-temporal streaming stores (VMOVNTDQ). + * Bypasses CPU caches to avoid Read-For-Ownership (RFO) traffic. + * Optimal for large buffers exceeding LLC (L3 cache). + * Aligns destination to 32-byte boundary and issues sfence upon completion. + */ +#if defined(__GNUC__) || defined(__clang__) +__attribute__((target("avx2"))) +#endif +inline void *copy_avx2_stream(void *dest, const void *src, size_t len) { + auto *d = static_cast(dest); + const auto *s = static_cast(src); + + if (len < 128) { + return copy_naive(dest, src, len); + } + + // Align destination to 32 bytes (required by _mm256_stream_si256) + size_t unaligned = (32 - (reinterpret_cast(d) & 31)) & 31; + if (unaligned > 0 && len >= unaligned) { + for (size_t i = 0; i < unaligned; ++i) { + *d++ = *s++; + } + len -= unaligned; + } + + while (len >= 128) { + __m256i v0 = _mm256_loadu_si256(reinterpret_cast(s)); + __m256i v1 = _mm256_loadu_si256(reinterpret_cast(s + 32)); + __m256i v2 = _mm256_loadu_si256(reinterpret_cast(s + 64)); + __m256i v3 = _mm256_loadu_si256(reinterpret_cast(s + 96)); + _mm256_stream_si256(reinterpret_cast<__m256i *>(d), v0); + _mm256_stream_si256(reinterpret_cast<__m256i *>(d + 32), v1); + _mm256_stream_si256(reinterpret_cast<__m256i *>(d + 64), v2); + _mm256_stream_si256(reinterpret_cast<__m256i *>(d + 96), v3); + d += 128; + s += 128; + len -= 128; + } + while (len >= 32) { + __m256i v = _mm256_loadu_si256(reinterpret_cast(s)); + _mm256_stream_si256(reinterpret_cast<__m256i *>(d), v); + d += 32; + s += 32; + len -= 32; + } + while (len--) { + *d++ = *s++; + } + _mm_sfence(); + return dest; +} + +/** + * AVX2 vector copy with software prefetching (PREFETCHT0). + HINT_T0 prefetches up to L1 cache + */ +#if defined(__GNUC__) || defined(__clang__) +__attribute__((target("avx2"))) +#endif +inline void *copy_avx2_prefetch(void *dest, const void *src, size_t len) { + auto *d = static_cast(dest); + const auto *s = static_cast(src); + constexpr size_t kPrefetchDist = 256; + + while (len >= 128) { + _mm_prefetch(reinterpret_cast(s + kPrefetchDist), + _MM_HINT_T0); + _mm_prefetch(reinterpret_cast(s + kPrefetchDist + 64), + _MM_HINT_T0); + __m256i v0 = _mm256_loadu_si256(reinterpret_cast(s)); + __m256i v1 = _mm256_loadu_si256(reinterpret_cast(s + 32)); + __m256i v2 = _mm256_loadu_si256(reinterpret_cast(s + 64)); + __m256i v3 = _mm256_loadu_si256(reinterpret_cast(s + 96)); + _mm256_storeu_si256(reinterpret_cast<__m256i *>(d), v0); + _mm256_storeu_si256(reinterpret_cast<__m256i *>(d + 32), v1); + _mm256_storeu_si256(reinterpret_cast<__m256i *>(d + 64), v2); + _mm256_storeu_si256(reinterpret_cast<__m256i *>(d + 96), v3); + d += 128; + s += 128; + len -= 128; + } + while (len >= 32) { + __m256i v = _mm256_loadu_si256(reinterpret_cast(s)); + _mm256_storeu_si256(reinterpret_cast<__m256i *>(d), v); + d += 32; + s += 32; + len -= 32; + } + while (len--) { + *d++ = *s++; + } + return dest; +} + +#endif // defined(__x86_64__) || defined(_M_X64) + +} // namespace memory_bench + +#endif // MEMCP_BENCH_MEMORY_COPY_COPY_H \ No newline at end of file diff --git a/lib/memory/copy/copy_test.cpp b/lib/memory/copy/copy_test.cpp new file mode 100644 index 0000000..92c3e61 --- /dev/null +++ b/lib/memory/copy/copy_test.cpp @@ -0,0 +1,87 @@ +#include "lib/memory/copy/copy.h" + +#include +#include +#include + +#include "gtest/gtest.h" + +namespace memory_bench { +namespace { + +using CopyFn = void* (*)(void*, const void*, std::size_t); + +class CopyImplementationsTest : public ::testing::TestWithParam> {}; + +TEST_P(CopyImplementationsTest, CopiesCorrectlyAcrossSizesAndAlignments) { + auto [name, copy_func] = GetParam(); + const std::vector test_sizes = { + 0, 1, 2, 3, 4, 7, 8, 9, 15, 16, 17, 31, 32, 33, 63, 64, 65, 127, 128, 129, 255, 256, 1024, 4096}; + + for (std::size_t size : test_sizes) { + for (std::size_t offset_src : {0, 1, 7, 15, 31}) { + for (std::size_t offset_dst : {0, 1, 7, 15, 31}) { + std::vector src(size + 64); + std::vector dst(size + 64, 0xEE); + + for (std::size_t i = 0; i < src.size(); ++i) { + src[i] = static_cast((i * 109 + 23) & 0xFF); + } + + void* ret = copy_func(dst.data() + offset_dst, src.data() + offset_src, size); + EXPECT_EQ(ret, dst.data() + offset_dst) << "Failed return pointer for " << name; + + if (size > 0) { + std::vector expected(src.begin() + offset_src, src.begin() + offset_src + size); + std::vector actual(dst.begin() + offset_dst, dst.begin() + offset_dst + size); + EXPECT_EQ(actual, expected) + << "Mismatch for " << name << " size=" << size + << " off_src=" << offset_src << " off_dst=" << offset_dst; + } + + // Verify guard bytes before and after dst are intact + for (std::size_t i = 0; i < offset_dst; ++i) { + EXPECT_EQ(dst[i], 0xEE) << "Underflow guard overwrite for " << name; + } + for (std::size_t i = offset_dst + size; i < dst.size(); ++i) { + EXPECT_EQ(dst[i], 0xEE) << "Overflow guard overwrite for " << name; + } + } + } + } +} + +const std::pair kImplementations[] = { + {"copy_memcpy", copy_memcpy}, + {"copy_naive", [](void* d, const void* s, std::size_t l) { return copy_naive(d, s, l); }}, + {"copy_duff", copy_duff}, + {"copy_words64", copy_words64}, + {"copy_words64_unrolled4", copy_words64_unrolled4}, + {"copy_small_overlapping", copy_small_overlapping}, +#if defined(__x86_64__) || defined(_M_X64) + {"copy_rep_movsb", copy_rep_movsb}, + {"copy_rep_movsq", copy_rep_movsq}, + {"copy_sse2", copy_sse2}, + {"copy_avx2", copy_avx2}, + {"copy_avx2_stream", copy_avx2_stream}, + {"copy_avx2_prefetch", copy_avx2_prefetch}, +#endif +}; + +INSTANTIATE_TEST_SUITE_P(CopyTests, CopyImplementationsTest, ::testing::ValuesIn(kImplementations)); + +TEST(CompileTimeCopyTest, TemplateNaiveAndFixed) { + constexpr std::size_t kSize = 16; + std::uint8_t src[kSize] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; + std::uint8_t dst1[kSize] = {0}; + std::uint8_t dst2[kSize] = {0}; + + copy_naive(dst1, src); + EXPECT_EQ(std::memcmp(dst1, src, kSize), 0); + + copy_fixed(dst2, src); + EXPECT_EQ(std::memcmp(dst2, src, kSize), 0); +} + +} // namespace +} // namespace memory_bench