diff --git a/README.md b/README.md index 6b4f89d..8bfb782 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ what exists is one vertical slice and it is not finished: | Lifting CIR calls into the dialect | works on hand-written IR; the C++ path is not yet exercised end to end | | Queue ownership analysis: SPSC / MPSC / SPMC / MPMC | prints a verdict; sound for direct calls, UNKNOWN wherever a function's address escapes; thread lifetime and escaping queues not modelled | | Lowering back to CIR | faithful round trip; crashes on operations that carry no source-type attribute | -| Specialised implementations | none yet | +| Specialised implementations | an unbounded SPSC queue, substituted for a global `wtc::queue` proven SPSC; 3.6x throughput on one producer and one consumer (`benchmarks/`) | `docs/audit.md` lists the known defects in full. Do not build anything on top of the analysis verdict until they are closed. diff --git a/benchmarks/README.md b/benchmarks/README.md index a00f10b..fac59e4 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -14,4 +14,36 @@ for, and the hand-written specialised implementation from `runtime/`. The second one is the ceiling: the compiler cannot beat code a human wrote for this exact case, it can only reach it without the human. -Nothing here yet. +## What is here + +- `queue_bench.cpp`: one producer and one consumer move 5M ints through a + global `wtc::queue`. The consumer checks the count, the sum and the + order, and the run fails if any is wrong. Nothing in it names the + substitute: the shim makes `wtc::runtime::spsc_queue` available. +- `spsc_direct.cpp`: the same workload written by hand against + `runtime/spsc_queue.hpp`, with its allocation-free pop. This is the ceiling. +- `run.sh`: builds `queue_bench.cpp` twice from the same CIR, once as + written and once with `--select-queue-impl` in the pipeline, plus + `spsc_direct`, through the pinned toolchain down to a native binary, and + reports the median of `RUNS` runs of each. Set `LLVM_BUILD` to the pinned + toolchain (default `/opt/llvm`, as in the image) and `WTC_BUILD` to this + project's build tree. + +Only throughput is measured so far, not the tail, and runs are not pinned. + +## Measured + +At the pin (`1ebf89d8c5e6`), 2026-10-18, on an Apple M1 Pro (8 cores, +16 GB), macOS, threads not pinned, five runs each, median: + +| variant | items/s | vs baseline | +| --- | ---: | ---: | +| baseline: `wtc::queue` | 2.07M | 1.0x | +| optimized: substituted by `--select-queue-impl` | 7.44M | 3.6x | +| direct: `spsc_queue` by hand | 43.1M | 20.9x | + +The gap between optimized and direct is almost all one allocation per item. +The substitute has to keep `wtc::queue::try_pop`'s interface, which +returns a `std::shared_ptr`, so every pop builds one; the hand-written +consumer uses `try_pop_optional` and allocates nothing. Closing it means +changing what the dialect's pop returns, not the queue. diff --git a/benchmarks/queue_bench.cpp b/benchmarks/queue_bench.cpp new file mode 100644 index 0000000..12d437b --- /dev/null +++ b/benchmarks/queue_bench.cpp @@ -0,0 +1,66 @@ +// One producer, one consumer, kNumItems ints through a global +// wtc::queue. Built twice from this one source by run.sh: once with the +// queue as written, once after --select-queue-impl has substituted +// wtc::runtime::spsc_queue for it. Nothing here names the substitute; +// the shim makes it available. +// +// The consumer checks what it receives: a substitution that drops, +// duplicates or reorders an item fails the run instead of timing it. + +#include "queue.hpp" +#include "thread.hpp" + +#include +#include + +constexpr long kNumItems = 5'000'000; + +wtc::queue g_queue; + +long g_received = 0; +long g_sum = 0; +long g_out_of_order = 0; + +void producer() { + for (long i = 0; i < kNumItems; ++i) { + int value = static_cast(i); + g_queue.push(value); + } +} + +void consumer() { + long expected = 0; + while (g_received < kNumItems) { + auto value = g_queue.try_pop(); + if (value) { + if (*value != expected) + ++g_out_of_order; + g_sum += *value; + ++expected; + ++g_received; + } + } +} + +int main() { + auto start = std::chrono::steady_clock::now(); + + auto producer_thread = wtc::spawn(producer); + auto consumer_thread = wtc::spawn(consumer); + producer_thread.join(); + consumer_thread.join(); + + auto end = std::chrono::steady_clock::now(); + double seconds = std::chrono::duration(end - start).count(); + + long expected_sum = kNumItems * (kNumItems - 1) / 2; + if (g_received != kNumItems || g_sum != expected_sum || g_out_of_order != 0) { + std::printf("FAIL received=%ld sum=%ld (expected %ld) out_of_order=%ld\n", + g_received, g_sum, expected_sum, g_out_of_order); + return 1; + } + + std::printf("items=%ld elapsed_ms=%.3f items_per_sec=%.0f\n", + kNumItems, seconds * 1000.0, static_cast(kNumItems) / seconds); + return 0; +} diff --git a/benchmarks/run.sh b/benchmarks/run.sh new file mode 100755 index 0000000..38c5e42 --- /dev/null +++ b/benchmarks/run.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# Builds the queue benchmark two ways from the same source -- baseline +# (wtc::queue, lifted and lowered back unchanged) and optimized (the +# same pipeline plus --select-queue-impl) -- and spsc_direct, the +# hand-written ceiling. Runs each RUNS times and reports the median. +# +# LLVM_BUILD the pinned toolchain, built or installed; default /opt/llvm, +# where the toolchain image puts it +# WTC_BUILD this project's build tree; default /build +# RUNS runs per variant; default 5 +set -euo pipefail + +here="$(cd "$(dirname "$0")" && pwd)" +repo="$(dirname "$here")" +LLVM_BUILD="${LLVM_BUILD:-/opt/llvm}" +WTC_BUILD="${WTC_BUILD:-$repo/build}" +RUNS="${RUNS:-5}" +out="$WTC_BUILD/benchmarks" +mkdir -p "$out" + +for tool in clang cir-opt mlir-translate llc; do + if [ ! -x "$LLVM_BUILD/bin/$tool" ]; then + echo "no $tool in $LLVM_BUILD/bin; set LLVM_BUILD to the pinned toolchain" >&2 + exit 1 + fi +done +wtc_opt="$WTC_BUILD/bin/wtc-opt" +[ -x "$wtc_opt" ] || { echo "no wtc-opt in $WTC_BUILD/bin; set WTC_BUILD" >&2; exit 1; } + +# A clang built from source does not know where macOS keeps the SDK, and the +# final link needs the system's C++ runtime either way. +sysroot=() +link_cxx=(c++) +if [ "$(uname)" = Darwin ]; then + sysroot=(-isysroot "$(xcrun --show-sdk-path)") + link_cxx=(xcrun clang++) +fi + +emit_cir() { # + "$LLVM_BUILD/bin/clang" -x c++ -std=c++17 -O1 -I "$repo/shim" "${sysroot[@]}" \ + -fclangir -emit-cir "$1" -o "$2" +} + +# cir-to-llvm leaves CIR types behind in attributes, even for plain clang +# output, so mlir-translate has to be told not to choke on them. +link() { # + "$LLVM_BUILD/bin/cir-opt" "$1" --cir-to-llvm -o "$out/$2.llvm.mlir" + "$LLVM_BUILD/bin/mlir-translate" --allow-unregistered-dialect --mlir-to-llvmir \ + "$out/$2.llvm.mlir" -o "$out/$2.ll" + "$LLVM_BUILD/bin/llc" -O2 -filetype=obj "$out/$2.ll" -o "$out/$2.o" + "${link_cxx[@]}" "$out/$2.o" -o "$out/$2" -lpthread +} + +emit_cir "$here/queue_bench.cpp" "$out/queue_bench.cir" + +"$wtc_opt" "$out/queue_bench.cir" --mlir-print-op-on-diagnostic=false \ + --lift-cir-to-wtc --queue-ownership-analysis --lower-wtc-to-cir \ + --reconcile-unrealized-casts -o "$out/queue_bench_baseline.cir" > /dev/null +link "$out/queue_bench_baseline.cir" queue_bench_baseline + +"$wtc_opt" "$out/queue_bench.cir" --mlir-print-op-on-diagnostic=false \ + --lift-cir-to-wtc --queue-ownership-analysis --select-queue-impl --lower-wtc-to-cir \ + --reconcile-unrealized-casts -o "$out/queue_bench_optimized.cir" > /dev/null +# Linkage keywords differ by target (ELF globals are dso_local), so match +# only the name and the type. +grep -qE 'cir\.global .*@g_queue = #cir\.zero : !rec_wtc3A3Aruntime' "$out/queue_bench_optimized.cir" \ + || { echo "--select-queue-impl did not substitute g_queue" >&2; exit 1; } +link "$out/queue_bench_optimized.cir" queue_bench_optimized + +emit_cir "$here/spsc_direct.cpp" "$out/spsc_direct.cir" +link "$out/spsc_direct.cir" spsc_direct + +for bin in queue_bench_baseline queue_bench_optimized spsc_direct; do + rates=() + for _ in $(seq "$RUNS"); do + line="$("$out/$bin")" + rates+=("$(sed -n 's/.*items_per_sec=\([0-9]*\).*/\1/p' <<<"$line")") + done + median="$(printf '%s\n' "${rates[@]}" | sort -n | sed -n "$(( (RUNS + 1) / 2 ))p")" + echo "$bin: median $median items/s over $RUNS runs (${rates[*]})" +done diff --git a/benchmarks/spsc_direct.cpp b/benchmarks/spsc_direct.cpp new file mode 100644 index 0000000..5a5e3d2 --- /dev/null +++ b/benchmarks/spsc_direct.cpp @@ -0,0 +1,61 @@ +// The same workload as queue_bench.cpp, written by hand against the +// substitute with its allocation-free pop. This is the ceiling: the +// compiler cannot beat it, only reach it without the human. + +#include "../runtime/spsc_queue.hpp" +#include "thread.hpp" + +#include +#include + +constexpr long kNumItems = 5'000'000; + +wtc::runtime::spsc_queue g_queue; + +long g_received = 0; +long g_sum = 0; +long g_out_of_order = 0; + +void producer() { + for (long i = 0; i < kNumItems; ++i) { + int value = static_cast(i); + g_queue.push(value); + } +} + +void consumer() { + long expected = 0; + while (g_received < kNumItems) { + auto value = g_queue.try_pop_optional(); + if (value) { + if (*value != expected) + ++g_out_of_order; + g_sum += *value; + ++expected; + ++g_received; + } + } +} + +int main() { + auto start = std::chrono::steady_clock::now(); + + auto producer_thread = wtc::spawn(producer); + auto consumer_thread = wtc::spawn(consumer); + producer_thread.join(); + consumer_thread.join(); + + auto end = std::chrono::steady_clock::now(); + double seconds = std::chrono::duration(end - start).count(); + + long expected_sum = kNumItems * (kNumItems - 1) / 2; + if (g_received != kNumItems || g_sum != expected_sum || g_out_of_order != 0) { + std::printf("FAIL received=%ld sum=%ld (expected %ld) out_of_order=%ld\n", + g_received, g_sum, expected_sum, g_out_of_order); + return 1; + } + + std::printf("items=%ld elapsed_ms=%.3f items_per_sec=%.0f\n", + kNumItems, seconds * 1000.0, static_cast(kNumItems) / seconds); + return 0; +} diff --git a/docs/audit.md b/docs/audit.md index 15335a2..57e5d45 100644 --- a/docs/audit.md +++ b/docs/audit.md @@ -191,6 +191,32 @@ this, and treat a call with no result as a case to handle rather than one to assume away. None of that is attempted in this document; it is the work, not the diagnosis. +## The first specialisation: SPSC queues — October 2026 + +`--select-queue-impl` substitutes `runtime/spsc_queue.hpp` for a global +`wtc::queue` whose verdict is SPSC. Built from 253a2bc, with these +changes measured or found on the way: + +- **A shared destructor was retargeted for every queue.** Two globals of the + same type share a destructor, and the earlier version rewrote every + reference to it, so substituting one queue registered the substitute's + destructor for the other too. Retargeting now follows the substituted + global's own `__cxa_atexit` registration + (`test/Transforms/SelectQueueImplementation/global-spsc.mlir`). +- **Any other use of the global was left with the wrong type.** The global's + type changes, so a use the pass does not retarget would read the new + storage as the old type. Such a global is now not substituted, and the + remark names the use (`refusals.mlir`). +- **The substitute was bounded and dropped items when full.** It is now + unbounded, like the queue it replaces. +- **The program had to name the substitute.** The shim now makes it + available, through an anchor the pass erases. +- **Alignment.** The data layout of a CIR record type does not see `alignas`; + `cir.record_layouts` does, and is what the pass uses. + +Only globals are substituted. A queue on the stack or reached only through +parameters keeps its implementation. + ## Infrastructure Fixed before the move into this repository: diff --git a/mlir/include/WTC/Transforms/Passes.h b/mlir/include/WTC/Transforms/Passes.h index feebfbf..26c9da6 100644 --- a/mlir/include/WTC/Transforms/Passes.h +++ b/mlir/include/WTC/Transforms/Passes.h @@ -6,6 +6,7 @@ namespace mlir::wtc { void registerLiftCIRToWTCPass(); void registerLowerWTCToCIRPass(); void registerQueueOwnershipAnalysisPass(); + void registerSelectQueueImplementationPass(); void analyseQueues(ModuleOp module); } diff --git a/mlir/include/WTC/Transforms/Utils.h b/mlir/include/WTC/Transforms/Utils.h index da66379..0f5a5eb 100644 --- a/mlir/include/WTC/Transforms/Utils.h +++ b/mlir/include/WTC/Transforms/Utils.h @@ -19,6 +19,12 @@ namespace mlir::wtc { bool isWTCAnnotatedCall(cir::CallOp op, StringRef annotation); + bool funcHasAnnotation(cir::FuncOp funcOp, StringRef annotation); + + // The element type of a queue whose push is pushFunc: the pointee of the + // reference it takes the value by. + Type deriveQueueElementTypeFromPush(cir::FuncOp pushFunc); + SemanticOpKind getSemanticKind(cir::FuncOp funcOp); // The argument through which funcOp returns its value indirectly, if it diff --git a/mlir/lib/WTC/Transforms/CMakeLists.txt b/mlir/lib/WTC/Transforms/CMakeLists.txt index 06c2e26..70ac46d 100644 --- a/mlir/lib/WTC/Transforms/CMakeLists.txt +++ b/mlir/lib/WTC/Transforms/CMakeLists.txt @@ -2,6 +2,7 @@ add_mlir_library(MLIRWTCTransforms LiftCIRToWTC.cpp LowerWTCToCIR.cpp QueueOwnershipAnalysis.cpp + SelectQueueImplementation.cpp Utils.cpp LINK_LIBS PUBLIC MLIRWTC @@ -11,6 +12,7 @@ add_mlir_library(MLIRWTCTransforms # wtc-opt linked MLIRCIR itself; a second consumer would not be so lucky. MLIRCIR MLIRCIRInterfaces + MLIRDataLayoutInterfaces MLIRIR MLIRPass diff --git a/mlir/lib/WTC/Transforms/SelectQueueImplementation.cpp b/mlir/lib/WTC/Transforms/SelectQueueImplementation.cpp new file mode 100644 index 0000000..e8e7d50 --- /dev/null +++ b/mlir/lib/WTC/Transforms/SelectQueueImplementation.cpp @@ -0,0 +1,367 @@ +#include "WTC/WTCOps.h" +#include "WTC/Transforms/Passes.h" +#include "WTC/Transforms/Utils.h" + +#include "clang/CIR/Dialect/IR/CIRDialect.h" +#include "clang/CIR/Dialect/IR/CIRAttrs.h" + +#include "mlir/Interfaces/DataLayoutInterfaces.h" +#include "mlir/IR/BuiltinOps.h" +#include "mlir/Pass/Pass.h" + +#include "llvm/ADT/SmallVector.h" + +using namespace mlir; +using namespace mlir::wtc; + +// Substitutes the storage of a global wtc::queue proven SPSC with +// runtime/spsc_queue.hpp's implementation for the same T. +// +// The substitute is found in the module, not synthesised: the shim's queue +// constructor odr-uses an anchor (annotated wtc_anchor) that instantiates +// wtc::runtime::spsc_queue, so every module using queue also carries +// spsc_queue's constructor, push, try_pop and destructor, each marked +// wtc_impl_spsc_*. The anchor is erased afterwards. +// +// Only globals are substituted, and only when every use of the global is +// one this pass knows how to retarget: the queue's own push and pop, its +// construction, and its destructor's registration with __cxa_atexit. A +// global that is used any other way keeps its original type, since that +// use would be left calling the original implementation on the new +// storage. + +namespace { + +constexpr StringRef kSpscPrefix = "wtc_impl_spsc_"; + +struct AltImpl { + cir::RecordType recordType; + Type elementType; + FlatSymbolRefAttr construct; + FlatSymbolRefAttr push; + FlatSymbolRefAttr tryPop; + FlatSymbolRefAttr dtor; + + bool complete() const { return elementType && construct && push && tryPop && dtor; } +}; + +// What one global's uses amount to, if every one of them can be retargeted. +struct GlobalUses { + llvm::SmallVector getGlobals; + llvm::SmallVector queueOps; + llvm::SmallVector constructions; + // The cir.get_global of the destructor each registration passes. + llvm::SmallVector dtorRefs; +}; + +cir::RecordType recordOfThis(cir::FuncOp funcOp) { + if (funcOp.getNumArguments() < 1) + return {}; + // Where try_pop returns indirectly, `this` follows the result slot. + unsigned self = sretArgIndex(funcOp) == 0u ? 1 : 0; + if (funcOp.getNumArguments() <= self) + return {}; + // From the signature, not the entry block: a declaration has none. + auto thisPtrType = mlir::dyn_cast(funcOp.getFunctionType().getInput(self)); + if (!thisPtrType) + return {}; + return mlir::dyn_cast(thisPtrType.getPointee()); +} + +Value stripCasts(Value v) { + while (auto castOp = v.getDefiningOp()) + v = castOp.getSrc(); + return v; +} + +} + +struct SelectQueueImplementationPass + : public PassWrapper> { + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(SelectQueueImplementationPass) + + StringRef getArgument() const final { + return "select-queue-impl"; + } + StringRef getDescription() const final { + return "Substitute wtc::runtime::spsc_queue for a global wtc::queue proven SPSC"; + } + + void getDependentDialects(DialectRegistry ®istry) const override { + registry.insert(); + registry.insert(); + } + + void runOnOperation() override { + ModuleOp module = getOperation(); + llvm::SmallVector altImpls = findAltImpls(module); + + for (cir::GlobalOp globalOp : llvm::to_vector(module.getOps())) { + auto kind = globalOp->getAttrOfType(kQueueKindAttrName); + if (!kind) + continue; + if (kind.getValue() != "SPSC") { + globalOp.emitRemark("not substituted: the queue is ") << kind.getValue(); + continue; + } + selectForGlobal(module, globalOp, altImpls); + } + + eraseAnchors(module); + } + + llvm::SmallVector findAltImpls(ModuleOp module) { + llvm::SmallVector impls; + auto implFor = [&](cir::RecordType recordType) -> AltImpl & { + for (AltImpl &impl : impls) + if (impl.recordType == recordType) + return impl; + impls.push_back({}); + impls.back().recordType = recordType; + return impls.back(); + }; + + // The complete and base constructors (C1, C2), and destructors (D1, + // D2), are interchangeable for a class without virtual bases, which + // spsc_queue is; the first one found is used. + module.walk([&](cir::FuncOp funcOp) { + auto sym = FlatSymbolRefAttr::get(funcOp.getSymNameAttr()); + auto setOnce = [&](FlatSymbolRefAttr &slot) { if (!slot) slot = sym; }; + + if (funcHasAnnotation(funcOp, (kSpscPrefix + "construction").str())) { + if (auto recordType = recordOfThis(funcOp)) + setOnce(implFor(recordType).construct); + } else if (funcHasAnnotation(funcOp, (kSpscPrefix + "dtor").str())) { + if (auto recordType = recordOfThis(funcOp)) + setOnce(implFor(recordType).dtor); + } else if (funcHasAnnotation(funcOp, (kSpscPrefix + "push").str())) { + if (auto recordType = recordOfThis(funcOp)) { + AltImpl &impl = implFor(recordType); + setOnce(impl.push); + impl.elementType = deriveQueueElementTypeFromPush(funcOp); + } + } else if (funcHasAnnotation(funcOp, (kSpscPrefix + "try_pop").str())) { + if (auto recordType = recordOfThis(funcOp)) + setOnce(implFor(recordType).tryPop); + } + }); + + llvm::erase_if(impls, [](const AltImpl &impl) { return !impl.complete(); }); + return impls; + } + + // Every use of the global, sorted into the kinds this pass can + // retarget; std::nullopt, with a remark, at the first it cannot. + std::optional collectUses(ModuleOp module, cir::GlobalOp globalOp) { + GlobalUses result; + auto refuse = [&](Operation *user, StringRef why) -> std::optional { + globalOp.emitRemark("not substituted: ") << why; + user->emitRemark("this use of @") << globalOp.getSymName() << " is the one"; + return std::nullopt; + }; + + auto uses = SymbolTable::getSymbolUses(globalOp.getOperation(), module.getOperation()); + if (!uses) + return result; + + for (const SymbolTable::SymbolUse &use : *uses) { + auto getGlobalOp = mlir::dyn_cast(use.getUser()); + if (!getGlobalOp) + return refuse(use.getUser(), "its address is used other than through cir.get_global"); + result.getGlobals.push_back(getGlobalOp); + + for (Operation *user : getGlobalOp.getResult().getUsers()) { + if (auto castOp = mlir::dyn_cast(user)) { + if (castOp.getNumResults() != 1 || !mlir::isa(castOp.getResult(0).getType())) + return refuse(user, "it is cast to something other than a wtc queue"); + for (Operation *queueUser : castOp.getResult(0).getUsers()) { + if (!mlir::isa(queueUser)) + return refuse(queueUser, "the lifted queue is used by something other than push and pop"); + auto opKind = queueUser->getAttrOfType(kQueueKindAttrName); + if (!opKind || opKind.getValue() != "SPSC") + return refuse(queueUser, "an op on it can also reach a queue that is not SPSC"); + result.queueOps.push_back(queueUser); + } + continue; + } + + auto callOp = mlir::dyn_cast(user); + if (callOp && isWTCAnnotatedCall(callOp, "wtc_queue_construction") && + !callOp.getArgOperands().empty() && callOp.getArgOperands()[0] == getGlobalOp.getResult()) { + result.constructions.push_back(callOp); + continue; + } + + // __cxa_atexit(dtor, &global, dso_handle), both pointers cast + // to their void forms first. + auto castOp = mlir::dyn_cast(user); + if (castOp && castOp.getResult().hasOneUse()) { + auto atexit = mlir::dyn_cast(*castOp.getResult().getUsers().begin()); + auto callee = atexit ? atexit.getCalleeAttr() : FlatSymbolRefAttr(); + if (callee && callee.getValue() == "__cxa_atexit" && + atexit.getArgOperands().size() == 3 && + atexit.getArgOperands()[1] == castOp.getResult()) { + auto dtorRef = stripCasts(atexit.getArgOperands()[0]).getDefiningOp(); + if (!dtorRef) + return refuse(atexit, "its destructor registration names no function"); + // The reference is changed in place, so it must + // serve this registration and nothing else. + Value v = dtorRef.getResult(); + while (v.hasOneUse() && mlir::isa(*v.getUsers().begin())) + v = v.getUsers().begin()->getResult(0); + if (!dtorRef.getResult().hasOneUse() || !v.hasOneUse()) + return refuse(atexit, "its destructor reference is shared with another use"); + result.dtorRefs.push_back(dtorRef); + continue; + } + } + + return refuse(user, "it has a use this pass cannot retarget"); + } + } + return result; + } + + void selectForGlobal(ModuleOp module, cir::GlobalOp globalOp, ArrayRef altImpls) { + std::optional uses = collectUses(module, globalOp); + if (!uses) + return; + if (uses->queueOps.empty()) + return; + + auto queueType = mlir::cast( + mlir::isa(uses->queueOps.front()) + ? mlir::cast(uses->queueOps.front()).getQueue().getType() + : mlir::cast(uses->queueOps.front()).getQueue().getType()); + + const AltImpl *chosen = nullptr; + for (const AltImpl &impl : altImpls) + if (impl.elementType == queueType.getElementType()) + chosen = &impl; + if (!chosen) { + globalOp.emitRemark("not substituted: no substitute implementation for element type ") + << queueType.getElementType() << " is instantiated in this module"; + return; + } + + auto newPtrType = cir::PointerType::get(chosen->recordType); + auto newDtor = SymbolTable::lookupNearestSymbolFrom(globalOp, chosen->dtor); + auto newTryPop = SymbolTable::lookupNearestSymbolFrom(globalOp, chosen->tryPop); + + // A pop lifted with a result slot has to call a try_pop that takes + // one, and one lifted with a result a try_pop that returns it. Both + // implementations are compiled for the same target, so they agree; + // check rather than assume. + bool substituteReturnsIndirectly = sretArgIndex(newTryPop).has_value(); + for (Operation *op : uses->queueOps) { + auto popOp = mlir::dyn_cast(op); + if (popOp && static_cast(popOp.getResultSlot()) != substituteReturnsIndirectly) { + globalOp.emitRemark("not substituted: the substitute's try_pop returns its value differently"); + return; + } + } + + globalOp.setSymTypeAttr(TypeAttr::get(chosen->recordType)); + globalOp.setInitialValueAttr(cir::ZeroAttr::get(chosen->recordType)); + // The record's own alignment, alignas included, is what ClangIR + // records in cir.record_layouts; the data layout of the record type + // does not see alignas, and spsc_queue aligns its cursors to 64. + uint64_t align = 0; + if (auto layouts = module->getAttrOfType("cir.record_layouts")) + if (auto layout = mlir::dyn_cast_or_null( + layouts.get(chosen->recordType.getName().getValue()))) + align = layout.getRecordAlign(); + if (!align) + align = DataLayout::closest(globalOp).getTypeABIAlignment(chosen->recordType); + globalOp.setAlignmentAttr(IntegerAttr::get(IntegerType::get(&getContext(), 64), align)); + + for (cir::GetGlobalOp getGlobalOp : uses->getGlobals) + getGlobalOp.getResult().setType(newPtrType); + + for (Operation *op : uses->queueOps) { + if (auto pushOp = mlir::dyn_cast(op)) { + pushOp.setFallbackImplAttr(chosen->push); + pushOp.setSourceQueueTypeAttr(TypeAttr::get(newPtrType)); + } else { + auto popOp = mlir::cast(op); + popOp.setFallbackImplAttr(chosen->tryPop); + popOp.setSourceQueueTypeAttr(TypeAttr::get(newPtrType)); + } + } + + auto newCtor = SymbolTable::lookupNearestSymbolFrom(globalOp, chosen->construct); + for (cir::CallOp callOp : uses->constructions) { + callOp.setCalleeAttr(chosen->construct); + copyArgAttrsFromCallee(callOp, newCtor); + } + + for (cir::GetGlobalOp dtorRef : uses->dtorRefs) { + dtorRef->setAttr("name", chosen->dtor); + dtorRef.getResult().setType(cir::PointerType::get(newDtor.getFunctionType())); + } + + globalOp.emitRemark("substituted ") << chosen->recordType.getName(); + } + + // [[gnu::used]] puts each anchor in @llvm.used, which is what makes the + // compiler emit it; take it out again, dropping @llvm.used if nothing + // else is left in it. + void dropFromLlvmUsed(ModuleOp module, StringRef symbol) { + auto used = module.lookupSymbol("llvm.used"); + if (!used) + return; + auto array = mlir::dyn_cast_or_null(used.getInitialValueAttr()); + auto elements = array ? mlir::dyn_cast(array.getElts()) : ArrayAttr(); + if (!elements) + return; + + llvm::SmallVector kept; + for (Attribute element : elements) { + auto view = mlir::dyn_cast(element); + if (!view || view.getSymbol().getValue() != symbol) + kept.push_back(element); + } + if (kept.size() == elements.size()) + return; + if (kept.empty()) { + used.erase(); + return; + } + + auto oldType = mlir::cast(array.getType()); + auto newType = cir::ArrayType::get(oldType.getElementType(), kept.size()); + used.setSymTypeAttr(TypeAttr::get(newType)); + used.setInitialValueAttr(cir::ConstArrayAttr::get(newType, ArrayAttr::get(&getContext(), kept))); + } + + // The anchors exist only to make the substitutes' functions appear in + // the module. Nothing calls them; @llvm.used keeps them alive, and the + // shim's constructor may take an address it then discards. + void eraseAnchors(ModuleOp module) { + for (cir::FuncOp funcOp : llvm::to_vector(module.getOps())) { + if (!funcHasAnnotation(funcOp, "wtc_anchor")) + continue; + + dropFromLlvmUsed(module, funcOp.getSymName()); + + if (auto uses = SymbolTable::getSymbolUses(funcOp.getOperation(), module.getOperation())) { + for (const SymbolTable::SymbolUse &use : llvm::make_early_inc_range(*uses)) { + auto getGlobalOp = mlir::dyn_cast(use.getUser()); + if (getGlobalOp && getGlobalOp.getResult().use_empty()) + getGlobalOp.erase(); + } + } + + if (SymbolTable::symbolKnownUseEmpty(funcOp.getOperation(), module.getOperation())) + funcOp.erase(); + else + funcOp.emitRemark("anchor kept: something other than a discarded address refers to it"); + } + } +}; + +namespace mlir::wtc { + void registerSelectQueueImplementationPass() { + PassRegistration(); + } +} diff --git a/mlir/lib/WTC/Transforms/Utils.cpp b/mlir/lib/WTC/Transforms/Utils.cpp index b33840e..f7ed81a 100644 --- a/mlir/lib/WTC/Transforms/Utils.cpp +++ b/mlir/lib/WTC/Transforms/Utils.cpp @@ -31,6 +31,30 @@ namespace mlir::wtc { return false; } + bool funcHasAnnotation(cir::FuncOp funcOp, StringRef annotation) { + if (!funcOp) + return false; + auto annotations = funcOp.getAnnotationsAttr(); + if (!annotations) + return false; + for (auto attr : annotations) { + auto annotAttr = mlir::dyn_cast(attr); + if (annotAttr && annotAttr.getName().getValue() == annotation) + return true; + } + return false; + } + + Type deriveQueueElementTypeFromPush(cir::FuncOp pushFunc) { + if (!pushFunc || pushFunc.getNumArguments() < 2) + return {}; + // From the signature, not the entry block: a declaration has none. + auto valuePtrType = mlir::dyn_cast(pushFunc.getFunctionType().getInput(1)); + if (!valuePtrType) + return {}; + return valuePtrType.getPointee(); + } + SemanticOpKind getSemanticKind(cir::FuncOp funcOp) { if (!funcOp) return SemanticOpKind::Unknown; diff --git a/mlir/tools/wtc-opt/wtc-opt.cpp b/mlir/tools/wtc-opt/wtc-opt.cpp index e627059..25e2823 100644 --- a/mlir/tools/wtc-opt/wtc-opt.cpp +++ b/mlir/tools/wtc-opt/wtc-opt.cpp @@ -20,6 +20,7 @@ int main(int argc, char **argv) { mlir::wtc::registerLiftCIRToWTCPass(); mlir::wtc::registerLowerWTCToCIRPass(); mlir::wtc::registerQueueOwnershipAnalysisPass(); + mlir::wtc::registerSelectQueueImplementationPass(); return mlir::asMainReturnCode( mlir::MlirOptMain(argc, argv, "wtc-opt: the Well-tempered Compiler optimizer driver\n", registry)); diff --git a/runtime/spsc_queue.hpp b/runtime/spsc_queue.hpp index 3079936..ab915e1 100644 --- a/runtime/spsc_queue.hpp +++ b/runtime/spsc_queue.hpp @@ -23,8 +23,11 @@ // once it has drained it and seen the next one; publishing `next` is the // producer's last access to a block, so nothing touches a freed block. // -// Measured numbers: not yet. See benchmarks/README.md once they are taken at -// the pinned toolchain. +// Measured, against wtc::queue on one producer and one consumer, at the +// pin on an Apple M1 Pro (benchmarks/README.md): 3.6 times the throughput +// when substituted by the compiler, which keeps try_pop's shared_ptr and so +// allocates once per item; 20.9 times when used by hand through +// try_pop_optional, which does not. #include #include diff --git a/shim/queue.hpp b/shim/queue.hpp index 2faafcc..6df3138 100644 --- a/shim/queue.hpp +++ b/shim/queue.hpp @@ -3,8 +3,27 @@ #include #include +#include "../runtime/spsc_queue.hpp" + namespace wtc { +namespace detail { + +// Never called. Its only job is to make the compiler emit, in every module +// that uses queue, the implementations --select-queue-impl may substitute +// for it, so that a program never has to name them. queue's constructor +// takes its address, which is enough for it, and through it spsc_queue, +// to be instantiated and emitted. The pass erases it afterwards. +template +[[gnu::used, clang::annotate("wtc_anchor")]] +void alternatives(const T &value) { + runtime::spsc_queue spsc; + spsc.push(value); + (void)spsc.try_pop(); +} + +} + template class queue { private: @@ -35,7 +54,9 @@ class queue { public: [[clang::annotate("wtc_queue_construction")]] - queue() : head(new node), tail(head.get()) {} + queue() : head(new node), tail(head.get()) { + (void)&detail::alternatives; + } queue(const queue&) = delete; queue& operator=(const queue&) = delete; diff --git a/test/Transforms/SelectQueueImplementation/global-spsc.mlir b/test/Transforms/SelectQueueImplementation/global-spsc.mlir new file mode 100644 index 0000000..84a44f5 --- /dev/null +++ b/test/Transforms/SelectQueueImplementation/global-spsc.mlir @@ -0,0 +1,114 @@ +// RUN: wtc-opt %s --select-queue-impl --mlir-print-op-on-diagnostic=false 2>&1 | FileCheck %s + +// Two globals of the same type, wtc::queue, reduced from real output. +// @q1 is SPSC and @q2 is MPSC (the verdicts are written here directly, as +// --queue-ownership-analysis would). @q1's storage, construction, push, +// pop and destructor registration are retargeted to spsc_queue; @q2 +// keeps every one of them. The two share a destructor, so retargeting must +// follow @q1's registration, not every reference to that destructor: the +// version in 253a2bc retargeted both and would have destroyed @q2, still a +// wtc::queue, with spsc_queue's destructor. +// +// The anchor exists only to make the substitute's functions appear. It is +// erased, along with its @llvm.used entry and the address the shim's +// constructor discards. + +!s32i = !cir.int +!u8i = !cir.int +!void = !cir.void +!queue = !cir.struct<"wtc::queue" {data !cir.ptr}> +!spsc = !cir.struct<"wtc::runtime::spsc_queue" {data !cir.ptr, data !cir.ptr}> +!shared_ptr = !cir.struct<"std::shared_ptr" {data !cir.ptr, data !cir.ptr}> + +module attributes {cir.record_layouts = {"wtc::runtime::spsc_queue" = #cir.record_layout}} { + cir.global "private" hidden external @__dso_handle : !u8i + cir.func private @__cxa_atexit(!cir.ptr)>>, !cir.ptr, !cir.ptr) -> !s32i + + cir.global external @q1 = #cir.zero : !queue {alignment = 8 : i64, wtc.queue_kind = "SPSC"} + cir.global external @q2 = #cir.zero : !queue {alignment = 8 : i64, wtc.queue_kind = "MPSC"} + cir.global "private" external @v : !s32i + + cir.func private @queue_ctor(!cir.ptr) [#cir.annotation<"wtc_queue_construction">] + cir.func private @queue_dtor(!cir.ptr) + cir.func private @queue_push(!cir.ptr, !cir.ptr) [#cir.annotation<"wtc_queue_push">] + cir.func private @queue_try_pop(!cir.ptr) -> !shared_ptr [#cir.annotation<"wtc_queue_try_pop">] + + cir.func private @spsc_ctor(!cir.ptr) [#cir.annotation<"wtc_impl_spsc_construction">] + cir.func private @spsc_dtor(!cir.ptr) [#cir.annotation<"wtc_impl_spsc_dtor">] + cir.func private @spsc_push(!cir.ptr, !cir.ptr) [#cir.annotation<"wtc_impl_spsc_push">] + cir.func private @spsc_try_pop(!cir.ptr) -> !shared_ptr [#cir.annotation<"wtc_impl_spsc_try_pop">] + + cir.func @anchor(%v: !cir.ptr) [#cir.annotation<"wtc_anchor">] { + cir.return + } + cir.global "private" appending @llvm.used = #cir.const_array<[#cir.global_view<@anchor> : !cir.ptr]> : !cir.array x 1> {section = "llvm.metadata"} + + cir.func internal private @__cxx_global_var_init() { + %0 = cir.get_global @q1 : !cir.ptr + cir.call @queue_ctor(%0) : (!cir.ptr) -> () + %1 = cir.get_global @q1 : !cir.ptr + %2 = cir.get_global @queue_dtor : !cir.ptr)>> + %3 = cir.cast bitcast %2 : !cir.ptr)>> -> !cir.ptr)>> + %4 = cir.cast bitcast %1 : !cir.ptr -> !cir.ptr + %5 = cir.get_global @__dso_handle : !cir.ptr + %6 = cir.call @__cxa_atexit(%3, %4, %5) : (!cir.ptr)>>, !cir.ptr, !cir.ptr) -> !s32i + cir.return + } + + cir.func internal private @__cxx_global_var_init.1() { + %0 = cir.get_global @q2 : !cir.ptr + cir.call @queue_ctor(%0) : (!cir.ptr) -> () + %1 = cir.get_global @q2 : !cir.ptr + %2 = cir.get_global @queue_dtor : !cir.ptr)>> + %3 = cir.cast bitcast %2 : !cir.ptr)>> -> !cir.ptr)>> + %4 = cir.cast bitcast %1 : !cir.ptr -> !cir.ptr + %5 = cir.get_global @__dso_handle : !cir.ptr + %6 = cir.call @__cxa_atexit(%3, %4, %5) : (!cir.ptr)>>, !cir.ptr, !cir.ptr) -> !s32i + cir.return + } + + cir.func @use_q1() { + %queue = cir.get_global @q1 : !cir.ptr + %value = cir.get_global @v : !cir.ptr + %qcast = builtin.unrealized_conversion_cast %queue : !cir.ptr to !wtc.queue + "wtc.queue_push"(%qcast, %value) {fallback_impl = @queue_push, source_queue_type = !cir.ptr, wtc.queue_kind = "SPSC"} : (!wtc.queue, !cir.ptr) -> () + %popped = "wtc.queue_pop"(%qcast) {fallback_impl = @queue_try_pop, source_queue_type = !cir.ptr, wtc.queue_kind = "SPSC"} : (!wtc.queue) -> !shared_ptr + cir.return + } + + cir.func @use_q2() { + %queue = cir.get_global @q2 : !cir.ptr + %value = cir.get_global @v : !cir.ptr + %qcast = builtin.unrealized_conversion_cast %queue : !cir.ptr to !wtc.queue + "wtc.queue_push"(%qcast, %value) {fallback_impl = @queue_push, source_queue_type = !cir.ptr, wtc.queue_kind = "MPSC"} : (!wtc.queue, !cir.ptr) -> () + cir.return + } +} + +// CHECK-DAG: remark: substituted "wtc::runtime::spsc_queue" +// CHECK-DAG: remark: not substituted: the queue is MPSC + +// CHECK-NOT: @llvm.used +// CHECK-NOT: cir.func @anchor + +// CHECK: [[SPSC:!rec[A-Za-z0-9_]*]] = !cir.struct<"wtc::runtime::spsc_queue" +// CHECK: cir.global external @q1 = #cir.zero : [[SPSC]] {alignment = 64 : i64 +// CHECK: cir.global external @q2 = #cir.zero : !rec_wtc3A3Aqueue3Cint3E {alignment = 8 : i64 + +// CHECK-LABEL: cir.func internal private @__cxx_global_var_init() +// CHECK: cir.get_global @q1 : !cir.ptr<[[SPSC]]> +// CHECK-NEXT: cir.call @spsc_ctor( +// CHECK: cir.get_global @spsc_dtor : !cir.ptr)>> +// CHECK: cir.call @__cxa_atexit( + +// CHECK-LABEL: cir.func internal private @__cxx_global_var_init.1() +// CHECK: cir.get_global @q2 : !cir.ptr +// CHECK-NEXT: cir.call @queue_ctor( +// CHECK: cir.get_global @queue_dtor : !cir.ptr)>> + +// CHECK-LABEL: cir.func @use_q1() +// CHECK: "wtc.queue_push"{{.*}}fallback_impl = @spsc_push, source_queue_type = !cir.ptr<[[SPSC]]> +// CHECK: "wtc.queue_pop"{{.*}}fallback_impl = @spsc_try_pop, source_queue_type = !cir.ptr<[[SPSC]]> + +// CHECK-LABEL: cir.func @use_q2() +// CHECK: "wtc.queue_push"{{.*}}fallback_impl = @queue_push diff --git a/test/Transforms/SelectQueueImplementation/refusals.mlir b/test/Transforms/SelectQueueImplementation/refusals.mlir new file mode 100644 index 0000000..7c45b71 --- /dev/null +++ b/test/Transforms/SelectQueueImplementation/refusals.mlir @@ -0,0 +1,91 @@ +// RUN: wtc-opt %s --select-queue-impl -split-input-file -verify-diagnostics + +// Every way a queue is left on its original implementation says why. The +// verdicts are written here directly, as --queue-ownership-analysis would. + +// A queue whose verdict is anything but SPSC is not touched. + +!s32i = !cir.int +!queue = !cir.struct<"wtc::queue" {data !cir.ptr}> +module { + // expected-remark @+1 {{not substituted: the queue is UNKNOWN}} + cir.global external @q = #cir.zero : !queue {wtc.queue_kind = "UNKNOWN"} +} + +// ----- + +// The global's address goes to a function this pass knows nothing about. +// Changing the global's type would leave that function reading spsc_queue +// storage as a wtc::queue. + +!s32i = !cir.int +!queue = !cir.struct<"wtc::queue" {data !cir.ptr}> +!spsc = !cir.struct<"wtc::runtime::spsc_queue" {data !cir.ptr}> +!shared_ptr = !cir.struct<"std::shared_ptr" {data !cir.ptr, data !cir.ptr}> +module { + // expected-remark @+1 {{not substituted: it has a use this pass cannot retarget}} + cir.global external @q = #cir.zero : !queue {wtc.queue_kind = "SPSC"} + + cir.func private @inspect(!cir.ptr) + cir.func private @spsc_ctor(!cir.ptr) [#cir.annotation<"wtc_impl_spsc_construction">] + cir.func private @spsc_dtor(!cir.ptr) [#cir.annotation<"wtc_impl_spsc_dtor">] + cir.func private @spsc_push(!cir.ptr, !cir.ptr) [#cir.annotation<"wtc_impl_spsc_push">] + cir.func private @spsc_try_pop(!cir.ptr) -> !shared_ptr [#cir.annotation<"wtc_impl_spsc_try_pop">] + + cir.func @f() { + %queue = cir.get_global @q : !cir.ptr + // expected-remark @+1 {{this use of @q is the one}} + cir.call @inspect(%queue) : (!cir.ptr) -> () + cir.return + } +} + +// ----- + +// An op on this queue can also reach another queue that is not SPSC, so +// retargeting it would be wrong for the other one. + +!s32i = !cir.int +!queue = !cir.struct<"wtc::queue" {data !cir.ptr}> +module { + // expected-remark @+1 {{not substituted: an op on it can also reach a queue that is not SPSC}} + cir.global external @q = #cir.zero : !queue {wtc.queue_kind = "SPSC"} + cir.global "private" external @v : !s32i + + cir.func @f() { + %queue = cir.get_global @q : !cir.ptr + %value = cir.get_global @v : !cir.ptr + %qcast = builtin.unrealized_conversion_cast %queue : !cir.ptr to !wtc.queue + // expected-remark @+1 {{this use of @q is the one}} + "wtc.queue_push"(%qcast, %value) {wtc.queue_kind = "MPSC"} : (!wtc.queue, !cir.ptr) -> () + cir.return + } +} + +// ----- + +// The substitute in the module is for another element type. + +!s32i = !cir.int +!s64i = !cir.int +!queue = !cir.struct<"wtc::queue" {data !cir.ptr}> +!spsc = !cir.struct<"wtc::runtime::spsc_queue" {data !cir.ptr}> +!shared_ptr = !cir.struct<"std::shared_ptr" {data !cir.ptr, data !cir.ptr}> +module { + // expected-remark @+1 {{not substituted: no substitute implementation for element type '!cir.int' is instantiated in this module}} + cir.global external @q = #cir.zero : !queue {wtc.queue_kind = "SPSC"} + cir.global "private" external @v : !s32i + + cir.func private @spsc_ctor(!cir.ptr) [#cir.annotation<"wtc_impl_spsc_construction">] + cir.func private @spsc_dtor(!cir.ptr) [#cir.annotation<"wtc_impl_spsc_dtor">] + cir.func private @spsc_push(!cir.ptr, !cir.ptr) [#cir.annotation<"wtc_impl_spsc_push">] + cir.func private @spsc_try_pop(!cir.ptr) -> !shared_ptr [#cir.annotation<"wtc_impl_spsc_try_pop">] + + cir.func @f() { + %queue = cir.get_global @q : !cir.ptr + %value = cir.get_global @v : !cir.ptr + %qcast = builtin.unrealized_conversion_cast %queue : !cir.ptr to !wtc.queue + "wtc.queue_push"(%qcast, %value) {wtc.queue_kind = "SPSC"} : (!wtc.queue, !cir.ptr) -> () + cir.return + } +}