From d221f72c29e29b5878ac3520fc35abe7b78d7c71 Mon Sep 17 00:00:00 2001 From: zwe Date: Sun, 18 Oct 2026 12:45:08 +0800 Subject: [PATCH 1/4] Substitute spsc_queue for a global queue proven SPSC --select-queue-impl, from 253a2bc (feat/select-queue-impl), ported to the pin and to the analysis now being sound. For a cir.global whose verdict is SPSC it retargets the storage, the construction, the push and pop ops, and the destructor registration to the module's wtc::runtime::spsc_queue for the same T. Changed on the way, each for a reason found at the pin: - The destructor is retargeted through this global's own __cxa_atexit registration. The earlier version rewrote every reference to the original destructor, and two globals of one type share one, so substituting @q1 also registered spsc_queue's destructor for @q2, still a wtc::queue. - Every use of the global is checked first. Its type changes, so a use the pass does not retarget -- the address passed elsewhere, an op that can also reach a non-SPSC queue -- would read the new storage as the old type. Such a global keeps its implementation, and a remark names the use. - The substitute is found by its wtc_impl_spsc_* annotations, not its record name, and its arguments are read from the signature rather than the entry block, which a declaration does not have. - Alignment comes from cir.record_layouts: the data layout of a CIR record type does not see alignas, and spsc_queue aligns to 64. - Pops that write through a result slot are only substituted when the substitute's try_pop takes one too; lowering restores sret. - Anchors (wtc_anchor) are erased afterwards, with their @llvm.used entries. Diagnostics are MLIR remarks, not stdout. Verified: 28 of 28 pass. On the prototype's example (q1 SPSC, q2 MPMC, one type) q1 is substituted and q2 is not, its destructor untouched; the result compiles to a native binary that runs to completion, and for x86-64 to LLVM IR whose substituted try_pop call carries sret and the new size. Co-Authored-By: Claude Opus 5.5 --- docs/audit.md | 26 ++ mlir/include/WTC/Transforms/Passes.h | 1 + mlir/include/WTC/Transforms/Utils.h | 6 + mlir/lib/WTC/Transforms/CMakeLists.txt | 2 + .../Transforms/SelectQueueImplementation.cpp | 367 ++++++++++++++++++ mlir/lib/WTC/Transforms/Utils.cpp | 24 ++ mlir/tools/wtc-opt/wtc-opt.cpp | 1 + .../global-spsc.mlir | 114 ++++++ .../SelectQueueImplementation/refusals.mlir | 91 +++++ 9 files changed, 632 insertions(+) create mode 100644 mlir/lib/WTC/Transforms/SelectQueueImplementation.cpp create mode 100644 test/Transforms/SelectQueueImplementation/global-spsc.mlir create mode 100644 test/Transforms/SelectQueueImplementation/refusals.mlir diff --git a/docs/audit.md b/docs/audit.md index 15335a2..57e5d45 100644 --- a/docs/audit.md +++ b/docs/audit.md @@ -191,6 +191,32 @@ this, and treat a call with no result as a case to handle rather than one to assume away. None of that is attempted in this document; it is the work, not the diagnosis. +## The first specialisation: SPSC queues — October 2026 + +`--select-queue-impl` substitutes `runtime/spsc_queue.hpp` for a global +`wtc::queue` whose verdict is SPSC. Built from 253a2bc, with these +changes measured or found on the way: + +- **A shared destructor was retargeted for every queue.** Two globals of the + same type share a destructor, and the earlier version rewrote every + reference to it, so substituting one queue registered the substitute's + destructor for the other too. Retargeting now follows the substituted + global's own `__cxa_atexit` registration + (`test/Transforms/SelectQueueImplementation/global-spsc.mlir`). +- **Any other use of the global was left with the wrong type.** The global's + type changes, so a use the pass does not retarget would read the new + storage as the old type. Such a global is now not substituted, and the + remark names the use (`refusals.mlir`). +- **The substitute was bounded and dropped items when full.** It is now + unbounded, like the queue it replaces. +- **The program had to name the substitute.** The shim now makes it + available, through an anchor the pass erases. +- **Alignment.** The data layout of a CIR record type does not see `alignas`; + `cir.record_layouts` does, and is what the pass uses. + +Only globals are substituted. A queue on the stack or reached only through +parameters keeps its implementation. + ## Infrastructure Fixed before the move into this repository: diff --git a/mlir/include/WTC/Transforms/Passes.h b/mlir/include/WTC/Transforms/Passes.h index feebfbf..26c9da6 100644 --- a/mlir/include/WTC/Transforms/Passes.h +++ b/mlir/include/WTC/Transforms/Passes.h @@ -6,6 +6,7 @@ namespace mlir::wtc { void registerLiftCIRToWTCPass(); void registerLowerWTCToCIRPass(); void registerQueueOwnershipAnalysisPass(); + void registerSelectQueueImplementationPass(); void analyseQueues(ModuleOp module); } diff --git a/mlir/include/WTC/Transforms/Utils.h b/mlir/include/WTC/Transforms/Utils.h index da66379..0f5a5eb 100644 --- a/mlir/include/WTC/Transforms/Utils.h +++ b/mlir/include/WTC/Transforms/Utils.h @@ -19,6 +19,12 @@ namespace mlir::wtc { bool isWTCAnnotatedCall(cir::CallOp op, StringRef annotation); + bool funcHasAnnotation(cir::FuncOp funcOp, StringRef annotation); + + // The element type of a queue whose push is pushFunc: the pointee of the + // reference it takes the value by. + Type deriveQueueElementTypeFromPush(cir::FuncOp pushFunc); + SemanticOpKind getSemanticKind(cir::FuncOp funcOp); // The argument through which funcOp returns its value indirectly, if it diff --git a/mlir/lib/WTC/Transforms/CMakeLists.txt b/mlir/lib/WTC/Transforms/CMakeLists.txt index 06c2e26..70ac46d 100644 --- a/mlir/lib/WTC/Transforms/CMakeLists.txt +++ b/mlir/lib/WTC/Transforms/CMakeLists.txt @@ -2,6 +2,7 @@ add_mlir_library(MLIRWTCTransforms LiftCIRToWTC.cpp LowerWTCToCIR.cpp QueueOwnershipAnalysis.cpp + SelectQueueImplementation.cpp Utils.cpp LINK_LIBS PUBLIC MLIRWTC @@ -11,6 +12,7 @@ add_mlir_library(MLIRWTCTransforms # wtc-opt linked MLIRCIR itself; a second consumer would not be so lucky. MLIRCIR MLIRCIRInterfaces + MLIRDataLayoutInterfaces MLIRIR MLIRPass diff --git a/mlir/lib/WTC/Transforms/SelectQueueImplementation.cpp b/mlir/lib/WTC/Transforms/SelectQueueImplementation.cpp new file mode 100644 index 0000000..e8e7d50 --- /dev/null +++ b/mlir/lib/WTC/Transforms/SelectQueueImplementation.cpp @@ -0,0 +1,367 @@ +#include "WTC/WTCOps.h" +#include "WTC/Transforms/Passes.h" +#include "WTC/Transforms/Utils.h" + +#include "clang/CIR/Dialect/IR/CIRDialect.h" +#include "clang/CIR/Dialect/IR/CIRAttrs.h" + +#include "mlir/Interfaces/DataLayoutInterfaces.h" +#include "mlir/IR/BuiltinOps.h" +#include "mlir/Pass/Pass.h" + +#include "llvm/ADT/SmallVector.h" + +using namespace mlir; +using namespace mlir::wtc; + +// Substitutes the storage of a global wtc::queue proven SPSC with +// runtime/spsc_queue.hpp's implementation for the same T. +// +// The substitute is found in the module, not synthesised: the shim's queue +// constructor odr-uses an anchor (annotated wtc_anchor) that instantiates +// wtc::runtime::spsc_queue, so every module using queue also carries +// spsc_queue's constructor, push, try_pop and destructor, each marked +// wtc_impl_spsc_*. The anchor is erased afterwards. +// +// Only globals are substituted, and only when every use of the global is +// one this pass knows how to retarget: the queue's own push and pop, its +// construction, and its destructor's registration with __cxa_atexit. A +// global that is used any other way keeps its original type, since that +// use would be left calling the original implementation on the new +// storage. + +namespace { + +constexpr StringRef kSpscPrefix = "wtc_impl_spsc_"; + +struct AltImpl { + cir::RecordType recordType; + Type elementType; + FlatSymbolRefAttr construct; + FlatSymbolRefAttr push; + FlatSymbolRefAttr tryPop; + FlatSymbolRefAttr dtor; + + bool complete() const { return elementType && construct && push && tryPop && dtor; } +}; + +// What one global's uses amount to, if every one of them can be retargeted. +struct GlobalUses { + llvm::SmallVector getGlobals; + llvm::SmallVector queueOps; + llvm::SmallVector constructions; + // The cir.get_global of the destructor each registration passes. + llvm::SmallVector dtorRefs; +}; + +cir::RecordType recordOfThis(cir::FuncOp funcOp) { + if (funcOp.getNumArguments() < 1) + return {}; + // Where try_pop returns indirectly, `this` follows the result slot. + unsigned self = sretArgIndex(funcOp) == 0u ? 1 : 0; + if (funcOp.getNumArguments() <= self) + return {}; + // From the signature, not the entry block: a declaration has none. + auto thisPtrType = mlir::dyn_cast(funcOp.getFunctionType().getInput(self)); + if (!thisPtrType) + return {}; + return mlir::dyn_cast(thisPtrType.getPointee()); +} + +Value stripCasts(Value v) { + while (auto castOp = v.getDefiningOp()) + v = castOp.getSrc(); + return v; +} + +} + +struct SelectQueueImplementationPass + : public PassWrapper> { + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(SelectQueueImplementationPass) + + StringRef getArgument() const final { + return "select-queue-impl"; + } + StringRef getDescription() const final { + return "Substitute wtc::runtime::spsc_queue for a global wtc::queue proven SPSC"; + } + + void getDependentDialects(DialectRegistry ®istry) const override { + registry.insert(); + registry.insert(); + } + + void runOnOperation() override { + ModuleOp module = getOperation(); + llvm::SmallVector altImpls = findAltImpls(module); + + for (cir::GlobalOp globalOp : llvm::to_vector(module.getOps())) { + auto kind = globalOp->getAttrOfType(kQueueKindAttrName); + if (!kind) + continue; + if (kind.getValue() != "SPSC") { + globalOp.emitRemark("not substituted: the queue is ") << kind.getValue(); + continue; + } + selectForGlobal(module, globalOp, altImpls); + } + + eraseAnchors(module); + } + + llvm::SmallVector findAltImpls(ModuleOp module) { + llvm::SmallVector impls; + auto implFor = [&](cir::RecordType recordType) -> AltImpl & { + for (AltImpl &impl : impls) + if (impl.recordType == recordType) + return impl; + impls.push_back({}); + impls.back().recordType = recordType; + return impls.back(); + }; + + // The complete and base constructors (C1, C2), and destructors (D1, + // D2), are interchangeable for a class without virtual bases, which + // spsc_queue is; the first one found is used. + module.walk([&](cir::FuncOp funcOp) { + auto sym = FlatSymbolRefAttr::get(funcOp.getSymNameAttr()); + auto setOnce = [&](FlatSymbolRefAttr &slot) { if (!slot) slot = sym; }; + + if (funcHasAnnotation(funcOp, (kSpscPrefix + "construction").str())) { + if (auto recordType = recordOfThis(funcOp)) + setOnce(implFor(recordType).construct); + } else if (funcHasAnnotation(funcOp, (kSpscPrefix + "dtor").str())) { + if (auto recordType = recordOfThis(funcOp)) + setOnce(implFor(recordType).dtor); + } else if (funcHasAnnotation(funcOp, (kSpscPrefix + "push").str())) { + if (auto recordType = recordOfThis(funcOp)) { + AltImpl &impl = implFor(recordType); + setOnce(impl.push); + impl.elementType = deriveQueueElementTypeFromPush(funcOp); + } + } else if (funcHasAnnotation(funcOp, (kSpscPrefix + "try_pop").str())) { + if (auto recordType = recordOfThis(funcOp)) + setOnce(implFor(recordType).tryPop); + } + }); + + llvm::erase_if(impls, [](const AltImpl &impl) { return !impl.complete(); }); + return impls; + } + + // Every use of the global, sorted into the kinds this pass can + // retarget; std::nullopt, with a remark, at the first it cannot. + std::optional collectUses(ModuleOp module, cir::GlobalOp globalOp) { + GlobalUses result; + auto refuse = [&](Operation *user, StringRef why) -> std::optional { + globalOp.emitRemark("not substituted: ") << why; + user->emitRemark("this use of @") << globalOp.getSymName() << " is the one"; + return std::nullopt; + }; + + auto uses = SymbolTable::getSymbolUses(globalOp.getOperation(), module.getOperation()); + if (!uses) + return result; + + for (const SymbolTable::SymbolUse &use : *uses) { + auto getGlobalOp = mlir::dyn_cast(use.getUser()); + if (!getGlobalOp) + return refuse(use.getUser(), "its address is used other than through cir.get_global"); + result.getGlobals.push_back(getGlobalOp); + + for (Operation *user : getGlobalOp.getResult().getUsers()) { + if (auto castOp = mlir::dyn_cast(user)) { + if (castOp.getNumResults() != 1 || !mlir::isa(castOp.getResult(0).getType())) + return refuse(user, "it is cast to something other than a wtc queue"); + for (Operation *queueUser : castOp.getResult(0).getUsers()) { + if (!mlir::isa(queueUser)) + return refuse(queueUser, "the lifted queue is used by something other than push and pop"); + auto opKind = queueUser->getAttrOfType(kQueueKindAttrName); + if (!opKind || opKind.getValue() != "SPSC") + return refuse(queueUser, "an op on it can also reach a queue that is not SPSC"); + result.queueOps.push_back(queueUser); + } + continue; + } + + auto callOp = mlir::dyn_cast(user); + if (callOp && isWTCAnnotatedCall(callOp, "wtc_queue_construction") && + !callOp.getArgOperands().empty() && callOp.getArgOperands()[0] == getGlobalOp.getResult()) { + result.constructions.push_back(callOp); + continue; + } + + // __cxa_atexit(dtor, &global, dso_handle), both pointers cast + // to their void forms first. + auto castOp = mlir::dyn_cast(user); + if (castOp && castOp.getResult().hasOneUse()) { + auto atexit = mlir::dyn_cast(*castOp.getResult().getUsers().begin()); + auto callee = atexit ? atexit.getCalleeAttr() : FlatSymbolRefAttr(); + if (callee && callee.getValue() == "__cxa_atexit" && + atexit.getArgOperands().size() == 3 && + atexit.getArgOperands()[1] == castOp.getResult()) { + auto dtorRef = stripCasts(atexit.getArgOperands()[0]).getDefiningOp(); + if (!dtorRef) + return refuse(atexit, "its destructor registration names no function"); + // The reference is changed in place, so it must + // serve this registration and nothing else. + Value v = dtorRef.getResult(); + while (v.hasOneUse() && mlir::isa(*v.getUsers().begin())) + v = v.getUsers().begin()->getResult(0); + if (!dtorRef.getResult().hasOneUse() || !v.hasOneUse()) + return refuse(atexit, "its destructor reference is shared with another use"); + result.dtorRefs.push_back(dtorRef); + continue; + } + } + + return refuse(user, "it has a use this pass cannot retarget"); + } + } + return result; + } + + void selectForGlobal(ModuleOp module, cir::GlobalOp globalOp, ArrayRef altImpls) { + std::optional uses = collectUses(module, globalOp); + if (!uses) + return; + if (uses->queueOps.empty()) + return; + + auto queueType = mlir::cast( + mlir::isa(uses->queueOps.front()) + ? mlir::cast(uses->queueOps.front()).getQueue().getType() + : mlir::cast(uses->queueOps.front()).getQueue().getType()); + + const AltImpl *chosen = nullptr; + for (const AltImpl &impl : altImpls) + if (impl.elementType == queueType.getElementType()) + chosen = &impl; + if (!chosen) { + globalOp.emitRemark("not substituted: no substitute implementation for element type ") + << queueType.getElementType() << " is instantiated in this module"; + return; + } + + auto newPtrType = cir::PointerType::get(chosen->recordType); + auto newDtor = SymbolTable::lookupNearestSymbolFrom(globalOp, chosen->dtor); + auto newTryPop = SymbolTable::lookupNearestSymbolFrom(globalOp, chosen->tryPop); + + // A pop lifted with a result slot has to call a try_pop that takes + // one, and one lifted with a result a try_pop that returns it. Both + // implementations are compiled for the same target, so they agree; + // check rather than assume. + bool substituteReturnsIndirectly = sretArgIndex(newTryPop).has_value(); + for (Operation *op : uses->queueOps) { + auto popOp = mlir::dyn_cast(op); + if (popOp && static_cast(popOp.getResultSlot()) != substituteReturnsIndirectly) { + globalOp.emitRemark("not substituted: the substitute's try_pop returns its value differently"); + return; + } + } + + globalOp.setSymTypeAttr(TypeAttr::get(chosen->recordType)); + globalOp.setInitialValueAttr(cir::ZeroAttr::get(chosen->recordType)); + // The record's own alignment, alignas included, is what ClangIR + // records in cir.record_layouts; the data layout of the record type + // does not see alignas, and spsc_queue aligns its cursors to 64. + uint64_t align = 0; + if (auto layouts = module->getAttrOfType("cir.record_layouts")) + if (auto layout = mlir::dyn_cast_or_null( + layouts.get(chosen->recordType.getName().getValue()))) + align = layout.getRecordAlign(); + if (!align) + align = DataLayout::closest(globalOp).getTypeABIAlignment(chosen->recordType); + globalOp.setAlignmentAttr(IntegerAttr::get(IntegerType::get(&getContext(), 64), align)); + + for (cir::GetGlobalOp getGlobalOp : uses->getGlobals) + getGlobalOp.getResult().setType(newPtrType); + + for (Operation *op : uses->queueOps) { + if (auto pushOp = mlir::dyn_cast(op)) { + pushOp.setFallbackImplAttr(chosen->push); + pushOp.setSourceQueueTypeAttr(TypeAttr::get(newPtrType)); + } else { + auto popOp = mlir::cast(op); + popOp.setFallbackImplAttr(chosen->tryPop); + popOp.setSourceQueueTypeAttr(TypeAttr::get(newPtrType)); + } + } + + auto newCtor = SymbolTable::lookupNearestSymbolFrom(globalOp, chosen->construct); + for (cir::CallOp callOp : uses->constructions) { + callOp.setCalleeAttr(chosen->construct); + copyArgAttrsFromCallee(callOp, newCtor); + } + + for (cir::GetGlobalOp dtorRef : uses->dtorRefs) { + dtorRef->setAttr("name", chosen->dtor); + dtorRef.getResult().setType(cir::PointerType::get(newDtor.getFunctionType())); + } + + globalOp.emitRemark("substituted ") << chosen->recordType.getName(); + } + + // [[gnu::used]] puts each anchor in @llvm.used, which is what makes the + // compiler emit it; take it out again, dropping @llvm.used if nothing + // else is left in it. + void dropFromLlvmUsed(ModuleOp module, StringRef symbol) { + auto used = module.lookupSymbol("llvm.used"); + if (!used) + return; + auto array = mlir::dyn_cast_or_null(used.getInitialValueAttr()); + auto elements = array ? mlir::dyn_cast(array.getElts()) : ArrayAttr(); + if (!elements) + return; + + llvm::SmallVector kept; + for (Attribute element : elements) { + auto view = mlir::dyn_cast(element); + if (!view || view.getSymbol().getValue() != symbol) + kept.push_back(element); + } + if (kept.size() == elements.size()) + return; + if (kept.empty()) { + used.erase(); + return; + } + + auto oldType = mlir::cast(array.getType()); + auto newType = cir::ArrayType::get(oldType.getElementType(), kept.size()); + used.setSymTypeAttr(TypeAttr::get(newType)); + used.setInitialValueAttr(cir::ConstArrayAttr::get(newType, ArrayAttr::get(&getContext(), kept))); + } + + // The anchors exist only to make the substitutes' functions appear in + // the module. Nothing calls them; @llvm.used keeps them alive, and the + // shim's constructor may take an address it then discards. + void eraseAnchors(ModuleOp module) { + for (cir::FuncOp funcOp : llvm::to_vector(module.getOps())) { + if (!funcHasAnnotation(funcOp, "wtc_anchor")) + continue; + + dropFromLlvmUsed(module, funcOp.getSymName()); + + if (auto uses = SymbolTable::getSymbolUses(funcOp.getOperation(), module.getOperation())) { + for (const SymbolTable::SymbolUse &use : llvm::make_early_inc_range(*uses)) { + auto getGlobalOp = mlir::dyn_cast(use.getUser()); + if (getGlobalOp && getGlobalOp.getResult().use_empty()) + getGlobalOp.erase(); + } + } + + if (SymbolTable::symbolKnownUseEmpty(funcOp.getOperation(), module.getOperation())) + funcOp.erase(); + else + funcOp.emitRemark("anchor kept: something other than a discarded address refers to it"); + } + } +}; + +namespace mlir::wtc { + void registerSelectQueueImplementationPass() { + PassRegistration(); + } +} diff --git a/mlir/lib/WTC/Transforms/Utils.cpp b/mlir/lib/WTC/Transforms/Utils.cpp index b33840e..f7ed81a 100644 --- a/mlir/lib/WTC/Transforms/Utils.cpp +++ b/mlir/lib/WTC/Transforms/Utils.cpp @@ -31,6 +31,30 @@ namespace mlir::wtc { return false; } + bool funcHasAnnotation(cir::FuncOp funcOp, StringRef annotation) { + if (!funcOp) + return false; + auto annotations = funcOp.getAnnotationsAttr(); + if (!annotations) + return false; + for (auto attr : annotations) { + auto annotAttr = mlir::dyn_cast(attr); + if (annotAttr && annotAttr.getName().getValue() == annotation) + return true; + } + return false; + } + + Type deriveQueueElementTypeFromPush(cir::FuncOp pushFunc) { + if (!pushFunc || pushFunc.getNumArguments() < 2) + return {}; + // From the signature, not the entry block: a declaration has none. + auto valuePtrType = mlir::dyn_cast(pushFunc.getFunctionType().getInput(1)); + if (!valuePtrType) + return {}; + return valuePtrType.getPointee(); + } + SemanticOpKind getSemanticKind(cir::FuncOp funcOp) { if (!funcOp) return SemanticOpKind::Unknown; diff --git a/mlir/tools/wtc-opt/wtc-opt.cpp b/mlir/tools/wtc-opt/wtc-opt.cpp index e627059..25e2823 100644 --- a/mlir/tools/wtc-opt/wtc-opt.cpp +++ b/mlir/tools/wtc-opt/wtc-opt.cpp @@ -20,6 +20,7 @@ int main(int argc, char **argv) { mlir::wtc::registerLiftCIRToWTCPass(); mlir::wtc::registerLowerWTCToCIRPass(); mlir::wtc::registerQueueOwnershipAnalysisPass(); + mlir::wtc::registerSelectQueueImplementationPass(); return mlir::asMainReturnCode( mlir::MlirOptMain(argc, argv, "wtc-opt: the Well-tempered Compiler optimizer driver\n", registry)); diff --git a/test/Transforms/SelectQueueImplementation/global-spsc.mlir b/test/Transforms/SelectQueueImplementation/global-spsc.mlir new file mode 100644 index 0000000..84a44f5 --- /dev/null +++ b/test/Transforms/SelectQueueImplementation/global-spsc.mlir @@ -0,0 +1,114 @@ +// RUN: wtc-opt %s --select-queue-impl --mlir-print-op-on-diagnostic=false 2>&1 | FileCheck %s + +// Two globals of the same type, wtc::queue, reduced from real output. +// @q1 is SPSC and @q2 is MPSC (the verdicts are written here directly, as +// --queue-ownership-analysis would). @q1's storage, construction, push, +// pop and destructor registration are retargeted to spsc_queue; @q2 +// keeps every one of them. The two share a destructor, so retargeting must +// follow @q1's registration, not every reference to that destructor: the +// version in 253a2bc retargeted both and would have destroyed @q2, still a +// wtc::queue, with spsc_queue's destructor. +// +// The anchor exists only to make the substitute's functions appear. It is +// erased, along with its @llvm.used entry and the address the shim's +// constructor discards. + +!s32i = !cir.int +!u8i = !cir.int +!void = !cir.void +!queue = !cir.struct<"wtc::queue" {data !cir.ptr}> +!spsc = !cir.struct<"wtc::runtime::spsc_queue" {data !cir.ptr, data !cir.ptr}> +!shared_ptr = !cir.struct<"std::shared_ptr" {data !cir.ptr, data !cir.ptr}> + +module attributes {cir.record_layouts = {"wtc::runtime::spsc_queue" = #cir.record_layout}} { + cir.global "private" hidden external @__dso_handle : !u8i + cir.func private @__cxa_atexit(!cir.ptr)>>, !cir.ptr, !cir.ptr) -> !s32i + + cir.global external @q1 = #cir.zero : !queue {alignment = 8 : i64, wtc.queue_kind = "SPSC"} + cir.global external @q2 = #cir.zero : !queue {alignment = 8 : i64, wtc.queue_kind = "MPSC"} + cir.global "private" external @v : !s32i + + cir.func private @queue_ctor(!cir.ptr) [#cir.annotation<"wtc_queue_construction">] + cir.func private @queue_dtor(!cir.ptr) + cir.func private @queue_push(!cir.ptr, !cir.ptr) [#cir.annotation<"wtc_queue_push">] + cir.func private @queue_try_pop(!cir.ptr) -> !shared_ptr [#cir.annotation<"wtc_queue_try_pop">] + + cir.func private @spsc_ctor(!cir.ptr) [#cir.annotation<"wtc_impl_spsc_construction">] + cir.func private @spsc_dtor(!cir.ptr) [#cir.annotation<"wtc_impl_spsc_dtor">] + cir.func private @spsc_push(!cir.ptr, !cir.ptr) [#cir.annotation<"wtc_impl_spsc_push">] + cir.func private @spsc_try_pop(!cir.ptr) -> !shared_ptr [#cir.annotation<"wtc_impl_spsc_try_pop">] + + cir.func @anchor(%v: !cir.ptr) [#cir.annotation<"wtc_anchor">] { + cir.return + } + cir.global "private" appending @llvm.used = #cir.const_array<[#cir.global_view<@anchor> : !cir.ptr]> : !cir.array x 1> {section = "llvm.metadata"} + + cir.func internal private @__cxx_global_var_init() { + %0 = cir.get_global @q1 : !cir.ptr + cir.call @queue_ctor(%0) : (!cir.ptr) -> () + %1 = cir.get_global @q1 : !cir.ptr + %2 = cir.get_global @queue_dtor : !cir.ptr)>> + %3 = cir.cast bitcast %2 : !cir.ptr)>> -> !cir.ptr)>> + %4 = cir.cast bitcast %1 : !cir.ptr -> !cir.ptr + %5 = cir.get_global @__dso_handle : !cir.ptr + %6 = cir.call @__cxa_atexit(%3, %4, %5) : (!cir.ptr)>>, !cir.ptr, !cir.ptr) -> !s32i + cir.return + } + + cir.func internal private @__cxx_global_var_init.1() { + %0 = cir.get_global @q2 : !cir.ptr + cir.call @queue_ctor(%0) : (!cir.ptr) -> () + %1 = cir.get_global @q2 : !cir.ptr + %2 = cir.get_global @queue_dtor : !cir.ptr)>> + %3 = cir.cast bitcast %2 : !cir.ptr)>> -> !cir.ptr)>> + %4 = cir.cast bitcast %1 : !cir.ptr -> !cir.ptr + %5 = cir.get_global @__dso_handle : !cir.ptr + %6 = cir.call @__cxa_atexit(%3, %4, %5) : (!cir.ptr)>>, !cir.ptr, !cir.ptr) -> !s32i + cir.return + } + + cir.func @use_q1() { + %queue = cir.get_global @q1 : !cir.ptr + %value = cir.get_global @v : !cir.ptr + %qcast = builtin.unrealized_conversion_cast %queue : !cir.ptr to !wtc.queue + "wtc.queue_push"(%qcast, %value) {fallback_impl = @queue_push, source_queue_type = !cir.ptr, wtc.queue_kind = "SPSC"} : (!wtc.queue, !cir.ptr) -> () + %popped = "wtc.queue_pop"(%qcast) {fallback_impl = @queue_try_pop, source_queue_type = !cir.ptr, wtc.queue_kind = "SPSC"} : (!wtc.queue) -> !shared_ptr + cir.return + } + + cir.func @use_q2() { + %queue = cir.get_global @q2 : !cir.ptr + %value = cir.get_global @v : !cir.ptr + %qcast = builtin.unrealized_conversion_cast %queue : !cir.ptr to !wtc.queue + "wtc.queue_push"(%qcast, %value) {fallback_impl = @queue_push, source_queue_type = !cir.ptr, wtc.queue_kind = "MPSC"} : (!wtc.queue, !cir.ptr) -> () + cir.return + } +} + +// CHECK-DAG: remark: substituted "wtc::runtime::spsc_queue" +// CHECK-DAG: remark: not substituted: the queue is MPSC + +// CHECK-NOT: @llvm.used +// CHECK-NOT: cir.func @anchor + +// CHECK: [[SPSC:!rec[A-Za-z0-9_]*]] = !cir.struct<"wtc::runtime::spsc_queue" +// CHECK: cir.global external @q1 = #cir.zero : [[SPSC]] {alignment = 64 : i64 +// CHECK: cir.global external @q2 = #cir.zero : !rec_wtc3A3Aqueue3Cint3E {alignment = 8 : i64 + +// CHECK-LABEL: cir.func internal private @__cxx_global_var_init() +// CHECK: cir.get_global @q1 : !cir.ptr<[[SPSC]]> +// CHECK-NEXT: cir.call @spsc_ctor( +// CHECK: cir.get_global @spsc_dtor : !cir.ptr)>> +// CHECK: cir.call @__cxa_atexit( + +// CHECK-LABEL: cir.func internal private @__cxx_global_var_init.1() +// CHECK: cir.get_global @q2 : !cir.ptr +// CHECK-NEXT: cir.call @queue_ctor( +// CHECK: cir.get_global @queue_dtor : !cir.ptr)>> + +// CHECK-LABEL: cir.func @use_q1() +// CHECK: "wtc.queue_push"{{.*}}fallback_impl = @spsc_push, source_queue_type = !cir.ptr<[[SPSC]]> +// CHECK: "wtc.queue_pop"{{.*}}fallback_impl = @spsc_try_pop, source_queue_type = !cir.ptr<[[SPSC]]> + +// CHECK-LABEL: cir.func @use_q2() +// CHECK: "wtc.queue_push"{{.*}}fallback_impl = @queue_push diff --git a/test/Transforms/SelectQueueImplementation/refusals.mlir b/test/Transforms/SelectQueueImplementation/refusals.mlir new file mode 100644 index 0000000..7c45b71 --- /dev/null +++ b/test/Transforms/SelectQueueImplementation/refusals.mlir @@ -0,0 +1,91 @@ +// RUN: wtc-opt %s --select-queue-impl -split-input-file -verify-diagnostics + +// Every way a queue is left on its original implementation says why. The +// verdicts are written here directly, as --queue-ownership-analysis would. + +// A queue whose verdict is anything but SPSC is not touched. + +!s32i = !cir.int +!queue = !cir.struct<"wtc::queue" {data !cir.ptr}> +module { + // expected-remark @+1 {{not substituted: the queue is UNKNOWN}} + cir.global external @q = #cir.zero : !queue {wtc.queue_kind = "UNKNOWN"} +} + +// ----- + +// The global's address goes to a function this pass knows nothing about. +// Changing the global's type would leave that function reading spsc_queue +// storage as a wtc::queue. + +!s32i = !cir.int +!queue = !cir.struct<"wtc::queue" {data !cir.ptr}> +!spsc = !cir.struct<"wtc::runtime::spsc_queue" {data !cir.ptr}> +!shared_ptr = !cir.struct<"std::shared_ptr" {data !cir.ptr, data !cir.ptr}> +module { + // expected-remark @+1 {{not substituted: it has a use this pass cannot retarget}} + cir.global external @q = #cir.zero : !queue {wtc.queue_kind = "SPSC"} + + cir.func private @inspect(!cir.ptr) + cir.func private @spsc_ctor(!cir.ptr) [#cir.annotation<"wtc_impl_spsc_construction">] + cir.func private @spsc_dtor(!cir.ptr) [#cir.annotation<"wtc_impl_spsc_dtor">] + cir.func private @spsc_push(!cir.ptr, !cir.ptr) [#cir.annotation<"wtc_impl_spsc_push">] + cir.func private @spsc_try_pop(!cir.ptr) -> !shared_ptr [#cir.annotation<"wtc_impl_spsc_try_pop">] + + cir.func @f() { + %queue = cir.get_global @q : !cir.ptr + // expected-remark @+1 {{this use of @q is the one}} + cir.call @inspect(%queue) : (!cir.ptr) -> () + cir.return + } +} + +// ----- + +// An op on this queue can also reach another queue that is not SPSC, so +// retargeting it would be wrong for the other one. + +!s32i = !cir.int +!queue = !cir.struct<"wtc::queue" {data !cir.ptr}> +module { + // expected-remark @+1 {{not substituted: an op on it can also reach a queue that is not SPSC}} + cir.global external @q = #cir.zero : !queue {wtc.queue_kind = "SPSC"} + cir.global "private" external @v : !s32i + + cir.func @f() { + %queue = cir.get_global @q : !cir.ptr + %value = cir.get_global @v : !cir.ptr + %qcast = builtin.unrealized_conversion_cast %queue : !cir.ptr to !wtc.queue + // expected-remark @+1 {{this use of @q is the one}} + "wtc.queue_push"(%qcast, %value) {wtc.queue_kind = "MPSC"} : (!wtc.queue, !cir.ptr) -> () + cir.return + } +} + +// ----- + +// The substitute in the module is for another element type. + +!s32i = !cir.int +!s64i = !cir.int +!queue = !cir.struct<"wtc::queue" {data !cir.ptr}> +!spsc = !cir.struct<"wtc::runtime::spsc_queue" {data !cir.ptr}> +!shared_ptr = !cir.struct<"std::shared_ptr" {data !cir.ptr, data !cir.ptr}> +module { + // expected-remark @+1 {{not substituted: no substitute implementation for element type '!cir.int' is instantiated in this module}} + cir.global external @q = #cir.zero : !queue {wtc.queue_kind = "SPSC"} + cir.global "private" external @v : !s32i + + cir.func private @spsc_ctor(!cir.ptr) [#cir.annotation<"wtc_impl_spsc_construction">] + cir.func private @spsc_dtor(!cir.ptr) [#cir.annotation<"wtc_impl_spsc_dtor">] + cir.func private @spsc_push(!cir.ptr, !cir.ptr) [#cir.annotation<"wtc_impl_spsc_push">] + cir.func private @spsc_try_pop(!cir.ptr) -> !shared_ptr [#cir.annotation<"wtc_impl_spsc_try_pop">] + + cir.func @f() { + %queue = cir.get_global @q : !cir.ptr + %value = cir.get_global @v : !cir.ptr + %qcast = builtin.unrealized_conversion_cast %queue : !cir.ptr to !wtc.queue + "wtc.queue_push"(%qcast, %value) {wtc.queue_kind = "SPSC"} : (!wtc.queue, !cir.ptr) -> () + cir.return + } +} From 8803c645298007b17ceb2fc412f69ca89873ee41 Mon Sep 17 00:00:00 2001 From: zwe Date: Sun, 18 Oct 2026 12:45:08 +0800 Subject: [PATCH 2/4] Make the substitute available from the shim, with no anchor in user code A program had to instantiate wtc::spsc_queue itself for the pass to find it, so a human still named the specialisation the compiler was meant to choose. queue's constructor now odr-uses wtc::detail::alternatives, which instantiates wtc::runtime::spsc_queue. Measured at the pin: it is [[gnu::used]], putting it in @llvm.used, that makes ClangIR emit it and everything it calls, at -O0 and -O1; the address taken in the constructor leaves nothing behind. --select-queue-impl erases it. Co-Authored-By: Claude Opus 5.5 --- shim/queue.hpp | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/shim/queue.hpp b/shim/queue.hpp index 2faafcc..6df3138 100644 --- a/shim/queue.hpp +++ b/shim/queue.hpp @@ -3,8 +3,27 @@ #include #include +#include "../runtime/spsc_queue.hpp" + namespace wtc { +namespace detail { + +// Never called. Its only job is to make the compiler emit, in every module +// that uses queue, the implementations --select-queue-impl may substitute +// for it, so that a program never has to name them. queue's constructor +// takes its address, which is enough for it, and through it spsc_queue, +// to be instantiated and emitted. The pass erases it afterwards. +template +[[gnu::used, clang::annotate("wtc_anchor")]] +void alternatives(const T &value) { + runtime::spsc_queue spsc; + spsc.push(value); + (void)spsc.try_pop(); +} + +} + template class queue { private: @@ -35,7 +54,9 @@ class queue { public: [[clang::annotate("wtc_queue_construction")]] - queue() : head(new node), tail(head.get()) {} + queue() : head(new node), tail(head.get()) { + (void)&detail::alternatives; + } queue(const queue&) = delete; queue& operator=(const queue&) = delete; From 57ea2a0af4435ce12aa6f8acbed21cc81aed4331 Mon Sep 17 00:00:00 2001 From: zwe Date: Sun, 18 Oct 2026 12:45:08 +0800 Subject: [PATCH 3/4] Measure the substitution at the pin, end to end, and check its results From 253a2bc, with three changes. queue_bench.cpp names no substitute, since the shim now provides one. Its consumer checks the count, the sum and the order of what it receives, and the run fails if any is wrong, because a fast wrong queue is not a result. run.sh takes the pinned toolchain from LLVM_BUILD (default /opt/llvm, as in the image), uses xcrun only on Darwin, refuses to time an "optimized" build in which nothing was substituted, and reports the median of RUNS runs. mlir-translate still needs --allow-unregistered-dialect: cir-to-llvm leaves CIR types in attributes even for plain clang output. wtc-opt no longer does. Measured on an Apple M1 Pro, median of five: baseline 2.07M items/s, substituted 3.6x that, hand-written 20.9x. The remaining gap is one shared_ptr allocation per pop, which the substitute keeps to match try_pop's interface. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- benchmarks/README.md | 34 +++++++++++++++- benchmarks/queue_bench.cpp | 66 +++++++++++++++++++++++++++++++ benchmarks/run.sh | 79 ++++++++++++++++++++++++++++++++++++++ benchmarks/spsc_direct.cpp | 61 +++++++++++++++++++++++++++++ runtime/spsc_queue.hpp | 7 +++- 6 files changed, 245 insertions(+), 4 deletions(-) create mode 100644 benchmarks/queue_bench.cpp create mode 100755 benchmarks/run.sh create mode 100644 benchmarks/spsc_direct.cpp diff --git a/README.md b/README.md index 6b4f89d..8bfb782 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ what exists is one vertical slice and it is not finished: | Lifting CIR calls into the dialect | works on hand-written IR; the C++ path is not yet exercised end to end | | Queue ownership analysis: SPSC / MPSC / SPMC / MPMC | prints a verdict; sound for direct calls, UNKNOWN wherever a function's address escapes; thread lifetime and escaping queues not modelled | | Lowering back to CIR | faithful round trip; crashes on operations that carry no source-type attribute | -| Specialised implementations | none yet | +| Specialised implementations | an unbounded SPSC queue, substituted for a global `wtc::queue` proven SPSC; 3.6x throughput on one producer and one consumer (`benchmarks/`) | `docs/audit.md` lists the known defects in full. Do not build anything on top of the analysis verdict until they are closed. diff --git a/benchmarks/README.md b/benchmarks/README.md index a00f10b..fac59e4 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -14,4 +14,36 @@ for, and the hand-written specialised implementation from `runtime/`. The second one is the ceiling: the compiler cannot beat code a human wrote for this exact case, it can only reach it without the human. -Nothing here yet. +## What is here + +- `queue_bench.cpp`: one producer and one consumer move 5M ints through a + global `wtc::queue`. The consumer checks the count, the sum and the + order, and the run fails if any is wrong. Nothing in it names the + substitute: the shim makes `wtc::runtime::spsc_queue` available. +- `spsc_direct.cpp`: the same workload written by hand against + `runtime/spsc_queue.hpp`, with its allocation-free pop. This is the ceiling. +- `run.sh`: builds `queue_bench.cpp` twice from the same CIR, once as + written and once with `--select-queue-impl` in the pipeline, plus + `spsc_direct`, through the pinned toolchain down to a native binary, and + reports the median of `RUNS` runs of each. Set `LLVM_BUILD` to the pinned + toolchain (default `/opt/llvm`, as in the image) and `WTC_BUILD` to this + project's build tree. + +Only throughput is measured so far, not the tail, and runs are not pinned. + +## Measured + +At the pin (`1ebf89d8c5e6`), 2026-10-18, on an Apple M1 Pro (8 cores, +16 GB), macOS, threads not pinned, five runs each, median: + +| variant | items/s | vs baseline | +| --- | ---: | ---: | +| baseline: `wtc::queue` | 2.07M | 1.0x | +| optimized: substituted by `--select-queue-impl` | 7.44M | 3.6x | +| direct: `spsc_queue` by hand | 43.1M | 20.9x | + +The gap between optimized and direct is almost all one allocation per item. +The substitute has to keep `wtc::queue::try_pop`'s interface, which +returns a `std::shared_ptr`, so every pop builds one; the hand-written +consumer uses `try_pop_optional` and allocates nothing. Closing it means +changing what the dialect's pop returns, not the queue. diff --git a/benchmarks/queue_bench.cpp b/benchmarks/queue_bench.cpp new file mode 100644 index 0000000..12d437b --- /dev/null +++ b/benchmarks/queue_bench.cpp @@ -0,0 +1,66 @@ +// One producer, one consumer, kNumItems ints through a global +// wtc::queue. Built twice from this one source by run.sh: once with the +// queue as written, once after --select-queue-impl has substituted +// wtc::runtime::spsc_queue for it. Nothing here names the substitute; +// the shim makes it available. +// +// The consumer checks what it receives: a substitution that drops, +// duplicates or reorders an item fails the run instead of timing it. + +#include "queue.hpp" +#include "thread.hpp" + +#include +#include + +constexpr long kNumItems = 5'000'000; + +wtc::queue g_queue; + +long g_received = 0; +long g_sum = 0; +long g_out_of_order = 0; + +void producer() { + for (long i = 0; i < kNumItems; ++i) { + int value = static_cast(i); + g_queue.push(value); + } +} + +void consumer() { + long expected = 0; + while (g_received < kNumItems) { + auto value = g_queue.try_pop(); + if (value) { + if (*value != expected) + ++g_out_of_order; + g_sum += *value; + ++expected; + ++g_received; + } + } +} + +int main() { + auto start = std::chrono::steady_clock::now(); + + auto producer_thread = wtc::spawn(producer); + auto consumer_thread = wtc::spawn(consumer); + producer_thread.join(); + consumer_thread.join(); + + auto end = std::chrono::steady_clock::now(); + double seconds = std::chrono::duration(end - start).count(); + + long expected_sum = kNumItems * (kNumItems - 1) / 2; + if (g_received != kNumItems || g_sum != expected_sum || g_out_of_order != 0) { + std::printf("FAIL received=%ld sum=%ld (expected %ld) out_of_order=%ld\n", + g_received, g_sum, expected_sum, g_out_of_order); + return 1; + } + + std::printf("items=%ld elapsed_ms=%.3f items_per_sec=%.0f\n", + kNumItems, seconds * 1000.0, static_cast(kNumItems) / seconds); + return 0; +} diff --git a/benchmarks/run.sh b/benchmarks/run.sh new file mode 100755 index 0000000..c22697c --- /dev/null +++ b/benchmarks/run.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# Builds the queue benchmark two ways from the same source -- baseline +# (wtc::queue, lifted and lowered back unchanged) and optimized (the +# same pipeline plus --select-queue-impl) -- and spsc_direct, the +# hand-written ceiling. Runs each RUNS times and reports the median. +# +# LLVM_BUILD the pinned toolchain, built or installed; default /opt/llvm, +# where the toolchain image puts it +# WTC_BUILD this project's build tree; default /build +# RUNS runs per variant; default 5 +set -euo pipefail + +here="$(cd "$(dirname "$0")" && pwd)" +repo="$(dirname "$here")" +LLVM_BUILD="${LLVM_BUILD:-/opt/llvm}" +WTC_BUILD="${WTC_BUILD:-$repo/build}" +RUNS="${RUNS:-5}" +out="$WTC_BUILD/benchmarks" +mkdir -p "$out" + +for tool in clang cir-opt mlir-translate llc; do + if [ ! -x "$LLVM_BUILD/bin/$tool" ]; then + echo "no $tool in $LLVM_BUILD/bin; set LLVM_BUILD to the pinned toolchain" >&2 + exit 1 + fi +done +wtc_opt="$WTC_BUILD/bin/wtc-opt" +[ -x "$wtc_opt" ] || { echo "no wtc-opt in $WTC_BUILD/bin; set WTC_BUILD" >&2; exit 1; } + +# A clang built from source does not know where macOS keeps the SDK, and the +# final link needs the system's C++ runtime either way. +sysroot=() +link_cxx=(c++) +if [ "$(uname)" = Darwin ]; then + sysroot=(-isysroot "$(xcrun --show-sdk-path)") + link_cxx=(xcrun clang++) +fi + +emit_cir() { # + "$LLVM_BUILD/bin/clang" -x c++ -std=c++17 -O1 -I "$repo/shim" "${sysroot[@]}" \ + -fclangir -emit-cir "$1" -o "$2" +} + +# cir-to-llvm leaves CIR types behind in attributes, even for plain clang +# output, so mlir-translate has to be told not to choke on them. +link() { # + "$LLVM_BUILD/bin/cir-opt" "$1" --cir-to-llvm -o "$out/$2.llvm.mlir" + "$LLVM_BUILD/bin/mlir-translate" --allow-unregistered-dialect --mlir-to-llvmir \ + "$out/$2.llvm.mlir" -o "$out/$2.ll" + "$LLVM_BUILD/bin/llc" -O2 -filetype=obj "$out/$2.ll" -o "$out/$2.o" + "${link_cxx[@]}" "$out/$2.o" -o "$out/$2" -lpthread +} + +emit_cir "$here/queue_bench.cpp" "$out/queue_bench.cir" + +"$wtc_opt" "$out/queue_bench.cir" --mlir-print-op-on-diagnostic=false \ + --lift-cir-to-wtc --queue-ownership-analysis --lower-wtc-to-cir \ + --reconcile-unrealized-casts -o "$out/queue_bench_baseline.cir" > /dev/null +link "$out/queue_bench_baseline.cir" queue_bench_baseline + +"$wtc_opt" "$out/queue_bench.cir" --mlir-print-op-on-diagnostic=false \ + --lift-cir-to-wtc --queue-ownership-analysis --select-queue-impl --lower-wtc-to-cir \ + --reconcile-unrealized-casts -o "$out/queue_bench_optimized.cir" > /dev/null +grep -q 'cir.global external @g_queue = #cir.zero : !rec_wtc3A3Aruntime' "$out/queue_bench_optimized.cir" \ + || { echo "--select-queue-impl did not substitute g_queue" >&2; exit 1; } +link "$out/queue_bench_optimized.cir" queue_bench_optimized + +emit_cir "$here/spsc_direct.cpp" "$out/spsc_direct.cir" +link "$out/spsc_direct.cir" spsc_direct + +for bin in queue_bench_baseline queue_bench_optimized spsc_direct; do + rates=() + for _ in $(seq "$RUNS"); do + line="$("$out/$bin")" + rates+=("$(sed -n 's/.*items_per_sec=\([0-9]*\).*/\1/p' <<<"$line")") + done + median="$(printf '%s\n' "${rates[@]}" | sort -n | sed -n "$(( (RUNS + 1) / 2 ))p")" + echo "$bin: median $median items/s over $RUNS runs (${rates[*]})" +done diff --git a/benchmarks/spsc_direct.cpp b/benchmarks/spsc_direct.cpp new file mode 100644 index 0000000..5a5e3d2 --- /dev/null +++ b/benchmarks/spsc_direct.cpp @@ -0,0 +1,61 @@ +// The same workload as queue_bench.cpp, written by hand against the +// substitute with its allocation-free pop. This is the ceiling: the +// compiler cannot beat it, only reach it without the human. + +#include "../runtime/spsc_queue.hpp" +#include "thread.hpp" + +#include +#include + +constexpr long kNumItems = 5'000'000; + +wtc::runtime::spsc_queue g_queue; + +long g_received = 0; +long g_sum = 0; +long g_out_of_order = 0; + +void producer() { + for (long i = 0; i < kNumItems; ++i) { + int value = static_cast(i); + g_queue.push(value); + } +} + +void consumer() { + long expected = 0; + while (g_received < kNumItems) { + auto value = g_queue.try_pop_optional(); + if (value) { + if (*value != expected) + ++g_out_of_order; + g_sum += *value; + ++expected; + ++g_received; + } + } +} + +int main() { + auto start = std::chrono::steady_clock::now(); + + auto producer_thread = wtc::spawn(producer); + auto consumer_thread = wtc::spawn(consumer); + producer_thread.join(); + consumer_thread.join(); + + auto end = std::chrono::steady_clock::now(); + double seconds = std::chrono::duration(end - start).count(); + + long expected_sum = kNumItems * (kNumItems - 1) / 2; + if (g_received != kNumItems || g_sum != expected_sum || g_out_of_order != 0) { + std::printf("FAIL received=%ld sum=%ld (expected %ld) out_of_order=%ld\n", + g_received, g_sum, expected_sum, g_out_of_order); + return 1; + } + + std::printf("items=%ld elapsed_ms=%.3f items_per_sec=%.0f\n", + kNumItems, seconds * 1000.0, static_cast(kNumItems) / seconds); + return 0; +} diff --git a/runtime/spsc_queue.hpp b/runtime/spsc_queue.hpp index 3079936..ab915e1 100644 --- a/runtime/spsc_queue.hpp +++ b/runtime/spsc_queue.hpp @@ -23,8 +23,11 @@ // once it has drained it and seen the next one; publishing `next` is the // producer's last access to a block, so nothing touches a freed block. // -// Measured numbers: not yet. See benchmarks/README.md once they are taken at -// the pinned toolchain. +// Measured, against wtc::queue on one producer and one consumer, at the +// pin on an Apple M1 Pro (benchmarks/README.md): 3.6 times the throughput +// when substituted by the compiler, which keeps try_pop's shared_ptr and so +// allocates once per item; 20.9 times when used by hand through +// try_pop_optional, which does not. #include #include From 3ee7ca2a2c17af083dfae36ef65994359272023b Mon Sep 17 00:00:00 2001 From: zwe Date: Sun, 18 Oct 2026 13:23:32 +0800 Subject: [PATCH 4/4] Check for the substitution without assuming macOS linkage keywords run.sh confirmed the optimized build was substituted by matching "cir.global external @g_queue". On Linux the global is "cir.global external dso_local @g_queue", so the check would have called a correct build unsubstituted. It now matches only the name and the type. Co-Authored-By: Claude Opus 5.5 --- benchmarks/run.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/benchmarks/run.sh b/benchmarks/run.sh index c22697c..38c5e42 100755 --- a/benchmarks/run.sh +++ b/benchmarks/run.sh @@ -61,7 +61,9 @@ link "$out/queue_bench_baseline.cir" queue_bench_baseline "$wtc_opt" "$out/queue_bench.cir" --mlir-print-op-on-diagnostic=false \ --lift-cir-to-wtc --queue-ownership-analysis --select-queue-impl --lower-wtc-to-cir \ --reconcile-unrealized-casts -o "$out/queue_bench_optimized.cir" > /dev/null -grep -q 'cir.global external @g_queue = #cir.zero : !rec_wtc3A3Aruntime' "$out/queue_bench_optimized.cir" \ +# Linkage keywords differ by target (ELF globals are dso_local), so match +# only the name and the type. +grep -qE 'cir\.global .*@g_queue = #cir\.zero : !rec_wtc3A3Aruntime' "$out/queue_bench_optimized.cir" \ || { echo "--select-queue-impl did not substitute g_queue" >&2; exit 1; } link "$out/queue_bench_optimized.cir" queue_bench_optimized