Pin inscount backend (--pin) for deterministic instruction counts

.github/workflows/test.yml

@@ -60,7 +60,7 @@ jobs:
         run: |
           if [ "${{ needs.test-shard.result }}" = "success" ]; then exit 0; else exit 1; fi
 
-  perf-callgrind:
+  perf-instrumentation:
     needs: test
     runs-on: ubuntu-latest
     permissions:
@@ -71,7 +71,7 @@ jobs:
       fail-fast: false
       matrix:
         shard: [0, 1, 2, 3]
-    name: perf-callgrind (shard ${{ matrix.shard }}/4)
+    name: perf-instrumentation (shard ${{ matrix.shard }}/4)
     steps:
       - uses: actions/checkout@v7
       - uses: ./.github/actions/profile-runner
@@ -96,20 +96,31 @@ jobs:
             --passphrase "$RESOURCES_PASSPHRASE" \
             -d tests/resources.tar.xz.asc | tar -xJ -C tests/resources --strip-components=1
 
-      - name: Install Valgrind
-        uses: awalsh128/cache-apt-pkgs-action@v1.6.3
+      - name: Cache instrumentation kit
+        uses: actions/cache@v5
         with:
-          packages: valgrind
-          version: 1.0
-
-      - name: Measure performance (Callgrind instructions)
-        # Timeout 600 (was 300): edit_distance/test_case_5 (~45B Ir)
-        # sits on the 300s wall-clock knife-edge under callgrind with
-        # --jobs 2, and shard rotation after #2 tipped it to
-        # usually-timeout. 2x headroom; genuine hangs still fail.
+          path: ~/instrumentation
+          key: instrumentation-pin-4.4-${{ hashFiles('scripts/ci/setup-instrumentation.sh', 'scripts/ci/instrumentation/inscount.cpp') }}
+
+      - name: Provision instrumentation (Pin 4.4)
+        run: bash scripts/ci/setup-instrumentation.sh
+
+      - name: Measure performance (Pin-only sweep)
+        # Pin-only sweep: --repeat 0 skips the native timed phase, so
+        # each case is measured ONCE under Pin (inscount) with the
+        # LD_PRELOAD alloc shim in the same run; correctness gates on
+        # the instrumented run's stdout. Pin budget 600/repeat: the
+        # heaviest cases (edit_distance) run minutes under
+        # instrumentation with --jobs 2. Pin timeouts degrade to
+        # pin-skipped (skipped cases contribute no BMF rows in this
+        # mode); genuine hangs still fail in the test-shard jobs,
+        # which run natively with --timeout 300.
         run: |
           python3 tools/measure.py --resources tests/resources \
-            --repeat 1 --jobs 2 --timeout 600 \
+            --repeat 0 --jobs 2 \
+            --pin --pin-bin "$INSTRUMENTATION_BIN" \
+            --pin-tool "$INSTRUMENTATION_TOOL" \
+            --pin-repeat 2 --pin-timeout 600 \
             --shard ${{ matrix.shard }}/4 \
             --output perf-results.json --bmf-output bencher-bmf.json \
             --summary-file "$GITHUB_STEP_SUMMARY"
@@ -144,7 +155,7 @@ jobs:
               --key '${{ secrets.BENCHER_API_KEY }}' \
               --branch "${{ github.ref_name }}" \
               --testbed ubuntu-latest-python \
-              --threshold-measure instructions \
+              --threshold-measure pin_instructions \
               --threshold-test percentage \
               --threshold-min-sample-size 10 \
               --threshold-max-sample-size 64 \
@@ -163,8 +174,8 @@ jobs:
             perf-results.json
             bencher-bmf.json
 
-  report:
-    needs: perf-callgrind
+  report-instrumentation:
+    needs: perf-instrumentation
     runs-on: ubuntu-latest
     permissions:
       contents: read

README.md

@@ -51,18 +51,21 @@ tools/build_bundle.sh <resource/shared dir> [passphrase-file]
 
 See `tools/build_bundle.sh` for rotation instructions when using a fresh passphrase.
 
-## Performance Tracking (Callgrind)
+## Performance Tracking (Intel Pin)
 
-`tools/measure.py` measures load-independent algorithmic complexity using Valgrind Callgrind:
+`tools/measure.py` measures load-independent algorithmic complexity using Intel Pin with the vendored inscount tool (`scripts/ci/instrumentation/`), in pin-only mode (`--repeat 0`): each case runs once under Pin, with an `LD_PRELOAD` allocation-counting shim in the same run:
 
-- `instructions` (`Ir` — simulated instruction reads) — completely load-invariant and deterministic across runs and runners.
-- `cpu_time` (user + sys CPU time)
-- `wall_time`
+- `pin_instructions` (executed application instructions under Pin 4.4) — completely load-invariant and deterministic across runs and runners.
+- `allocated_bytes` (cumulative libc allocation in the same Pin run) — the memory signal. It includes Pin's own JIT/VM allocations, so like `pin_instructions` it is only comparable within one Pin version.
+
+Peak RSS is never read from instrumented runs — Pin's own footprint pollutes it — so timed signals (`cpu_time`, `wall_time`, `maxrss`) are absent from pin-only sweeps; the summary shows `-` and BMF omits those rows rather than emitting zeros. Locally, `--repeat N` (N ≥ 1) still runs a native timed phase first (plus a Callgrind `instructions` signal when Valgrind happens to be installed); CI no longer installs Valgrind, so that measure is absent there.
 
 ```sh
-python3 tools/measure.py --resources tests/resources --repeat 1 \
+bash scripts/ci/setup-instrumentation.sh  # one-time: fetch Pin 4.4 kit, build tool
+python3 tools/measure.py --resources tests/resources --repeat 0 \
+  --pin --pin-bin "$INSTRUMENTATION_BIN" --pin-tool "$INSTRUMENTATION_TOOL" \
   --output perf.json --bmf-output bencher.json
 python3 tools/measure.py --self-test
 ```
 
-CI runs correctness across sharded runners in the `test-shard` jobs, aggregated by `test`. Then the `perf-callgrind` job measures instruction counts under Valgrind Callgrind across 4 shards, reporting to Bencher under testbed `ubuntu-latest-python` and measure `instructions` (gated by `BENCHER_API_KEY`). A fan-in `report` job merges all shard results, verifies equality against the plain BMF union (`jq -s 'add'`), and publishes a combined Markdown summary table to the GitHub Actions job summary.
+CI runs correctness across sharded runners in the `test-shard` jobs, aggregated by `test`. Then the `perf-instrumentation` job provisions the cached Pin 4.4 kit and measures instruction counts plus allocated bytes under Pin across 4 shards, reporting to Bencher under testbed `ubuntu-latest-python` and measures `pin_instructions` / `allocated_bytes` (gated by `BENCHER_API_KEY`). A fan-in `report-instrumentation` job merges all shard results, verifies equality against the plain BMF union (`jq -s 'add'`), and publishes a combined Markdown summary table to the GitHub Actions job summary.

scripts/ci/instrumentation/allocshim.c

@@ -0,0 +1,231 @@
+// allocshim: LD_PRELOAD allocation-counting shim (Linux/glibc
+// only) for the python snippet perf sweep.
+//
+// Interposes the libc allocation entry points, sums requested
+// bytes, and a destructor reports the total on fd 2 as:
+//
+//	snip_alloc_total_bytes=N
+//
+// Freed bytes are never subtracted, so the total is the cumulative
+// allocation for the whole process run. Deterministic programs
+// yield deterministic totals; exit-path allocations after the
+// destructor (if any) are missed identically every run, so they
+// shift the total by a constant, never by noise.
+//
+// tools/measure.py builds this file on demand (cc -shared -fPIC)
+// and preloads it in the SAME Pin run as the inscount tool
+// (LD_PRELOAD shim + pin -t inscount -- python ...), so each
+// instrumented run yields both pin_instructions and
+// allocated_bytes. The total therefore includes Pin's own
+// JIT/VM allocations: deterministic within a Pin version, but
+// only comparable within that version (like pin_instructions).
+// Peak RSS is never read from instrumented runs -- Pin's own
+// footprint pollutes it -- so allocated_bytes is the memory
+// signal and maxrss stays null in pin-only sweeps.
+//
+// Ported from the hhh-pa/clojure-code-snippet alloc shim
+// (scripts/ci/go/internal/measure/allocshim.go, embedded C
+// source -- that repo has no standalone allocshim.c file),
+// as a standalone source so the Python harness keeps one
+// obvious build input next to inscount.cpp.
+#define _GNU_SOURCE
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+#include <unistd.h>
+#include <dlfcn.h>
+#include <stdatomic.h>
+#include <sys/types.h>
+#include <errno.h>
+
+static atomic_ullong total_bytes = 0;
+
+/* Bootstrap pool: dlsym(RTLD_NEXT, ...) itself allocates, so
+ * allocations arriving before init completes are served from here
+ * (uncounted) instead of recursing. 64KB dwarfs dlsym's needs. */
+#define POOL_SIZE (64u * 1024u)
+_Alignas(64) static char bootstrap_pool[POOL_SIZE];
+static atomic_size_t pool_used = 0;
+static volatile int resolving = 0;
+static void *(*real_malloc)(size_t);
+static void (*real_free)(void *);
+static void *(*real_calloc)(size_t, size_t);
+static void *(*real_realloc)(void *, size_t);
+static void *(*real_reallocarray)(void *, size_t, size_t);
+static int (*real_posix_memalign)(void **, size_t, size_t);
+static void *(*real_aligned_alloc)(size_t, size_t);
+
+static void *pool_alloc(size_t size) {
+	if (size == 0)
+		size = 1;
+	size = (size + 15) & ~(size_t)15;
+	size_t off = atomic_fetch_add(&pool_used, size);
+	if (off + size > POOL_SIZE)
+		return NULL;
+	return bootstrap_pool + off;
+}
+
+static int in_pool(const void *p) {
+	const char *c = (const char *)p;
+	return c >= bootstrap_pool && c < bootstrap_pool + POOL_SIZE;
+}
+
+static size_t pool_tail(const void *p) {
+	return (size_t)(bootstrap_pool + POOL_SIZE - (const char *)p);
+}
+
+static void resolve_all(void) {
+	real_malloc = dlsym(RTLD_NEXT, "malloc");
+	real_free = dlsym(RTLD_NEXT, "free");
+	real_calloc = dlsym(RTLD_NEXT, "calloc");
+	real_realloc = dlsym(RTLD_NEXT, "realloc");
+	real_reallocarray = dlsym(RTLD_NEXT, "reallocarray");
+	real_posix_memalign = dlsym(RTLD_NEXT, "posix_memalign");
+	real_aligned_alloc = dlsym(RTLD_NEXT, "aligned_alloc");
+}
+
+/* True once the real entry points are resolved. Reentrant calls
+ * during resolution (dlsym allocates) take the pool path. */
+static int ensure_init(void) {
+	if (real_malloc)
+		return 1;
+	if (resolving)
+		return 0;
+	resolving = 1;
+	resolve_all();
+	resolving = 0;
+	return real_malloc != 0;
+}
+
+__attribute__((constructor)) static void shim_init(void) {
+	resolving = 1;
+	resolve_all();
+	resolving = 0;
+}
+
+void *malloc(size_t size) {
+	if (!ensure_init())
+		return pool_alloc(size);
+	void *p = real_malloc(size);
+	if (p)
+		atomic_fetch_add(&total_bytes, (unsigned long long)size);
+	return p;
+}
+
+void free(void *p) {
+	if (!p || in_pool(p))
+		return;
+	if (!ensure_init())
+		return;
+	real_free(p);
+}
+
+void *calloc(size_t n, size_t size) {
+	if (!ensure_init()) {
+		if (size != 0 && n > POOL_SIZE / size)
+			return NULL;
+		void *p = pool_alloc(n * size);
+		if (p)
+			memset(p, 0, n * size);
+		return p;
+	}
+	void *p = real_calloc(n, size);
+	if (p)
+		atomic_fetch_add(&total_bytes, (unsigned long long)n * (unsigned long long)size);
+	return p;
+}
+
+void *realloc(void *ptr, size_t size) {
+	if (ptr && in_pool(ptr)) {
+		/* Bootstrap block outliving init: re-home it. The old
+		 * size is unknown, so the copy clamps to the pool tail;
+		 * glibc never reallocs its bootstrap blocks, so this
+		 * path is unreachable in practice. */
+		void *p = ensure_init() ? real_malloc(size ? size : 1) : pool_alloc(size);
+		if (p) {
+			size_t tail = pool_tail(ptr);
+			memcpy(p, ptr, size < tail ? size : tail);
+			if (real_malloc)
+				atomic_fetch_add(&total_bytes, (unsigned long long)size);
+		}
+		return p;
+	}
+	if (!ensure_init())
+		return pool_alloc(size);
+	void *p = real_realloc(ptr, size);
+	if (p)
+		atomic_fetch_add(&total_bytes, (unsigned long long)size);
+	return p;
+}
+
+void *reallocarray(void *ptr, size_t n, size_t size) {
+	if (size != 0 && n > (size_t)-1 / size) {
+		errno = ENOMEM;
+		return NULL;
+	}
+	if (real_reallocarray || ensure_init()) {
+		if (real_reallocarray && !(ptr && in_pool(ptr))) {
+			void *p = real_reallocarray(ptr, n, size);
+			if (p)
+				atomic_fetch_add(&total_bytes, (unsigned long long)n * (unsigned long long)size);
+			return p;
+		}
+	}
+	return realloc(ptr, n * size);
+}
+
+int posix_memalign(void **memptr, size_t alignment, size_t size) {
+	if (!ensure_init() || !real_posix_memalign) {
+		if (alignment < sizeof(void *) || (alignment & (alignment - 1)) != 0)
+			return 22; /* EINVAL */
+		if (size + alignment < size)
+			return 12; /* ENOMEM */
+		void *raw = pool_alloc(size + alignment);
+		if (!raw)
+			return 12; /* ENOMEM */
+		uintptr_t a = ((uintptr_t)raw + alignment - 1) & ~(uintptr_t)(alignment - 1);
+		*memptr = (void *)a;
+		return 0;
+	}
+	int r = real_posix_memalign(memptr, alignment, size);
+	if (r == 0)
+		atomic_fetch_add(&total_bytes, (unsigned long long)size);
+	return r;
+}
+
+void *aligned_alloc(size_t alignment, size_t size) {
+	if (!ensure_init() || !real_aligned_alloc) {
+		void *p = NULL;
+		int r = posix_memalign(&p, alignment, size);
+		if (r != 0) {
+			errno = r;
+			return NULL;
+		}
+		return p;
+	}
+	void *p = real_aligned_alloc(alignment, size);
+	if (p)
+		atomic_fetch_add(&total_bytes, (unsigned long long)size);
+	return p;
+}
+
+__attribute__((destructor)) static void shim_fini(void) {
+	static const char marker[] = "snip_alloc_total_bytes=";
+	char buf[64];
+	unsigned long long v = atomic_load(&total_bytes);
+	size_t n = sizeof(marker) - 1;
+	memcpy(buf, marker, n);
+	char rev[24];
+	int r = 0;
+	if (v == 0)
+		rev[r++] = '0';
+	while (v > 0) {
+		rev[r++] = (char)('0' + (v % 10));
+		v /= 10;
+	}
+	while (r > 0)
+		buf[n++] = rev[--r];
+	buf[n++] = '\n';
+	ssize_t w = write(2, buf, n);
+	(void)w;
+}

scripts/ci/instrumentation/inscount.cpp

@@ -0,0 +1,57 @@
+// inscount: minimal Intel Pin tool counting executed application
+// instructions for the python snippet perf sweep.
+//
+// Instruments at TRACE/BBL granularity: each basic block gets one
+// analysis call adding the block's static instruction count to a
+// process-wide total (all threads, all application images; the Pin
+// VM itself is never instrumented). At shutdown the tool prints a
+// single line:
+//
+//	Count N
+//
+// to the file named by the -o knob (default inscount.out).
+// tools/measure.py parses that line (parse_pin_count) and records
+// it as pin_instructions. Build out of the kit with:
+//
+//	make PIN_ROOT=<kit> -C <this dir>
+//
+// which drops obj-intel64/inscount.so next to this file.
+#include <cstdio>
+#include "pin.H"
+
+static UINT64 g_total = 0;
+static FILE *g_out = nullptr;
+static KNOB<std::string> KnobOut(KNOB_MODE_WRITEONCE, "pintool", "o", "inscount.out",
+                                 "output file for the Count line");
+
+static VOID AddBlock(UINT32 n) { g_total += n; }
+
+static VOID Trace(TRACE trace, VOID *) {
+	for (BBL bbl = TRACE_BblHead(trace); BBL_Valid(bbl); bbl = BBL_Next(bbl)) {
+		BBL_InsertCall(bbl, IPOINT_BEFORE, (AFUNPTR)AddBlock, IARG_UINT32, BBL_NumIns(bbl),
+		               IARG_END);
+	}
+}
+
+static VOID Fini(INT32, VOID *) {
+	if (g_out != nullptr) {
+		fprintf(g_out, "Count %llu\n", (unsigned long long)g_total);
+		fclose(g_out);
+		g_out = nullptr;
+	}
+}
+
+int main(int argc, char *argv[]) {
+	PIN_InitSymbols();
+	if (PIN_Init(argc, argv)) {
+		return 1;
+	}
+	g_out = fopen(KnobOut.Value().c_str(), "w");
+	if (g_out == nullptr) {
+		return 1;
+	}
+	TRACE_AddInstrumentFunction(Trace, nullptr);
+	PIN_AddFiniFunction(Fini, nullptr);
+	PIN_StartProgram();
+	return 0;
+}